stageflow 0.39.0 → 0.42.0

package/dist/mcp/server.js

@@ -49,6 +49,7 @@ function toolDepsFrom(deps) {
         ...(deps.agentConversationBackend !== undefined
             ? { agentConversationBackend: deps.agentConversationBackend }
             : {}),
+        ...(deps.evalRunManager !== undefined ? { evalRunManager: deps.evalRunManager } : {}),
     };
 }
 function attachBusNotifications(server, bus) {

package/dist/mcp/tools.js

@@ -5,6 +5,7 @@ import { registerControlTools } from "./controlTools.js";
 import { registerProjectMcpTools } from "./projectMcpTools.js";
 import { registerPipelineAuthorTools } from "./pipelineAuthorTools.js";
 import { registerProviderTools } from "./providerTools.js";
+import { registerEvalTools } from "./evalTools.js";
 import { registerTriggerTools } from "./triggerTools.js";
 import { textResult } from "./toolResults.js";
 import { DEFAULT_TIMEOUT_MS, waitRun } from "./waitRun.js";
@@ -19,6 +20,7 @@ export function registerMcpTools(server, deps) {
     registerProviderTools(server, deps);
     registerProjectMcpTools(server, deps);
     registerTriggerTools(server, deps);
+    registerEvalTools(server, deps);
     registerPipelineAuthorTools(server, deps);
     registerAgentTools(server, deps);
     server.registerTool("wait_run", {

package/dist/models/response.d.ts

@@ -1,11 +1,16 @@
 import type { RunStore } from "../runstore/port.js";
 import { listModelsMultiProject } from "../config/multiProjectCatalog.js";
-import { type CatalogProvider } from "./catalog.js";
+import { type CatalogModel, type CatalogProvider } from "./catalog.js";
+export type ModelsResponseProvider = Omit<CatalogProvider, "models"> & {
+    models: Array<CatalogModel & {
+        judge_capable: boolean;
+    }>;
+};
 export type ModelsResponse = {
     /** Catalog ids plus ids pipelines already use, sorted. Back-compatible with the old flat list. */
     models: string[];
     /** Connected providers with their models. */
-    providers: CatalogProvider[];
+    providers: ModelsResponseProvider[];
     /** The model a new chat, stage or agent starts on. */
     default: string;
     fetchedAt: string;

package/dist/models/response.js

@@ -1,4 +1,5 @@
 import { listModelsMultiProject } from "../config/multiProjectCatalog.js";
+import { judgeCapableModels } from "../evals/judgeProviders.js";
 import { getModelCatalog } from "./catalog.js";
 /** The shared answer for GET /api/models, the MCP `list_models` tool and every picker. */
 export async function buildModelsResponse(options) {
@@ -9,12 +10,16 @@ export async function buildModelsResponse(options) {
         catalog.defaultModel(),
     ]);
     const listed = new Set(snapshot.models);
+    const judgeCapable = new Set(judgeCapableModels(snapshot.providers.flatMap((p) => p.models.map((m) => m.id))));
     const unlisted = used.models.filter((id) => !listed.has(id));
     return {
         used,
         response: {
             models: [...new Set([...snapshot.models, ...used.models])].sort((a, b) => a.localeCompare(b)),
-            providers: snapshot.providers,
+            providers: snapshot.providers.map((provider) => ({
+                ...provider,
+                models: provider.models.map((model) => ({ ...model, judge_capable: judgeCapable.has(model.id) })),
+            })),
             default: defaultModel,
             fetchedAt: snapshot.fetchedAt,
             unlisted,

package/dist/operatorAgent/agentAuthorContext.d.ts

@@ -10,7 +10,7 @@ export type AgentAuthorContext = {
     projectRoot?: string;
     draft: AgentAuthorDraft;
 };
-export declare const AGENT_AUTHOR_GREETING = "What should this agent do? Tell me the job, the tools it needs, and anything it must never do without asking.";
+export declare const AGENT_AUTHOR_GREETING = "Who is this agent for, and what does a good result look like? I'll ask a few questions before I fill the config.";
 export declare function emptyAgentAuthorDraft(): AgentAuthorDraft;
 export declare function isAgentAuthorContext(value: unknown): value is AgentAuthorContext;
 export declare function readAgentAuthorDraft(context: unknown): AgentAuthorDraft;

package/dist/operatorAgent/agentAuthorContext.js

@@ -1,4 +1,4 @@
-export const AGENT_AUTHOR_GREETING = "What should this agent do? Tell me the job, the tools it needs, and anything it must never do without asking.";
+export const AGENT_AUTHOR_GREETING = "Who is this agent for, and what does a good result look like? I'll ask a few questions before I fill the config.";
 export function emptyAgentAuthorDraft() {
     return {};
 }

package/dist/operatorAgent/piModel.js

@@ -11,7 +11,7 @@ import path from "node:path";
 import { defineTool, ModelRuntime, resolveCliModel, SessionManager, SettingsManager, } from "@earendil-works/pi-coding-agent";
 import { Type } from "typebox";
 import { createPiAgentSession, createSealedResourceLoader, resolveWorkshopToolNames, } from "../agent/piSessionFactory.js";
-import { bindCursorApiKey, cursorBridgePrompt } from "../agent/cursorProvider.js";
+import { bindCursorApiKey, cursorBridgePrompt, isCursorModelRef } from "../agent/cursorProvider.js";
 import { findProviderSupport } from "../agent/providerSupport.js";
 import "../agent/cursorProvider.js";
 import { logger as rootLogger } from "../logging/logger.js";
@@ -19,7 +19,7 @@ import { globalStageflowHome } from "../project/globalHome.js";
 import { isUsableAuthFile, resolveCredentialBinding, } from "../runtime/credentialBinding.js";
 import { resolveWorkshopModel } from "../workshop/modelSettings.js";
 import { createOperatorAgentHost, invokeProfileTool, } from "./host.js";
-import { createAgentAuthorProfile } from "./profiles/agentAuthor.js";
+import { AGENT_AUTHOR_PROFILE_ID, agentAuthorBridgeHint, createAgentAuthorProfile, formatAgentAuthorTurnPrompt, } from "./profiles/agentAuthor.js";
 import { createWorkshopAuthorProfile, } from "./profiles/workshopAuthor.js";
 import { readDraftFromContext } from "./draftContext.js";
 export function sessionNeedsExtensionReload(loaded, needed) {
@@ -434,7 +434,7 @@ export function createPiOperatorAgentModel(options) {
             syncAgentMessages(state.handle.session, state.handle.sessionManager);
             state.replayed = true;
         },
-        async complete({ profile, message, tools, onDelta, onActivity, modelId }) {
+        async complete({ profile, message, contextSnapshot, tools, onDelta, onActivity, modelId }) {
             let state;
             try {
                 state = await ensureBound(profile, tools, modelId);
@@ -472,8 +472,21 @@ export function createPiOperatorAgentModel(options) {
                     }
                 }
             });
+            const toolNames = profile.tools.map((tool) => tool.name);
+            let prompt = message;
+            if (profile.id === AGENT_AUTHOR_PROFILE_ID) {
+                prompt = formatAgentAuthorTurnPrompt(message, contextSnapshot);
+                prompt = `${prompt}\n\n${agentAuthorBridgeHint(toolNames)}`;
+                if (modelId && isCursorModelRef(modelId) && toolNames.length > 0) {
+                    const aliasLines = toolNames.map((name) => `- pi__${name} (playbook name: ${name})`);
+                    prompt = `${prompt}\n${aliasLines.join("\n")}`;
+                }
+            }
+            else {
+                prompt = cursorBridgePrompt(prompt, modelId, toolNames);
+            }
             try {
-                await state.handle.session.prompt(cursorBridgePrompt(message, modelId, profile.tools.map((tool) => tool.name)));
+                await state.handle.session.prompt(prompt);
             }
             catch (err) {
                 unsubscribe();

package/dist/operatorAgent/profiles/agentAuthor.d.ts

@@ -1,9 +1,11 @@
 import { type AgentAuthorContext, type AgentAuthorDraft } from "../agentAuthorContext.js";
 import type { OperatorAgentProfile } from "../types.js";
 export declare const AGENT_AUTHOR_PROFILE_ID = "agent-author";
+export declare function formatAgentAuthorTurnPrompt(message: string, contextSnapshot: unknown): string;
+export declare function agentAuthorBridgeHint(toolNames: readonly string[]): string;
 /** Objects merge key by key; arrays and scalars replace; `null` removes the key. */
 export declare function mergeDraftPatch(draft: AgentAuthorDraft, patch: Record<string, unknown>): AgentAuthorDraft;
-export declare const AGENT_AUTHOR_TOOL_NAMES: readonly ["read_project_file", "list_project_files", "list_mcp_servers", "update_agent_draft"];
+export declare const AGENT_AUTHOR_TOOL_NAMES: readonly ["read_agent_draft", "read_project_file", "list_project_files", "list_mcp_servers", "list_author_catalog", "update_agent_draft"];
 export declare function createAgentAuthorProfile(): OperatorAgentProfile;
 export declare function createAgentAuthorContext(input: {
     projectRoot: string;

package/dist/operatorAgent/profiles/agentAuthor.js

@@ -1,9 +1,15 @@
 import { readdir, readFile } from "node:fs/promises";
 import path from "node:path";
+import { parse as parseYaml } from "yaml";
 import { agentAuthorContextAdapter, emptyAgentAuthorDraft, isAgentAuthorContext, readAgentAuthorDraft, } from "../agentAuthorContext.js";
 import { AGENT_ID_PATTERN, parseAgentDefinition } from "../../config/loadAgent.js";
 import { listProjectMcpCatalog } from "../../config/resolveStageMcpServers.js";
 import { AGENT_AUTHOR_PLAYBOOK } from "./agentAuthorPlaybook.js";
+import { authorReadRefusal } from "./authorReadable.js";
+import { connectionsFilePath } from "../../channels/registry.js";
+import { loadConnectionsFile } from "../../channels/config.js";
+import { listSkills } from "../../config/listSkills.js";
+import { globalStageflowHome } from "../../project/globalHome.js";
 export const AGENT_AUTHOR_PROFILE_ID = "agent-author";
 const MAX_READ_CHARS = 32_000;
 let proposalSeq = 0;
@@ -38,6 +44,60 @@ function stringArg(args, key) {
 function isPlainObject(value) {
     return value !== null && typeof value === "object" && !Array.isArray(value);
 }
+function coercePlainObject(value) {
+    if (isPlainObject(value))
+        return value;
+    if (typeof value !== "string")
+        return null;
+    const trimmed = value.trim();
+    if (!trimmed)
+        return null;
+    try {
+        const parsed = JSON.parse(trimmed);
+        if (isPlainObject(parsed))
+            return parsed;
+    }
+    catch {
+        /* not JSON */
+    }
+    try {
+        const parsed = parseYaml(trimmed);
+        if (isPlainObject(parsed))
+            return parsed;
+    }
+    catch {
+        /* not YAML */
+    }
+    return null;
+}
+export function formatAgentAuthorTurnPrompt(message, contextSnapshot) {
+    const draft = readAgentAuthorDraft(contextSnapshot);
+    const draftJson = JSON.stringify(draft, null, 2);
+    return [
+        "Current agent draft JSON. Ask until the job is clear. Call update_agent_draft only when you draft or revise; do not reply with YAML instead:",
+        draftJson,
+        "",
+        "Operator message:",
+        message.trim(),
+    ].join("\n");
+}
+export function agentAuthorBridgeHint(toolNames) {
+    return [
+        "Agent author: a thin first request is not a spec. Ask what you still need, or offer one suggestion when the answer is vague. Do not call update_agent_draft until they confirm the job or tell you to go ahead.",
+        "When you do change the draft, call update_agent_draft before you finish. Chat YAML does not update the panel.",
+        "You may read only docs/, policies/, AGENTS.md, and README.md. Do not grant a connection, MCP server, or skill that list_author_catalog or list_mcp_servers did not return. Do not set tools.browser.",
+        `Tools: ${toolNames.join(", ")}`,
+    ].join("\n");
+}
+function patchFromToolArgs(args) {
+    const patch = coercePlainObject(args.patch);
+    if (patch)
+        return patch;
+    const { summary: _summary, patch: _patch, ...rest } = args;
+    if (Object.keys(rest).length === 0)
+        return null;
+    return coercePlainObject(rest) ?? (isPlainObject(rest) ? rest : null);
+}
 /** Objects merge key by key; arrays and scalars replace; `null` removes the key. */
 export function mergeDraftPatch(draft, patch) {
     const next = structuredClone(draft);
@@ -75,6 +135,9 @@ const readProjectFile = {
         const rejected = rejectEscape(root, rel);
         if (rejected)
             return { ok: false, content: null, error: rejected };
+        const docsOnly = authorReadRefusal(rel);
+        if (docsOnly)
+            return { ok: false, content: null, error: docsOnly };
         try {
             const bytes = await readFile(path.resolve(root, rel));
             if (bytes.includes(0))
@@ -95,32 +158,61 @@ const readProjectFile = {
 };
 const listProjectFiles = {
     name: "list_project_files",
-    description: "List readable project files: AGENTS.md when present, and YAML files under agents/. Does not write.",
+    description: "List documentation the author may read: docs/ and policies/ (.md and .txt), plus AGENTS.md and README.md at the project root. Does not list source code.",
     async handler(_args, ctx) {
         const root = projectRootOf(ctx);
         if (!root)
             return { ok: false, content: null, error: "project root is not set" };
         const files = [];
-        try {
-            await readFile(path.join(root, "AGENTS.md"));
-            files.push("AGENTS.md");
-        }
-        catch {
-            /* absent */
+        for (const name of ["AGENTS.md", "README.md"]) {
+            try {
+                await readFile(path.join(root, name));
+                files.push(name);
+            }
+            catch {
+                /* absent */
+            }
         }
-        try {
-            const entries = await readdir(path.join(root, "agents"), { withFileTypes: true });
+        async function walk(rel) {
+            let entries;
+            try {
+                entries = await readdir(path.join(root, rel), { withFileTypes: true });
+            }
+            catch {
+                return;
+            }
             for (const entry of entries) {
-                if (entry.isFile() && /\.ya?ml$/i.test(entry.name))
-                    files.push(`agents/${entry.name}`);
+                if (entry.name.startsWith(".") || entry.name === "node_modules")
+                    continue;
+                const child = `${rel}/${entry.name}`;
+                if (entry.isDirectory())
+                    await walk(child);
+                else if (!authorReadRefusal(child))
+                    files.push(child);
             }
         }
-        catch {
-            /* no agents dir */
-        }
+        await walk("docs");
+        await walk("policies");
+        files.sort();
         return { ok: true, content: { files } };
     },
 };
+const readAgentDraft = {
+    name: "read_agent_draft",
+    description: "Read the current unsaved agent draft (agents/<id>.agent.yaml shape). Use before update_agent_draft when you need the latest fields.",
+    handler(_args, ctx) {
+        const draft = readAgentAuthorDraft(ctx.getContext());
+        const context = ctx.getContext();
+        const projectRoot = isAgentAuthorContext(context) ? context.projectRoot : undefined;
+        return {
+            ok: true,
+            content: {
+                ...(projectRoot ? { projectRoot } : {}),
+                draft,
+            },
+        };
+    },
+};
 const listMcpServers = {
     name: "list_mcp_servers",
     description: "List MCP server names declared in the project's .mcp.json. Use these names under tools.mcp. Does not connect.",
@@ -132,6 +224,32 @@ const listMcpServers = {
         return { ok: true, content: { status: catalog.status, servers: catalog.servers.map((server) => server.name) } };
     },
 };
+const listAuthorCatalog = {
+    name: "list_author_catalog",
+    description: "List host connections (id and provider only) and installed skill names. No secrets. Use these ids in channel.connection_id and tools.skills. If a name is not listed, do not grant it.",
+    async handler(_args, ctx) {
+        const root = projectRootOf(ctx);
+        if (!root)
+            return { ok: false, content: null, error: "project root is not set" };
+        let connections = [];
+        try {
+            const loaded = loadConnectionsFile(connectionsFilePath(root));
+            connections = loaded.connections.map((entry) => ({ id: entry.id, provider: entry.provider }));
+        }
+        catch {
+            connections = [];
+        }
+        let skills = [];
+        try {
+            const catalog = await listSkills({ cwd: root, agentDir: path.join(globalStageflowHome(), "workshop", "agent") });
+            skills = catalog.skills.map((skill) => skill.name);
+        }
+        catch {
+            skills = [];
+        }
+        return { ok: true, content: { connections, skills } };
+    },
+};
 const updateAgentDraft = {
     name: "update_agent_draft",
     description: [
@@ -140,9 +258,9 @@ const updateAgentDraft = {
         "The result lists validation issues; fix them in a follow-up patch. Does not write YAML.",
     ].join(" "),
     handler(args, ctx) {
-        const patch = args.patch;
-        if (!isPlainObject(patch))
-            return { ok: false, content: null, error: "patch must be an object" };
+        const patch = patchFromToolArgs(args);
+        if (!patch)
+            return { ok: false, content: null, error: "patch must be an object (JSON or YAML)" };
         const currentContext = ctx.getContext();
         const current = readAgentAuthorDraft(currentContext);
         const next = mergeDraftPatch(current, patch);
@@ -172,14 +290,21 @@ const updateAgentDraft = {
         };
     },
 };
-export const AGENT_AUTHOR_TOOL_NAMES = ["read_project_file", "list_project_files", "list_mcp_servers", "update_agent_draft"];
+export const AGENT_AUTHOR_TOOL_NAMES = [
+    "read_agent_draft",
+    "read_project_file",
+    "list_project_files",
+    "list_mcp_servers",
+    "list_author_catalog",
+    "update_agent_draft",
+];
 export function createAgentAuthorProfile() {
     return {
         id: AGENT_AUTHOR_PROFILE_ID,
         title: "Agent author",
         playbook: AGENT_AUTHOR_PLAYBOOK,
-        greeting: "What should this agent do?",
-        tools: [readProjectFile, listProjectFiles, listMcpServers, updateAgentDraft],
+        greeting: "Who is this agent for, and what does a good result look like? I'll ask a few questions before I fill the config.",
+        tools: [readAgentDraft, readProjectFile, listProjectFiles, listMcpServers, listAuthorCatalog, updateAgentDraft],
         contextAdapter: agentAuthorContextAdapter(),
     };
 }

package/dist/operatorAgent/profiles/agentAuthorPlaybook.d.ts

@@ -1 +1 @@
-export declare const AGENT_AUTHOR_PLAYBOOK = "You are the Stageflow Agent author. You draft one sealed project agent with the operator. You are not a task runner, and you do not write files.\n\n## How you talk\n- Short turns, plain words, present tense. One to four sentences.\n- Do not narrate tool names. Say what changed in the config.\n- The config panel shows the draft. The operator can edit it directly; your next patch builds on what they changed.\n\n## Tools\n- read_project_file and list_project_files are read-only. Use them when the operator points at the project, an existing agent file, or AGENTS.md.\n- list_mcp_servers names the servers in .mcp.json. Only grant MCP servers that exist there; if the operator wants one that isn't there, say so.\n- update_agent_draft applies a patch to the draft immediately. Call it as soon as you know enough. Do not wait for Keep. If it reports issues, fix them with another patch.\n- Never use bash, write, or edit. Save is the operator's button, not a tool.\n\n## The draft (agents/<id>.agent.yaml schema)\n- id (snake_case, starts with a letter; derived from name when omitted), name, description, greeting, instructions (required).\n- model: { backend: pi | claude, id?: \"provider/model\", thinking: low | medium | high }. Leave id out to use the project default unless they name one, and say so.\n- tools: { builtin: [read | write | shell] (whole groups), builtin_tools: [read | grep | find | ls | write | edit | bash] (single tools), stageflow: { groups: [...], tools: [...] }, mcp: { <server>: [tool names] or [\"*\"] }, skills: [names], browser: bool }.\n- approvals: { ask_first: [tool ids], approvers: { operators: bool, channel_ids: [] }, timeout_minutes, on_timeout: skip | deny }.\n  Tool ids: builtin names (read, grep, find, ls, write, edit, bash), \"<server>.<tool>\" for MCP, \"agent:<id>\" for sub-agents. ask_first may only name granted tools.\n- sub_agents: [agent ids], observability: { capture: off | metadata | full }, limits: { session_minutes, max_turns, cost_usd | null, long_context: summarize | truncate }, guardrails: { block_patterns: [JavaScript regex] }, schedule: { cron, timezone?, message? }.\n\n## Defaults\n- A new agent saves disabled and starts sealed: read only. Write and shell stay off unless the job needs them.\n- Anything that sends, posts, writes, merges or deletes goes in approvals.ask_first.\n- Capture stays metadata unless they ask for full content.\n\n## After an edit\n- One sentence on what the agent will do now. Mention any assumption or open question. Do not paste YAML unless they ask.\n";
+export declare const AGENT_AUTHOR_PLAYBOOK = "You are the Stageflow Agent author. You help an operator design one sealed project agent that will actually be useful. You are not a task runner, and you do not write files.\n\nA first message is a starting point, not a spec. Your job is to understand the use case before you fill the config.\n\n## How you talk\n- Short turns, plain words, present tense.\n- Two or three questions a turn. Do not dump a questionnaire.\n- Do not narrate tool names.\n- The config panel shows the draft. The operator can edit it directly; your next patch builds on what they changed.\n\n## Understand the job first\nDo not call update_agent_draft while the brief is still thin. Asking questions is a complete turn.\n\nLearn, in whatever order the gaps are:\n- Who uses it, and in what situation (a person in chat, a schedule, a channel).\n- What a good result looks like, and what a bad one looks like.\n- What it may read, and what it may change.\n- What it must never do without asking.\n- Where the source of truth lives. You may read only documentation: docs/, policies/, AGENTS.md, and README.md (.md and .txt). You cannot read source code. If they point at code, say you only read documentation and ask which doc describes the behavior.\n\nUse what they already said. Ask only what would change the draft.\n\nWhen an answer is vague, confused, or contradicts an earlier one:\n- Restate what you heard in one sentence.\n- Offer one concrete suggestion they can accept or correct (a name, a boundary, a tool grant, or a refusal).\n- Ask which part is wrong.\n\nWhen you can describe the job in two or three sentences, say that summary and ask them to confirm or correct it. Do not draft before they confirm, unless they tell you to go ahead anyway.\n\nWhen the brief is thin, offer one concrete agent: a name, what it will do, what it will not do, and which tools. When they accept that suggestion, call update_agent_draft in that same turn. Do not wait for another question.\n\nIf they say \"just build it\" before the brief is solid:\n- State the assumptions in one or two sentences.\n- Offer the narrowest useful version.\n- Draft only after they accept those assumptions, or if they insist a second time. Name every assumption in the reply after you draft.\n\n## When you draft\n- Call update_agent_draft in that turn. Chat text does not update the panel. Do not paste YAML unless they ask to see it.\n- instructions must describe this job: who it serves, how it decides, what it refuses, and what a good reply contains. Do not write a generic \"be helpful\" prompt.\n- Grant only the tools the job needs. A new agent stays sealed: read only, until the job needs write or shell.\n- Anything that sends, posts, writes, merges, or deletes goes in approvals.ask_first.\n- Leave model.id unset unless they name a model, and say you did that.\n- Capture stays metadata unless they ask for full content.\n- If update_agent_draft reports issues, fix them with another patch before you finish.\n\nAfter the patch, one sentence on what the agent will do. Then one sentence naming anything they asked for that you did not turn on: a doc you could not read, a channel or skill that list_author_catalog or list_mcp_servers did not return, or browsing. Do not set tools.browser. Do not invent a connection id, MCP server, or skill name.\n\n## Tools\n- read_agent_draft returns the current unsaved draft JSON. Each turn also includes that draft in the operator message.\n- read_project_file and list_project_files are read-only and documentation only: docs/, policies/, AGENTS.md, and README.md, and only .md or .txt. A refused path means you tell them you cannot read code or other files.\n- list_mcp_servers names the servers in .mcp.json. list_author_catalog names host connections (id and provider) and installed skills. Grant only names those tools return. If they want one that is not listed, say it is not connected or not installed.\n- update_agent_draft applies a patch immediately. Objects merge key by key, arrays replace, null removes a key. Do not wait for Keep.\n- Never use bash, write, or edit. Save is the operator's button, not a tool.\n\n## The draft (agents/<id>.agent.yaml schema)\n- id (snake_case, starts with a letter; derived from name when omitted), name, description, greeting, instructions (required).\n- model: { backend: pi | claude, id?: \"provider/model\", thinking: low | medium | high }.\n- tools: { builtin: [read | write | shell] (whole groups), builtin_tools: [read | grep | find | ls | write | edit | bash] (single tools), stageflow: { groups: [...], tools: [...] }, mcp: { <server>: [tool names] or [\"*\"] }, skills: [names], browser: bool }.\n- approvals: { ask_first: [tool ids], approvers: { operators: bool, channel_ids: [] }, timeout_minutes, on_timeout: skip | deny }.\n  Tool ids: builtin names (read, grep, find, ls, write, edit, bash), \"<server>.<tool>\" for MCP, \"agent:<id>\" for sub-agents. ask_first may only name granted tools.\n- sub_agents: [agent ids], observability: { capture: off | metadata | full }, limits: { session_minutes, max_turns, cost_usd | null, long_context: summarize | truncate }, guardrails: { block_patterns: [JavaScript regex] }, schedule: { cron, timezone?, message? }.\n";

package/dist/operatorAgent/profiles/agentAuthorPlaybook.js

@@ -1,29 +1,62 @@
-export const AGENT_AUTHOR_PLAYBOOK = `You are the Stageflow Agent author. You draft one sealed project agent with the operator. You are not a task runner, and you do not write files.
+export const AGENT_AUTHOR_PLAYBOOK = `You are the Stageflow Agent author. You help an operator design one sealed project agent that will actually be useful. You are not a task runner, and you do not write files.
+
+A first message is a starting point, not a spec. Your job is to understand the use case before you fill the config.
 
 ## How you talk
-- Short turns, plain words, present tense. One to four sentences.
-- Do not narrate tool names. Say what changed in the config.
+- Short turns, plain words, present tense.
+- Two or three questions a turn. Do not dump a questionnaire.
+- Do not narrate tool names.
 - The config panel shows the draft. The operator can edit it directly; your next patch builds on what they changed.
 
+## Understand the job first
+Do not call update_agent_draft while the brief is still thin. Asking questions is a complete turn.
+
+Learn, in whatever order the gaps are:
+- Who uses it, and in what situation (a person in chat, a schedule, a channel).
+- What a good result looks like, and what a bad one looks like.
+- What it may read, and what it may change.
+- What it must never do without asking.
+- Where the source of truth lives. You may read only documentation: docs/, policies/, AGENTS.md, and README.md (.md and .txt). You cannot read source code. If they point at code, say you only read documentation and ask which doc describes the behavior.
+
+Use what they already said. Ask only what would change the draft.
+
+When an answer is vague, confused, or contradicts an earlier one:
+- Restate what you heard in one sentence.
+- Offer one concrete suggestion they can accept or correct (a name, a boundary, a tool grant, or a refusal).
+- Ask which part is wrong.
+
+When you can describe the job in two or three sentences, say that summary and ask them to confirm or correct it. Do not draft before they confirm, unless they tell you to go ahead anyway.
+
+When the brief is thin, offer one concrete agent: a name, what it will do, what it will not do, and which tools. When they accept that suggestion, call update_agent_draft in that same turn. Do not wait for another question.
+
+If they say "just build it" before the brief is solid:
+- State the assumptions in one or two sentences.
+- Offer the narrowest useful version.
+- Draft only after they accept those assumptions, or if they insist a second time. Name every assumption in the reply after you draft.
+
+## When you draft
+- Call update_agent_draft in that turn. Chat text does not update the panel. Do not paste YAML unless they ask to see it.
+- instructions must describe this job: who it serves, how it decides, what it refuses, and what a good reply contains. Do not write a generic "be helpful" prompt.
+- Grant only the tools the job needs. A new agent stays sealed: read only, until the job needs write or shell.
+- Anything that sends, posts, writes, merges, or deletes goes in approvals.ask_first.
+- Leave model.id unset unless they name a model, and say you did that.
+- Capture stays metadata unless they ask for full content.
+- If update_agent_draft reports issues, fix them with another patch before you finish.
+
+After the patch, one sentence on what the agent will do. Then one sentence naming anything they asked for that you did not turn on: a doc you could not read, a channel or skill that list_author_catalog or list_mcp_servers did not return, or browsing. Do not set tools.browser. Do not invent a connection id, MCP server, or skill name.
+
 ## Tools
-- read_project_file and list_project_files are read-only. Use them when the operator points at the project, an existing agent file, or AGENTS.md.
-- list_mcp_servers names the servers in .mcp.json. Only grant MCP servers that exist there; if the operator wants one that isn't there, say so.
-- update_agent_draft applies a patch to the draft immediately. Call it as soon as you know enough. Do not wait for Keep. If it reports issues, fix them with another patch.
+- read_agent_draft returns the current unsaved draft JSON. Each turn also includes that draft in the operator message.
+- read_project_file and list_project_files are read-only and documentation only: docs/, policies/, AGENTS.md, and README.md, and only .md or .txt. A refused path means you tell them you cannot read code or other files.
+- list_mcp_servers names the servers in .mcp.json. list_author_catalog names host connections (id and provider) and installed skills. Grant only names those tools return. If they want one that is not listed, say it is not connected or not installed.
+- update_agent_draft applies a patch immediately. Objects merge key by key, arrays replace, null removes a key. Do not wait for Keep.
 - Never use bash, write, or edit. Save is the operator's button, not a tool.
 
 ## The draft (agents/<id>.agent.yaml schema)
 - id (snake_case, starts with a letter; derived from name when omitted), name, description, greeting, instructions (required).
-- model: { backend: pi | claude, id?: "provider/model", thinking: low | medium | high }. Leave id out to use the project default unless they name one, and say so.
+- model: { backend: pi | claude, id?: "provider/model", thinking: low | medium | high }.
 - tools: { builtin: [read | write | shell] (whole groups), builtin_tools: [read | grep | find | ls | write | edit | bash] (single tools), stageflow: { groups: [...], tools: [...] }, mcp: { <server>: [tool names] or ["*"] }, skills: [names], browser: bool }.
 - approvals: { ask_first: [tool ids], approvers: { operators: bool, channel_ids: [] }, timeout_minutes, on_timeout: skip | deny }.
   Tool ids: builtin names (read, grep, find, ls, write, edit, bash), "<server>.<tool>" for MCP, "agent:<id>" for sub-agents. ask_first may only name granted tools.
 - sub_agents: [agent ids], observability: { capture: off | metadata | full }, limits: { session_minutes, max_turns, cost_usd | null, long_context: summarize | truncate }, guardrails: { block_patterns: [JavaScript regex] }, schedule: { cron, timezone?, message? }.
-
-## Defaults
-- A new agent saves disabled and starts sealed: read only. Write and shell stay off unless the job needs them.
-- Anything that sends, posts, writes, merges or deletes goes in approvals.ask_first.
-- Capture stays metadata unless they ask for full content.
-
-## After an edit
-- One sentence on what the agent will do now. Mention any assumption or open question. Do not paste YAML unless they ask.
 `;

package/dist/operatorAgent/profiles/authorReadable.d.ts

@@ -0,0 +1,2 @@
+/** Relative project path the agent author may read. Null when it is documentation. */
+export declare function authorReadRefusal(rel: string): string | null;

package/dist/operatorAgent/profiles/authorReadable.js

@@ -0,0 +1,26 @@
+const DOC_ROOTS = new Set(["docs", "policies"]);
+const ROOT_FILES = new Set(["AGENTS.md", "README.md"]);
+/** Relative project path the agent author may read. Null when it is documentation. */
+export function authorReadRefusal(rel) {
+    const normalized = rel.replace(/\\/g, "/").replace(/^\.\//, "");
+    if (!normalized || normalized.startsWith("/") || normalized.split("/").includes("..")) {
+        return "path is outside the readable project surface";
+    }
+    const segments = normalized.split("/").filter(Boolean);
+    if (segments.some((part) => part === ".git" || part === "node_modules" || part.startsWith("."))) {
+        return "path is outside the readable project surface";
+    }
+    if (ROOT_FILES.has(normalized))
+        return null;
+    const [head, ...rest] = segments;
+    if (!head || !DOC_ROOTS.has(head)) {
+        return "only documentation under docs/ and policies/, plus AGENTS.md and README.md, can be read";
+    }
+    if (rest.length === 0)
+        return "path must be a documentation file";
+    const base = rest[rest.length - 1] ?? "";
+    if (!base.endsWith(".md") && !base.endsWith(".txt")) {
+        return "only .md and .txt documentation can be read";
+    }
+    return null;
+}

package/dist/package-meta.d.ts

@@ -1,3 +1,3 @@
 export declare const PACKAGE_NAME = "stageflow";
-export declare const PACKAGE_VERSION = "0.39.0";
+export declare const PACKAGE_VERSION = "0.42.0";
 export declare const BUILD_SHA: string;

package/dist/package-meta.js

@@ -1,3 +1,3 @@
 export const PACKAGE_NAME = "stageflow";
-export const PACKAGE_VERSION = "0.39.0";
+export const PACKAGE_VERSION = "0.42.0";
 export const BUILD_SHA = process.env.STAGEFLOW_BUILD_SHA ?? "unknown";

package/dist/runstore/port.d.ts

@@ -1,4 +1,5 @@
 import type { StageLogLine } from "../agent/activity.js";
+import type { EvalTargetKind } from "../evals/constants.js";
 import type { ConfigOriginRecord } from "../config/configOrigin.js";
 import type { RunSubmission, RunSubmissionRecord } from "./submission.js";
 import type { AskOperatorPrompt } from "../tools/askOperator.js";
@@ -8,6 +9,9 @@ import type { CompletionCheck } from "../types/completion.js";
 import type { StageGateKind } from "../types/stage.js";
 import type { StageUsage } from "../types/usage.js";
 import type { OverviewRunRow } from "./overviewSummary.js";
+import type { CreateEvalRunInput, EvalCaseInput, EvalCaseRow, EvalGraderVersionInput, EvalResultInput, EvalResultWithGrades, EvalReviewPatch, EvalReviewRow, EvalRunPage, EvalRunProgress, EvalRunRow, FinishEvalRunInput, ListEvalRunsQuery } from "../evals/storeTypes.js";
+import type { SuggestionStatus } from "../evals/triage/constants.js";
+import type { SuggestionInput, SuggestionRow } from "../evals/triage/types.js";
 export type { ConfigOriginRecord };
 export type RunStatus = "created" | "queued" | "running" | "succeeded" | "failed" | "cancelled";
 export type RunPipelineDagSnapshot = ResolvedPipelineDag & {
@@ -40,6 +44,7 @@ export type PipelineTrackProjection = {
     nodes: PipelineTrackNode[];
     edges: PipelineTrackEdge[];
 };
+export type RunOrigin = "eval";
 export type RunMeta = {
     run_id: string;
     pipeline_id: string;
@@ -69,6 +74,8 @@ export type RunMeta = {
     config_origins?: ConfigOriginRecord[];
     pipeline_source?: "inline" | "path";
     caller_id?: string;
+    /** `eval` for stage runs started by an eval run; absent for normal runs. */
+    origin?: RunOrigin;
     run_manifest?: unknown;
     skip_gates?: boolean;
     /**
@@ -372,6 +379,7 @@ export type CreateRunInput = {
      */
     inlinePipeline?: InlinePipelineDefinition;
     callerId?: string;
+    origin?: RunOrigin;
     runManifest?: unknown;
     skipGates?: boolean;
 };
@@ -382,6 +390,8 @@ export type ListRunsFilter = {
     /** Match pipeline_id or pipeline_path */
     pipeline?: string;
     caller_id?: string;
+    /** Default (undefined) hides eval stage runs; `"any"` includes them; `"eval"` returns only them. */
+    origin?: RunOrigin | "any";
 };
 /**
  * Persistence port for pipeline run state.
@@ -466,6 +476,39 @@ export interface RunStore {
     listRuns(filter?: ListRunsFilter): Promise<RunSummary[]>;
     /** One light row per run created at or after `since` (ISO), newest first, with summed stage cost. No stages are loaded. */
     listRunOverviewRows(since: string): Promise<OverviewRunRow[]>;
+    createEvalRun(input: CreateEvalRunInput): Promise<EvalRunRow>;
+    insertEvalCases(evalRunId: string, cases: EvalCaseInput[]): Promise<void>;
+    /** Result row and its grade rows in one transaction; replaces any earlier grades for the same result. */
+    upsertEvalResult(result: EvalResultInput): Promise<void>;
+    updateEvalRunProgress(evalRunId: string, progress: EvalRunProgress): Promise<void>;
+    finishEvalRun(evalRunId: string, input: FinishEvalRunInput): Promise<void>;
+    getEvalRun(evalRunId: string): Promise<EvalRunRow | null>;
+    getEvalRunByNumber(projectRoot: string, evalId: string, runNumber: number): Promise<EvalRunRow | null>;
+    listEvalRuns(query: ListEvalRunsQuery): Promise<EvalRunPage>;
+    listEvalResults(evalRunId: string): Promise<EvalResultWithGrades[]>;
+    getEvalResultsForCase(evalRunId: string, caseId: string): Promise<EvalResultWithGrades[]>;
+    getEvalCases(evalRunId: string): Promise<EvalCaseRow[]>;
+    /** `tags` replaces the tag set and is accepted only for results with `passed = 0`. */
+    putEvalReview(evalRunId: string, caseId: string, repeatIndex: number, patch: EvalReviewPatch): Promise<EvalReviewRow>;
+    getEvalReviews(evalRunId: string): Promise<EvalReviewRow[]>;
+    /** The eval's failure-type vocabulary, most recently used first. */
+    listFailureTypes(projectRoot: string, evalId: string): Promise<string[]>;
+    /** Inserts unseen grader hashes with the next ordinal per grader id; returns grader id to ordinal. */
+    recordGraderVersions(projectRoot: string, evalId: string, graders: EvalGraderVersionInput[]): Promise<Record<string, number>>;
+    /** Deletes the eval run and its rows; returns the stage run ids it started so the caller can delete those runs. */
+    deleteEvalRun(evalRunId: string): Promise<string[]>;
+    /** Distinct non-null result subject refs (stage run ids or agent session ids) when the run's target kind matches. */
+    listEvalSubjectRefs(evalRunId: string, targetKind: EvalTargetKind): Promise<string[]>;
+    /** Inserts or replaces pending suggestions (accepted and dismissed are kept); returns the rows written. */
+    upsertEvalSuggestions(evalRunId: string, inputs: SuggestionInput[]): Promise<SuggestionRow[]>;
+    listEvalSuggestions(evalRunId: string, status?: SuggestionStatus): Promise<SuggestionRow[]>;
+    /** Pending only; returns how many changed. */
+    dismissEvalSuggestions(evalRunId: string, ids: string[]): Promise<number>;
+    /** Pending only; adds the failure type and `bucket:` tags through the review write path. */
+    acceptEvalSuggestions(evalRunId: string, ids: string[]): Promise<{
+        accepted: number;
+        tagged: number;
+    }>;
     /** Idempotent upsert; returns the realpath/resolve-normalized absolute key. */
     ensureProject(absPath: string): Promise<string>;
     /** Absolute roots from the durable projects registry. */