stageflow 0.40.0 → 0.44.0

package/dist/server/evalTriageRoutes.d.ts

@@ -0,0 +1,2 @@
+import { type Ctx } from "./evalRouteContext.js";
+export declare function matchEvalTriageRoute(c: Ctx, seg: (value: string | undefined) => string): (() => Promise<void>) | undefined;

package/dist/server/evalTriageRoutes.js

@@ -0,0 +1,95 @@
+import { EvalApiError } from "../evals/read/api.js";
+import { SUGGESTION_STATUSES, TRIAGE_BUCKETS, TRIAGE_MAX_LIMIT } from "../evals/triage/constants.js";
+import { isTriageBucket, TriageService } from "../evals/triage/service.js";
+import { audit, bad, optStringArray, readOptionalBody, runByIdOr404 } from "./evalRouteContext.js";
+function triageOf(c) {
+    return new TriageService({ store: c.store, ...(c.triage ?? {}), loadPrices: c.triage?.loadPrices ?? c.loadPrices });
+}
+async function handleSort(c, id) {
+    const body = await readOptionalBody(c);
+    const row = await runByIdOr404(c, id);
+    const classifier = body.classifier ?? "auto";
+    if (classifier !== "auto" && classifier !== "jev" && classifier !== "judge") {
+        throw bad(`"classifier" must be auto, jev or judge`);
+    }
+    if (body.only_unsorted !== undefined && typeof body.only_unsorted !== "boolean")
+        throw bad(`"only_unsorted" must be a boolean`);
+    if (body.limit !== undefined &&
+        (typeof body.limit !== "number" || !Number.isInteger(body.limit) || body.limit < 1 || body.limit > TRIAGE_MAX_LIMIT)) {
+        throw bad(`"limit" must be an integer from 1 to ${TRIAGE_MAX_LIMIT}`);
+    }
+    try {
+        const result = await triageOf(c).sort(row, {
+            classifier: classifier,
+            ...(body.only_unsorted !== undefined ? { onlyUnsorted: body.only_unsorted } : {}),
+            ...(body.limit !== undefined ? { limit: body.limit } : {}),
+        });
+        audit(c, "eval_failures_sort", "ok", id);
+        c.json(c.res, 200, result);
+    }
+    catch (err) {
+        audit(c, "eval_failures_sort", "error", id, err instanceof EvalApiError ? err.code : "internal_error");
+        throw err;
+    }
+}
+async function handleSuggestionsList(c, id) {
+    const row = await runByIdOr404(c, id);
+    const status = c.url.searchParams.get("status") ?? undefined;
+    if (status !== undefined && !SUGGESTION_STATUSES.includes(status)) {
+        throw bad(`"status" must be one of ${SUGGESTION_STATUSES.join(", ")}`);
+    }
+    c.json(c.res, 200, await triageOf(c).list(row, status));
+}
+function idsOf(body, required) {
+    const ids = optStringArray(body, "ids");
+    if (ids === undefined && required)
+        throw bad(`"ids" is required`);
+    return ids;
+}
+async function handleSuggestionsAccept(c, id) {
+    const body = await readOptionalBody(c);
+    const row = await runByIdOr404(c, id);
+    const ids = idsOf(body, false);
+    if (body.min_confidence !== undefined && (typeof body.min_confidence !== "number" || body.min_confidence < 0 || body.min_confidence > 1)) {
+        throw bad(`"min_confidence" must be a number from 0 to 1`);
+    }
+    if (body.bucket !== undefined && !isTriageBucket(body.bucket))
+        throw bad(`"bucket" must be one of ${TRIAGE_BUCKETS.join(", ")}`);
+    const result = await triageOf(c).accept(row, {
+        ...(ids !== undefined ? { ids } : {}),
+        ...(typeof body.min_confidence === "number" ? { minConfidence: body.min_confidence } : {}),
+        ...(isTriageBucket(body.bucket) ? { bucket: body.bucket } : {}),
+    });
+    audit(c, "eval_suggestions_accept", "ok", id);
+    c.json(c.res, 200, result);
+}
+async function handleSuggestionsDismiss(c, id) {
+    const body = await readOptionalBody(c);
+    const row = await runByIdOr404(c, id);
+    const result = await triageOf(c).dismiss(row, idsOf(body, true));
+    audit(c, "eval_suggestions_dismiss", "ok", id);
+    c.json(c.res, 200, result);
+}
+async function handleSortStatus(c, id) {
+    c.json(c.res, 200, await triageOf(c).status(await runByIdOr404(c, id)));
+}
+export function matchEvalTriageRoute(c, seg) {
+    const { method, pathname } = c;
+    const m = (re) => pathname.match(re);
+    let match = m(/^\/api\/eval-runs\/([^/]+)\/failures\/sort$/);
+    if (method === "POST" && match)
+        return () => handleSort(c, seg(match[1]));
+    match = m(/^\/api\/eval-runs\/([^/]+)\/failures\/suggestions$/);
+    if (method === "GET" && match)
+        return () => handleSuggestionsList(c, seg(match[1]));
+    match = m(/^\/api\/eval-runs\/([^/]+)\/failures\/suggestions\/accept$/);
+    if (method === "POST" && match)
+        return () => handleSuggestionsAccept(c, seg(match[1]));
+    match = m(/^\/api\/eval-runs\/([^/]+)\/failures\/suggestions\/dismiss$/);
+    if (method === "POST" && match)
+        return () => handleSuggestionsDismiss(c, seg(match[1]));
+    match = m(/^\/api\/eval-runs\/([^/]+)\/failures\/status$/);
+    if (method === "GET" && match)
+        return () => handleSortStatus(c, seg(match[1]));
+    return undefined;
+}

package/dist/server/guideRoutes.d.ts

@@ -0,0 +1,5 @@
+import type { IncomingMessage, ServerResponse } from "node:http";
+import type { OperatorAgentHost } from "../operatorAgent/index.js";
+type Json = (res: ServerResponse, status: number, body: unknown) => void;
+export declare function handleGuideRoutes(req: IncomingMessage, method: string, pathname: string, host: OperatorAgentHost, res: ServerResponse, json: Json): Promise<boolean>;
+export {};

package/dist/server/guideRoutes.js

@@ -0,0 +1,166 @@
+import { abortGuideTurn, GuideError, guideAgentInfo, isGuideThreadRunning, streamGuideTurn, } from "../guide/chat.js";
+import { createGuideThread, listGuideThreads, readGuideThread, setGuideThreadModel, } from "../guide/threadStore.js";
+async function readBody(req) {
+    const chunks = [];
+    let size = 0;
+    for await (const chunk of req) {
+        const buffer = typeof chunk === "string" ? Buffer.from(chunk) : chunk;
+        size += buffer.length;
+        if (size > 1_500_000)
+            return { __tooLarge: true };
+        chunks.push(buffer);
+    }
+    if (chunks.length === 0)
+        return {};
+    try {
+        const parsed = JSON.parse(Buffer.concat(chunks).toString("utf8"));
+        return typeof parsed === "object" && parsed !== null && !Array.isArray(parsed)
+            ? parsed
+            : { __invalid: true };
+    }
+    catch {
+        return { __invalid: true };
+    }
+}
+function threadSummary(record) {
+    return { id: record.id, title: record.title, updatedAt: record.updatedAt };
+}
+function canWrite(res) {
+    return !res.writableEnded && !res.destroyed;
+}
+export async function handleGuideRoutes(req, method, pathname, host, res, json) {
+    const root = pathname === "/api/guide";
+    const threads = pathname === "/api/guide/threads";
+    const threadMatch = /^\/api\/guide\/threads\/([^/]+)$/.exec(pathname);
+    const messageMatch = /^\/api\/guide\/threads\/([^/]+)\/messages$/.exec(pathname);
+    const stopMatch = /^\/api\/guide\/threads\/([^/]+)\/stop$/.exec(pathname);
+    if (!root && !threads && !threadMatch && !messageMatch && !stopMatch)
+        return false;
+    if (root && method === "GET") {
+        json(res, 200, { agent: guideAgentInfo(host) });
+        return true;
+    }
+    if (threads && method === "GET") {
+        json(res, 200, {
+            threads: listGuideThreads().map((thread) => ({
+                ...thread,
+                status: isGuideThreadRunning(thread.id) ? "running" : "open",
+            })),
+        });
+        return true;
+    }
+    if (threads && method === "POST") {
+        const record = createGuideThread();
+        json(res, 201, { thread: threadSummary(record) });
+        return true;
+    }
+    if (stopMatch && method === "POST") {
+        const threadId = decodeURIComponent(stopMatch[1] ?? "");
+        const record = readGuideThread(threadId);
+        if (!record) {
+            json(res, 404, { code: "guide.not_found", message: "" });
+            return true;
+        }
+        json(res, 200, { ok: true, stopped: abortGuideTurn(threadId) });
+        return true;
+    }
+    if (threadMatch && method === "GET") {
+        const threadId = decodeURIComponent(threadMatch[1] ?? "");
+        const record = readGuideThread(threadId);
+        if (!record) {
+            json(res, 404, { code: "guide.not_found", message: "" });
+            return true;
+        }
+        json(res, 200, {
+            thread: threadSummary(record),
+            status: isGuideThreadRunning(record.id) ? "running" : "open",
+            messages: record.messages,
+            model: record.model ?? null,
+        });
+        return true;
+    }
+    if (threadMatch && method === "PATCH") {
+        const threadId = decodeURIComponent(threadMatch[1] ?? "");
+        const body = await readBody(req);
+        if (body.__tooLarge || body.__invalid) {
+            json(res, 400, { code: "guide.invalid", message: "" });
+            return true;
+        }
+        let record = readGuideThread(threadId);
+        if (!record) {
+            json(res, 404, { code: "guide.not_found", message: "" });
+            return true;
+        }
+        if ("model" in body) {
+            if (typeof body.model === "string") {
+                const next = setGuideThreadModel(threadId, body.model.trim() || null);
+                if (!next) {
+                    json(res, 404, { code: "guide.not_found", message: "" });
+                    return true;
+                }
+                record = next;
+            }
+            else if (body.model === null) {
+                const next = setGuideThreadModel(threadId, null);
+                if (!next) {
+                    json(res, 404, { code: "guide.not_found", message: "" });
+                    return true;
+                }
+                record = next;
+            }
+            else {
+                json(res, 400, { code: "guide.invalid", message: "" });
+                return true;
+            }
+        }
+        json(res, 200, { thread: threadSummary(record), model: record.model ?? null });
+        return true;
+    }
+    if (messageMatch && method === "POST") {
+        const threadId = decodeURIComponent(messageMatch[1] ?? "");
+        const body = await readBody(req);
+        if (body.__tooLarge || body.__invalid) {
+            json(res, 400, { code: "guide.invalid", message: "" });
+            return true;
+        }
+        const text = typeof body.text === "string" ? body.text : "";
+        let started = false;
+        try {
+            for await (const frame of streamGuideTurn({
+                host,
+                threadId,
+                text,
+                attachments: body.attachments,
+                model: typeof body.model === "string" ? body.model : undefined,
+            })) {
+                if (!canWrite(res))
+                    continue;
+                if (!started) {
+                    started = true;
+                    res.writeHead(200, {
+                        "Content-Type": "application/x-ndjson; charset=utf-8",
+                        "Cache-Control": "no-store",
+                    });
+                }
+                res.write(`${JSON.stringify(frame)}\n`);
+            }
+        }
+        catch (err) {
+            const code = err instanceof GuideError ? err.code : "guide.failed";
+            const status = err instanceof GuideError ? err.status : 500;
+            if (!started && canWrite(res))
+                json(res, status, { code, message: "" });
+            else if (started && canWrite(res))
+                res.write(`${JSON.stringify({ type: "error", code, message: "" })}\n`);
+        }
+        finally {
+            if (!res.headersSent && canWrite(res))
+                json(res, 200, { code: "guide.failed", message: "" });
+            else if (canWrite(res))
+                res.end();
+        }
+        return true;
+    }
+    json(res, 405, { code: "guide.invalid", message: "" });
+    return true;
+}

package/dist/server/http.d.ts

@@ -1,5 +1,8 @@
 import type { StagePort } from "../agent/port.js";
 import { type ProviderAuthContext } from "../agent/providerAuth.js";
+import type { ChannelService } from "../channels/service.js";
+import type { EvalRunManager } from "../evals/manager.js";
+import type { MemoryHost } from "../memory/host.js";
 import { type LiveViewRoutesOptions } from "./liveViewRoutes.js";
 import { WorkshopChatSessionRegistry } from "../workshop/chatTurn.js";
 import { type AgentPort } from "../operatorAgent/index.js";
@@ -10,6 +13,7 @@ import type { RunChangeBus } from "../runtime/runChangeBus.js";
 import type { StageBrowserSupport } from "../browser/browserHost.js";
 import { type AgentHostContext } from "../agents/hostContext.js";
 import type { ConversationBackend } from "../agents/sessions.js";
+import { type EvalServices } from "./evalInternalRoutes.js";
 import { EmailTriggers } from "../email/triggers.js";
 import { type AllowedHosts } from "./allowedHosts.js";
 import { type ControlTokens } from "./controlToken.js";
@@ -68,6 +72,17 @@ export type OperatorRouteDeps = {
     agentConversationBackend?: ConversationBackend;
     /** Shared with MCP when set by bootstrap. */
     agentHost?: AgentHostContext;
+    /** Bootstrap fills this once the operator host exists, so evals can run against it. */
+    operatorHostSlot?: {
+        current?: import("../operatorAgent/types.js").OperatorAgentHost;
+    };
+    evalRunManager?: EvalRunManager;
+    /** Wired by the eval manager; until then the internal eval callback answers 404. */
+    evalServices?: EvalServices;
+    /** Host connections. Guide and MCP use the same service the console does. */
+    channels?: ChannelService;
+    /** Host memories. Guide and MCP use the same host the console does. */
+    memories?: MemoryHost;
 };
 /**
  * The full operator-console route surface: REST API (runs, stages,

package/dist/server/http.js

@@ -8,6 +8,10 @@ import { mapProviderAuthError } from "../agent/providerInspect.js";
 import { handleProviderRoutes } from "./providerRoutes.js";
 import { handleProjectMcpRoutes } from "./projectMcpRoutes.js";
 import { handleTriggerRoutes } from "./triggerRoutes.js";
+import { guideWriteRoot } from "../guide/writeRoot.js";
+import { handleEvalRoutes } from "./evalRoutes.js";
+import { defaultLoadJudgePrices } from "../evals/engine/promptfoo/judgeServices.js";
+import { engineStatus as promptfooEngineStatus } from "../evals/engine/promptfoo/status.js";
 import { readCatalogFileForHttp, writeCatalogFileForHttp, } from "./readCatalogFile.js";
 import { applyLiveViewHeaders, createLiveViewRoutes, isLiveViewDialogPath, isLiveViewInputPath, isLiveViewPath, isLiveViewReopenTabPath, } from "./liveViewRoutes.js";
 import { createPipeline, parseCreatePipelineBody } from "../config/createPipeline.js";
@@ -59,8 +63,11 @@ import { buildModelsResponse } from "../models/response.js";
 import { createAgentHostContext } from "../agents/hostContext.js";
 import { AgentScheduleSource, agentScheduleTickIntervalMsFromEnv, } from "../agents/scheduleSource.js";
 import { handleAgentAuthorRoutes } from "./agentAuthorRoutes.js";
+import { handleGuideRoutes } from "./guideRoutes.js";
+import { handleInternalAgentRoutes } from "./internalAgentRoutes.js";
 import { handleAgentRoutes } from "./agentRoutes.js";
 import { handleChannelRoutes } from "./channelRoutes.js";
+import { handleEvalInternalRoutes, isEvalInternalPath } from "./evalInternalRoutes.js";
 import { handleMemoryRoutes } from "./memoryRoutes.js";
 import { bootstrapStageflowHost, } from "./bootstrap.js";
 import { assertAllowedHttpAccess, isTrustedLocalHttpRequest, resolveAllowedHosts, } from "./allowedHosts.js";
@@ -183,7 +190,7 @@ export function defaultUiDistDir() {
     return path.resolve(here, "../ui");
 }
 export function createOperatorRoutes(deps) {
-    const { manager, store, cwd, agentDir, rootDir, providerAuthContext, uiDistDir, workshopOperatorHost, agentConversationBackend, agentHost: injectedHost, } = deps;
+    const { manager, store, cwd, agentDir, rootDir, providerAuthContext, uiDistDir, workshopOperatorHost, agentConversationBackend, agentHost: injectedHost, evalServices, } = deps;
     const agentConnection = injectedHost ? undefined : sqliteConnectionFromStore(store);
     const agentHost = injectedHost ??
         (agentConnection
@@ -213,7 +220,27 @@ export function createOperatorRoutes(deps) {
         resolveAgentPort({
             cwd,
             projectRoot: cwd,
+            guideDeps: {
+                manager,
+                store,
+                cwd,
+                agentDir,
+                projectRoot: guideWriteRoot(rootDir),
+                ...(providerAuthContext ? { providerAuthContext } : {}),
+                ...(deps.evalRunManager ? { evalRunManager: deps.evalRunManager } : {}),
+                ...(deps.channels ? { channels: deps.channels } : {}),
+                ...(deps.memories ? { memories: deps.memories } : {}),
+                ...(agentHost
+                    ? {
+                        agentService: agentHost.service,
+                        agentChat: agentHost.chat,
+                        agentConversationBackend: agentHost.backend,
+                    }
+                    : {}),
+            },
         });
+    if (deps.operatorHostSlot)
+        deps.operatorHostSlot.current = operatorHost;
     const workshopChatRegistry = deps.workshopChatRegistry ?? new WorkshopChatSessionRegistry(operatorHost);
     const workshopSessionStoreRoot = () => resolveWorkshopSessionStoreRoot();
     const workshopBuildStoreRoot = () => resolveWorkshopBuildStoreRoot();
@@ -250,10 +277,21 @@ export function createOperatorRoutes(deps) {
             try {
                 if (await liveView.handle(req, res))
                     return true;
-                if (pathname === "/api/agent-author/chat" || pathname === "/api/agent-author/undo") {
+                if (isEvalInternalPath(pathname) && (await handleEvalInternalRoutes(req, method, pathname, evalServices, res, json)))
+                    return true;
+                if (pathname === "/api/agent-author/chat" ||
+                    pathname === "/api/agent-author/undo" ||
+                    pathname === "/api/agent-author/sessions" ||
+                    pathname.startsWith("/api/agent-author/sessions/")) {
                     if (await handleAgentAuthorRoutes(req, method, pathname, cwd, operatorHost, res, json))
                         return true;
                 }
+                if (pathname === "/api/guide" || pathname.startsWith("/api/guide/")) {
+                    if (await handleGuideRoutes(req, method, pathname, operatorHost, res, json))
+                        return true;
+                }
+                if (handleInternalAgentRoutes(method, pathname, operatorHost, res, json))
+                    return true;
                 if (PublicWebChat.owns(pathname)) {
                     if (!agentHost) {
                         json(res, 503, { code: "web_chat.unavailable" });
@@ -377,6 +415,9 @@ export function createOperatorRoutes(deps) {
                         filter.pipeline = pipeline;
                     if (callerId !== null)
                         filter.caller_id = callerId;
+                    const origin = url.searchParams.get("origin");
+                    if (origin === "eval" || origin === "any")
+                        filter.origin = origin;
                     json(res, 200, {
                         runs: await store.listRuns(Object.keys(filter).length > 0 ? filter : undefined),
                     });
@@ -1052,7 +1093,9 @@ export function createOperatorRoutes(deps) {
                     await readJsonBody(req);
                     const runId = decodeURIComponent(retryMatch[1] ?? "");
                     const stageId = decodeURIComponent(retryMatch[2] ?? "");
-                    const result = await manager.retryStage(runId, stageId);
+                    const result = await manager.retryStage(runId, stageId, {
+                        awaitRoot: false,
+                    });
                     if (!result.ok) {
                         json(res, result.status ?? 500, mapRetryStageFailure(result));
                         return true;
@@ -2894,6 +2937,21 @@ export function createOperatorRoutes(deps) {
                 })) {
                     return true;
                 }
+                if (await handleEvalRoutes(req, res, {
+                    cwd,
+                    store,
+                    manager,
+                    evalRunManager: deps.evalRunManager,
+                    json,
+                    readJsonBody,
+                    auditLog,
+                    engineStatus: () => promptfooEngineStatus({ home: globalStageflowHome() }),
+                    agents: () => agentHost?.service,
+                    loadPrices: defaultLoadJudgePrices,
+                    triage: { judgeBackend: () => agentHost?.backend },
+                })) {
+                    return true;
+                }
                 if (await handleTriggerRoutes(req, res, {
                     cwd,
                     manager,
@@ -3151,6 +3209,11 @@ export async function startUiServer(options) {
                 ? { workshopOperatorHost: options.workshopOperatorHost }
                 : {}),
             ...(boot.agentHost ? { agentHost: boot.agentHost } : {}),
+            ...(boot.operatorHostSlot ? { operatorHostSlot: boot.operatorHostSlot } : {}),
+            ...(boot.evalRunManager ? { evalRunManager: boot.evalRunManager } : {}),
+            ...(boot.evalServices ? { evalServices: boot.evalServices } : {}),
+            ...(boot.channels ? { channels: boot.channels } : {}),
+            ...(boot.memories ? { memories: boot.memories } : {}),
             ...(options.agentConversationBackend
                 ? { agentConversationBackend: options.agentConversationBackend }
                 : {}),
@@ -3163,6 +3226,7 @@ export async function startUiServer(options) {
         controlTokens,
         routes,
     });
+    boot.setEvalCallbackBase?.(envelope.url);
     const closeServer = envelope.server.close.bind(envelope.server);
     envelope.server.close = ((callback) => closeServer(error => {
         void (boot.email?.stop() ?? Promise.resolve()).then(() => callback?.(error), () => callback?.(error ?? new Error("Email host shutdown failed")));

package/dist/server/internalAgentRoutes.d.ts

@@ -0,0 +1,6 @@
+import type { ServerResponse } from "node:http";
+import type { OperatorAgentHost } from "../operatorAgent/index.js";
+type Json = (res: ServerResponse, status: number, body: unknown) => void;
+export declare const INTERNAL_AGENTS_PATH = "/api/internal-agents";
+export declare function handleInternalAgentRoutes(method: string, pathname: string, host: OperatorAgentHost, res: ServerResponse, json: Json, env?: NodeJS.ProcessEnv): boolean;
+export {};

package/dist/server/internalAgentRoutes.js

@@ -0,0 +1,12 @@
+import { describeInternalAgents, internalAgentsVisible } from "../operatorAgent/internalAgents.js";
+export const INTERNAL_AGENTS_PATH = "/api/internal-agents";
+export function handleInternalAgentRoutes(method, pathname, host, res, json, env = process.env) {
+    if (pathname !== INTERNAL_AGENTS_PATH || method !== "GET")
+        return false;
+    if (!internalAgentsVisible(env)) {
+        json(res, 200, { visible: false, agents: [] });
+        return true;
+    }
+    json(res, 200, { visible: true, agents: describeInternalAgents(host) });
+    return true;
+}

package/dist/server/mcpHost.js

@@ -28,6 +28,10 @@ export async function startMcpServer(options) {
             controlTokens,
             getShutdown: () => shutdown,
             ...(boot.agentHost ? { agentHost: boot.agentHost } : {}),
+            ...(boot.evalRunManager ? { evalRunManager: boot.evalRunManager } : {}),
+            ...(boot.evalServices ? { evalServices: boot.evalServices } : {}),
+            ...(boot.channels ? { channels: boot.channels } : {}),
+            ...(boot.memories ? { memories: boot.memories } : {}),
         });
     const envelope = await createHttpHost({
         boot,
@@ -39,6 +43,7 @@ export async function startMcpServer(options) {
         maxConnections: options.maxConnections,
         routes,
     });
+    boot.setEvalCallbackBase?.(envelope.url);
     shutdown = installShutdownController({
         server: envelope.server,
         host: makeDrainableHostFromOptional(envelope.manager, envelope.store),

package/dist/toolkit/meta.d.ts

@@ -4,7 +4,7 @@
  * can import it without pulling in the runtime.
  */
 export type ToolRisk = "read" | "write" | "destructive" | "external";
-export type ToolGroupId = "discover" | "runs.read" | "runs.control" | "runs.cleanup" | "gates.read" | "gates.decide" | "triggers" | "agents.read" | "agents.manage" | "agents.cleanup" | "agents.converse" | "pipelines.author";
+export type ToolGroupId = "discover" | "runs.read" | "runs.control" | "runs.cleanup" | "gates.read" | "gates.decide" | "triggers" | "agents.read" | "agents.manage" | "agents.cleanup" | "agents.converse" | "evals.read" | "evals.run" | "evals.write" | "evals.cleanup" | "pipelines.author" | "connections" | "connections.cleanup" | "memories" | "memories.cleanup";
 export type ToolGroup = {
     id: ToolGroupId;
     title: string;

package/dist/toolkit/meta.js

@@ -16,6 +16,14 @@ export const TOOL_GROUPS = [
     { id: "agents.cleanup", title: "Archive agents", summary: "Archive an agent. Destructive." },
     { id: "agents.converse", title: "Talk to agents", summary: "Chat with an agent and stop its running turn." },
     { id: "pipelines.author", title: "Author pipelines and tasks", summary: "Read, create and edit pipelines, stages and tasks directly on disk." },
+    { id: "connections", title: "Connections", summary: "List, create, test and update host connections. Secret values are stored and never returned." },
+    { id: "connections.cleanup", title: "Remove connections", summary: "Remove a connection and its stored secrets. Destructive." },
+    { id: "memories", title: "Memories", summary: "List, create and edit host memories and their cards." },
+    { id: "memories.cleanup", title: "Delete memory cards", summary: "Delete one card from a host memory. Destructive." },
+    { id: "evals.read", title: "Read evals", summary: "List evals, runs and results; validate and compare. Free." },
+    { id: "evals.run", title: "Run evals", summary: "Start and cancel eval runs, and try one case. Calls models and costs money." },
+    { id: "evals.write", title: "Author evals", summary: "Create and edit evals and datasets, install a sample, and review or sort suggestions." },
+    { id: "evals.cleanup", title: "Delete eval runs", summary: "Delete an eval run and its stored results. Destructive." },
 ];
 const t = (group, risk, operatorOnly) => ({ group, risk, ...(operatorOnly ? { operatorOnly } : {}) });
 export const TOOL_META = {
@@ -68,6 +76,8 @@ export const TOOL_META = {
     get_agent: t("agents.read", "read"),
     validate_agent: t("agents.read", "read"),
     list_agent_revisions: t("agents.read", "read"),
+    get_agent_revision: t("agents.read", "read"),
+    diff_agent_revisions: t("agents.read", "read"),
     get_agent_session: t("agents.read", "read"),
     list_agent_sessions: t("agents.read", "read"),
     create_agent: t("agents.manage", "write"),
@@ -87,6 +97,49 @@ export const TOOL_META = {
     edit_stage: t("pipelines.author", "write"),
     create_task: t("pipelines.author", "write"),
     edit_task: t("pipelines.author", "write"),
+    create_task_file: t("pipelines.author", "write"),
+    update_task_file: t("pipelines.author", "write"),
+    import_task_file: t("pipelines.author", "external"),
+    create_stage_file: t("pipelines.author", "write"),
+    write_catalog_file: t("pipelines.author", "write"),
+    validate_catalog_draft: t("pipelines.author", "read"),
+    list_connections: t("connections", "read"),
+    create_connection: t("connections", "write"),
+    update_connection: t("connections", "write"),
+    test_connection: t("connections", "external"),
+    validate_connection_target: t("connections", "read"),
+    remove_connection: t("connections.cleanup", "destructive"),
+    list_memories: t("memories", "read"),
+    get_memory: t("memories", "read"),
+    create_memory: t("memories", "write"),
+    write_memory: t("memories", "write"),
+    write_memory_card: t("memories", "write"),
+    delete_memory_card: t("memories.cleanup", "destructive"),
+    list_evals: t("evals.read", "read"),
+    get_eval: t("evals.read", "read"),
+    validate_eval: t("evals.read", "read"),
+    get_eval_run: t("evals.read", "read"),
+    list_eval_runs: t("evals.read", "read"),
+    get_eval_results: t("evals.read", "read"),
+    get_eval_result: t("evals.read", "read"),
+    compare_eval_runs: t("evals.read", "read"),
+    get_eval_failure_suggestions: t("evals.read", "read"),
+    get_eval_failure_sort_status: t("evals.read", "read"),
+    get_eval_run_export: t("evals.read", "read"),
+    run_eval: t("evals.run", "external"),
+    sort_eval_failures: t("evals.run", "external"),
+    cancel_eval_run: t("evals.run", "write"),
+    try_eval_case: t("evals.run", "external"),
+    create_eval: t("evals.write", "write"),
+    update_eval: t("evals.write", "write"),
+    create_dataset: t("evals.write", "write"),
+    update_dataset_cases: t("evals.write", "write"),
+    convert_dataset: t("evals.write", "write"),
+    install_eval_sample: t("evals.write", "write"),
+    put_eval_review: t("evals.write", "write"),
+    accept_eval_failure_suggestions: t("evals.write", "write"),
+    dismiss_eval_failure_suggestions: t("evals.write", "write"),
+    delete_eval_run: t("evals.cleanup", "destructive"),
 };
 /** The grant id the policy gate and `approvals.ask_first` use for a Stageflow tool. */
 export const STAGEFLOW_GRANT_PREFIX = "stageflow.";

package/dist/toolkit/profiles.d.ts

@@ -3,7 +3,12 @@
  * Their tools are checked through the same grant-id check as any agent's tools,
  * so there is one permission layer. The lists are literals so this module has no
  * dependency on the profiles; tests keep them equal to the profiles' tool names.
+ * Guide is the exception: it may call every Stageflow tool a person could grant,
+ * except the ones only a person may decide.
  */
+export declare const GUIDE_OWN_TOOL_NAMES: readonly ["confirm_action", "show_ui", "list_guide_topics", "read_guide_topic"];
+/** Tool names Guide may call. Operator-only decisions stay out. */
+export declare function stageflowGuideToolNames(): readonly string[];
 export declare const PROFILE_GRANTS: Readonly<Record<string, {
     prefix: string;
     tools: readonly string[];

package/dist/toolkit/profiles.js

@@ -1,9 +1,17 @@
+import { isOperatorOnlyTool, TOOL_META } from "./meta.js";
 /**
  * The Workshop and Agent-author profiles are agents with a fixed grant list.
  * Their tools are checked through the same grant-id check as any agent's tools,
  * so there is one permission layer. The lists are literals so this module has no
  * dependency on the profiles; tests keep them equal to the profiles' tool names.
+ * Guide is the exception: it may call every Stageflow tool a person could grant,
+ * except the ones only a person may decide.
  */
+export const GUIDE_OWN_TOOL_NAMES = ["confirm_action", "show_ui", "list_guide_topics", "read_guide_topic"];
+/** Tool names Guide may call. Operator-only decisions stay out. */
+export function stageflowGuideToolNames() {
+    return [...Object.keys(TOOL_META).filter((name) => !isOperatorOnlyTool(name)), ...GUIDE_OWN_TOOL_NAMES];
+}
 export const PROFILE_GRANTS = {
     "workshop-author": {
         prefix: "workshop.",
@@ -11,7 +19,11 @@ export const PROFILE_GRANTS = {
     },
     "agent-author": {
         prefix: "agent_author.",
-        tools: ["read_project_file", "list_project_files", "list_mcp_servers", "update_agent_draft"],
+        tools: ["read_agent_draft", "read_project_file", "list_project_files", "list_mcp_servers", "list_author_catalog", "update_agent_draft"],
+    },
+    "stageflow-guide": {
+        prefix: "guide.",
+        tools: stageflowGuideToolNames(),
     },
 };
 export function profileGrantIds(profileId) {

package/dist/tools/askOperator.js

@@ -7,6 +7,7 @@
  * Do not abort the session from inside `execute`: the tool Promise is the
  * live blocking surface until the operator answers (or stage teardown).
  */
+import { ASK_OPERATOR_TOOL_NAME } from "./stageToolNames.js";
 import { randomUUID } from "node:crypto";
 import { Type } from "typebox";
 import { prepareAskOperatorArguments } from "./prepareToolArguments.js";
@@ -399,7 +400,7 @@ export function createAskOperatorTool(options) {
         ? [...allowed].join(", ")
         : ASK_OPERATOR_KINDS.join(", ");
     return {
-        name: "ask_operator",
+        name: ASK_OPERATOR_TOOL_NAME,
         label: "Ask operator",
         description: `Ask the operator a question and wait for their answer. Supports ${kindList} prompts. Does not complete the stage — call emit_stage_envelope when finished.`,
         parameters: askOperatorParamsSchemaFor(allowedKinds),