stageflow 0.42.0 → 0.46.0

package/dist/server/evalRoutes.js

@@ -16,10 +16,12 @@ import { resolveSeededExamplesPath } from "../config/seededCatalog.js";
 import { applyCaseOps, appendOrReplaceCases, convertDatasetToJsonl, createDatasetFile, } from "../config/writeDataset.js";
 import { createEvalFile, previewEvalUpdate, renderEvalCreateYaml, updateEvalFile, } from "../config/writeEval.js";
 import { CompareMismatchError } from "../evals/compare.js";
+import { suggestExamples } from "../evals/calibrationSuggest.js";
 import { EVAL_LIMITS, EVAL_REVIEW_LABELS, EVAL_STARTED_VIA } from "../evals/constants.js";
 import { PINNED_PROMPTFOO_VERSION } from "../evals/engine/promptfoo/pinned.js";
 import { AGENT_ESTIMATE_ASSUMPTIONS, estimateRun, } from "../evals/estimate.js";
 import { EvalRunError, } from "../evals/manager.js";
+import { buildLabelQueue, buildLabelSummary } from "../evals/read/labelQueue.js";
 import { buildEvalPlan } from "../evals/plan.js";
 import { buildEvalDetail, buildResultDetail, buildResultsPage, DEFAULT_LIMIT, ENGINE_NAME, engineCheck as engineCheckStatus, EvalApiError, evalScopeRoots, listEvalRunsPage, loadAggregatedEvalList, locateDataset, locateEval, locateEvalRoot, wireRootOf, MAX_LIMIT, planFindings as planFindingsShared, rel, validateEvalText, } from "../evals/read/api.js";
 import { listAgentTargets } from "../evals/read/agentTargets.js";
@@ -31,6 +33,7 @@ import { EvalStoreError } from "../evals/storeError.js";
 import { audit, bad, isRecord, optStringArray, readOptionalBody, runByIdOr404, } from "./evalRouteContext.js";
 import { matchEvalTriageRoute } from "./evalTriageRoutes.js";
 import { callerIdFromRequestAuth } from "./requestAuthContext.js";
+import { isJudged, isJudgedType } from "../evals/judged.js";
 const ID_RE = /^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$/;
 const CANCEL_WAIT_MS = 5000;
 const WRITE_ERROR_STATUS = {
@@ -57,6 +60,8 @@ const RUN_ERROR_STATUS = {
     EVAL_ENGINE_UNAVAILABLE: 503,
     EVAL_NODE_TOO_OLD: 503,
     EVAL_TAGS_ON_PASSED_RESULT: 400,
+    EVAL_CALIBRATION_NO_LABELS: 422,
+    EVAL_CALIBRATION_CANCELLED: 409,
 };
 function sendError(c, status, code, message, extra = {}) {
     c.json(c.res, status, { error: message, code, ...extra });
@@ -141,7 +146,8 @@ function agentsOf(c) {
 }
 function agentsInput(c) {
     const agents = agentsOf(c);
-    return agents ? { agents } : {};
+    const internalAgents = c.evalRunManager.visibleInternalAgents();
+    return { ...(agents ? { agents } : {}), ...(internalAgents.length > 0 ? { internalAgents } : {}) };
 }
 function engineCheck(c) {
     engineCheckStatus(c.engineStatus());
@@ -254,11 +260,30 @@ function parseDefinitionOrYaml(body) {
         throw bad(`"definition" must be an object`);
     return { definition: body.definition };
 }
+function validateEvalId(input) {
+    if (isRecord(input.definition) && typeof input.definition.id === "string")
+        return input.definition.id;
+    if (typeof input.yaml !== "string")
+        return undefined;
+    const match = /^id:\s*["']?([A-Za-z0-9][A-Za-z0-9._-]{0,63})["']?\s*$/m.exec(input.yaml);
+    return match?.[1];
+}
+async function existingEvalFile(c, wire, id) {
+    try {
+        const located = await evalRootFor(c, wire, id);
+        return { projectRoot: located.projectRoot, absPath: located.ref.absPath };
+    }
+    catch (err) {
+        if (err instanceof EvalApiError && err.status === 404)
+            return undefined;
+        throw err;
+    }
+}
 async function handleEvalValidate(c) {
     const body = await readBody(c);
     const input = parseDefinitionOrYaml(body);
-    const projectRoot = await projectRootFor(c, wireFromBody(body));
-    const manifest = await manifestForWrite(projectRoot);
+    const wire = wireFromBody(body);
+    let projectRoot = await projectRootFor(c, wire);
     const suppliedPath = optString(body, "path");
     const datasetCases = optDatasetCases(body);
     let evalAbs;
@@ -267,10 +292,18 @@ async function handleEvalValidate(c) {
         assertInsideProject(projectRoot, evalAbs, "eval path");
     }
     else {
-        const rawId = isRecord(input.definition) ? input.definition.id : undefined;
-        const dir = await resolveWriteDir(projectRoot, manifest, "evals");
-        const fileName = typeof rawId === "string" && ID_RE.test(rawId) ? `${rawId}.eval.yaml` : "new.eval.yaml";
-        evalAbs = path.join(dir.absDir, fileName);
+        const rawId = validateEvalId(input);
+        const located = rawId !== undefined && ID_RE.test(rawId) ? await existingEvalFile(c, wire, rawId) : undefined;
+        if (located) {
+            projectRoot = located.projectRoot;
+            evalAbs = located.absPath;
+        }
+        else {
+            const manifest = await manifestForWrite(projectRoot);
+            const dir = await resolveWriteDir(projectRoot, manifest, "evals");
+            const fileName = rawId !== undefined && ID_RE.test(rawId) ? `${rawId}.eval.yaml` : "new.eval.yaml";
+            evalAbs = path.join(dir.absDir, fileName);
+        }
     }
     let derived;
     if (input.yaml !== undefined) {
@@ -639,7 +672,7 @@ async function handleRunEstimate(c, id) {
             resultsExecuted: stats.results_executed,
             rubricGradesGraded: results
                 .flatMap((r) => r.grades)
-                .filter((g) => (g.graderType === "rubric" || g.graderType === "goal_reached") && g.status === "graded").length,
+                .filter((g) => isJudgedType(g.graderType) && g.status === "graded").length,
             wallSeconds: last.startedAt !== null && last.finishedAt !== null
                 ? Math.max(0, (Date.parse(last.finishedAt) - Date.parse(last.startedAt)) / 1000)
                 : null,
@@ -664,7 +697,7 @@ async function agentEstimateBlock(c, plan) {
         }
     }
     const count = Math.max(1, plan.cases.length);
-    const judge = plan.graders.find((g) => g.type === "rubric" || g.type === "goal_reached");
+    const judge = plan.graders.find(isJudged);
     const judgeModel = judge !== undefined && "judge" in judge && judge.judge !== "" ? judge.judge : undefined;
     const wanted = [plan.stageModel, plan.userModel ?? undefined, judgeModel].filter((id) => typeof id === "string" && id !== "");
     const prices = (await c.loadPrices?.([...new Set(wanted)], plan.projectRoot).catch(() => undefined)) ?? new Map();
@@ -681,7 +714,7 @@ async function sendEstimate(c, plan, previous) {
     const agent = plan.target.kind === "agent" ? await agentEstimateBlock(c, plan) : undefined;
     const estimate = estimateRun({
         resultsTotal: plan.cases.length * plan.repeats,
-        rubricGraderCount: plan.graders.filter((g) => g.type === "rubric" || g.type === "goal_reached").length,
+        rubricGraderCount: plan.graders.reduce((calls, g) => (isJudged(g) ? calls + (g.judgeSettings?.votes ?? 1) : calls), 0),
         previous,
         recentStageCosts: await recentStageCosts(c, {
             ...(plan.target.kind === "stage" && plan.target.pipelinePath !== undefined ? { pipelinePath: plan.target.pipelinePath } : {}),
@@ -806,6 +839,10 @@ async function handleResultsList(c, id) {
 async function handleResultDetail(c, id, caseId) {
     c.json(c.res, 200, await buildResultDetail(c.store, id, caseId));
 }
+async function handleLabelQueue(c, id, graderId) {
+    const summary = c.url.searchParams.get("summary") === "true";
+    c.json(c.res, 200, summary ? await buildLabelSummary(c.store, id, graderId) : await buildLabelQueue(c.store, id, graderId));
+}
 async function handleReviewPut(c, id, caseId, repeat) {
     const body = await readBody(c);
     await runByIdOr404(c, id);
@@ -842,6 +879,191 @@ async function handleReviewPut(c, id, caseId, repeat) {
         throw err;
     }
 }
+const CALIBRATION_SPLITS = ["all", "dev", "held_out"];
+function candidateFrom(raw) {
+    if (raw === undefined || raw === null)
+        return undefined;
+    if (!isRecord(raw))
+        throw bad(`"candidate" must be an object`);
+    const out = {};
+    if (raw.judge_settings !== undefined) {
+        if (!isRecord(raw.judge_settings))
+            throw bad(`"candidate.judge_settings" must be an object`);
+        const js = raw.judge_settings;
+        const settings = {};
+        if (js.temperature !== undefined) {
+            if (typeof js.temperature !== "number" || js.temperature < 0 || js.temperature > 1) {
+                throw bad(`"candidate.judge_settings.temperature" must be a number from 0 to 1`);
+            }
+            settings.temperature = js.temperature;
+        }
+        if (js.reasoning !== undefined) {
+            if (typeof js.reasoning !== "boolean")
+                throw bad(`"candidate.judge_settings.reasoning" must be a boolean`);
+            settings.reasoning = js.reasoning;
+        }
+        if (js.votes !== undefined) {
+            if (typeof js.votes !== "number" || ![1, 3, 5].includes(js.votes))
+                throw bad(`"candidate.judge_settings.votes" must be 1, 3 or 5`);
+            settings.votes = js.votes;
+        }
+        out.judgeSettings = settings;
+    }
+    if (raw.rubric !== undefined) {
+        if (typeof raw.rubric !== "string" || raw.rubric.trim() === "")
+            throw bad(`"candidate.rubric" must be a non-empty string`);
+        out.rubric = raw.rubric;
+    }
+    if (raw.judge !== undefined) {
+        if (typeof raw.judge !== "string" || raw.judge.trim() === "")
+            throw bad(`"candidate.judge" must be a non-empty string`);
+        out.judge = raw.judge;
+    }
+    if (raw.examples !== undefined) {
+        if (!Array.isArray(raw.examples) || raw.examples.length > EVAL_LIMITS.rubricExamplesMax) {
+            throw bad(`"candidate.examples" must be an array of at most ${EVAL_LIMITS.rubricExamplesMax}`);
+        }
+        out.examples = raw.examples.map((e) => {
+            if (!isRecord(e) || typeof e.output !== "string")
+                throw bad(`Each candidate example needs an "output" string`);
+            const reason = typeof e.reason === "string" ? { reason: e.reason } : {};
+            if (e.verdict === "pass" || e.verdict === "fail")
+                return { verdict: e.verdict, output: e.output, ...reason };
+            if (typeof e.score === "number" && Number.isInteger(e.score) && e.score >= 1 && e.score <= 5) {
+                return { score: e.score, output: e.output, ...reason };
+            }
+            throw bad(`Each candidate example needs a verdict (pass or fail) or a score from 1 to 5`);
+        });
+    }
+    return out;
+}
+async function handleGraderLabelPut(c, id, caseId, repeat, graderId) {
+    const body = await readBody(c);
+    await runByIdOr404(c, id);
+    let label;
+    if (body.label === null)
+        label = null;
+    else if (typeof body.label === "string" && EVAL_REVIEW_LABELS.includes(body.label)) {
+        label = body.label;
+    }
+    else
+        throw bad(`"label" must be pass, fail or null`);
+    if (body.note !== undefined && body.note !== null) {
+        if (typeof body.note !== "string" || body.note.length > EVAL_LIMITS.reviewNoteMaxChars) {
+            throw bad(`"note" must be a string of at most ${EVAL_LIMITS.reviewNoteMaxChars} characters or null`);
+        }
+    }
+    const results = await c.store.getEvalResultsForCase(id, caseId);
+    const result = results.find((r) => r.repeatIndex === repeat);
+    if (!result)
+        throw new EvalStoreError("EVAL_RUN_NOT_FOUND", `No result for ${caseId}#${repeat} in ${id}`);
+    const grade = result.grades.find((g) => g.graderId === graderId);
+    if (!grade || !isJudgedType(grade.graderType)) {
+        throw bad(`Grader ${graderId} is not a judged grader on this result`);
+    }
+    const row = await c.store.putEvalGraderLabel(id, caseId, repeat, graderId, {
+        label,
+        ...(body.note !== undefined ? { note: body.note } : {}),
+    });
+    audit(c, "eval_grader_label", "ok", id);
+    c.json(c.res, 200, row === null ? { label: null, note: null } : { label: row.label, note: row.note });
+}
+function calibrationToWire(r) {
+    return {
+        id: r.id,
+        grader_id: r.graderId,
+        grader_hash: r.graderHash,
+        judge: r.judge,
+        prompt_version: r.promptVersion,
+        split: r.split,
+        candidate: r.candidate,
+        stats: r.stats,
+        disagreements: r.disagreements.map((d) => ({
+            ...(d.evalRunId !== undefined ? { eval_run_id: d.evalRunId } : {}),
+            case_id: d.caseId,
+            repeat_index: d.repeatIndex,
+            label: d.label,
+            judge: d.judge,
+            reason: d.reason,
+        })),
+        cost_usd: r.costUsd,
+        created_at: r.createdAt,
+    };
+}
+async function handleCalibrationList(c, id, graderId) {
+    const root = await locateEvalRoot(rootOptions(c), wireFromQuery(c), id);
+    const candidates = c.url.searchParams.get("include_candidates") === "true";
+    const items = await c.store.listEvalCalibrations(root.path, id, graderId, candidates);
+    c.json(c.res, 200, { items: items.map(calibrationToWire), next_cursor: null });
+}
+async function handleSuggestExamples(c, id, graderId) {
+    const body = await readBody(c);
+    if (body.calibration_id !== undefined && typeof body.calibration_id !== "string") {
+        throw bad(`"calibration_id" must be a string`);
+    }
+    const { projectRoot, ref } = await evalRootFor(c, wireFromBody(body), id);
+    const outcome = await loadEvalOutcome(ref.absPath, projectRoot);
+    if (!outcome.ok)
+        throw new EvalApiError(422, "EVAL_INVALID", "Eval definition is invalid");
+    const grader = outcome.value.graders.find((g) => g.id === graderId);
+    if (!grader || grader.type !== "rubric") {
+        throw new EvalApiError(422, "EVAL_INVALID", `Grader ${graderId} is not a rubric grader of this eval`);
+    }
+    const calibrations = await c.store.listEvalCalibrations(projectRoot, id, graderId, true, 50);
+    const calibration = body.calibration_id !== undefined
+        ? calibrations.find((x) => x.id === body.calibration_id)
+        : calibrations.find((x) => !x.candidate);
+    if (!calibration) {
+        throw new EvalApiError(404, "EVAL_RUN_NOT_FOUND", "No calibration to suggest examples from; check the judge against your labels first");
+    }
+    const items = await suggestExamples(c.store, {
+        projectRoot,
+        evalId: id,
+        graderId,
+        scale: grader.scale,
+        calibration,
+    });
+    c.json(c.res, 200, { calibration_id: calibration.id, items });
+}
+async function handleCalibrate(c, id, graderId) {
+    const body = await readBody(c);
+    const split = body.split === undefined ? "all" : body.split;
+    if (typeof split !== "string" || !CALIBRATION_SPLITS.includes(split)) {
+        throw bad(`"split" must be one of ${CALIBRATION_SPLITS.join(", ")}`);
+    }
+    if (body.max_cost_usd !== undefined && (typeof body.max_cost_usd !== "number" || !(body.max_cost_usd > 0))) {
+        throw bad(`"max_cost_usd" must be a positive number`);
+    }
+    const candidate = candidateFrom(body.candidate);
+    if (body.stability !== undefined && typeof body.stability !== "boolean")
+        throw bad(`"stability" must be a boolean`);
+    const { projectRoot, ref } = await evalRootFor(c, wireFromBody(body), id);
+    engineCheck(c);
+    const closed = new AbortController();
+    const onClose = () => closed.abort();
+    c.res.once("close", onClose);
+    try {
+        const out = await c.evalRunManager.calibrate({
+            signal: closed.signal,
+            projectRoot,
+            evalPath: ref.absPath,
+            graderId,
+            split: split,
+            ...(candidate !== undefined ? { candidate } : {}),
+            ...(body.max_cost_usd !== undefined ? { maxCostUsd: body.max_cost_usd } : {}),
+            ...(body.stability === true ? { stability: true } : {}),
+        });
+        audit(c, "eval_calibrate", "ok");
+        c.json(c.res, 200, calibrationToWire(out.calibration));
+    }
+    catch (err) {
+        audit(c, "eval_calibrate", "error", undefined, err instanceof EvalRunError ? err.code : "internal_error");
+        throw err;
+    }
+    finally {
+        c.res.off("close", onClose);
+    }
+}
 async function handleFailureTypes(c, id) {
     const root = await locateEvalRoot(rootOptions(c), wireFromQuery(c), id);
     c.json(c.res, 200, { items: await c.store.listFailureTypes(root.path, id), next_cursor: null });
@@ -955,7 +1177,7 @@ async function datasetDetailFor(c, projectRoot, id, absPath, evalId, query) {
     let inputSchema;
     let agent;
     if (useEval !== undefined) {
-        if (useEval.loaded.file.target.kind === "agent") {
+        if (useEval.loaded.file.target.kind === "agent" || useEval.loaded.file.target.kind === "internal_agent") {
             agent = { mode: resolveConversation(useEval.loaded.file.conversation).mode };
         }
         else {
@@ -1014,7 +1236,8 @@ async function conversationModeOf(c, body) {
         return undefined;
     const { projectRoot, ref } = await evalRootFor(c, wireFromBody(body), evalId);
     const outcome = await loadEvalOutcome(ref.absPath, projectRoot);
-    return outcome.ok && outcome.value.file.target.kind === "agent"
+    const kind = outcome.ok ? outcome.value.file.target.kind : undefined;
+    return outcome.ok && (kind === "agent" || kind === "internal_agent")
         ? resolveConversation(outcome.value.file.conversation).mode
         : undefined;
 }
@@ -1370,6 +1593,15 @@ export async function handleEvalRoutes(req, res, deps) {
     match = m(/^\/api\/evals\/([^/]+)\/runs$/);
     if (method === "POST" && match)
         return run(() => handleRunStart(c, seg(match[1])));
+    match = m(/^\/api\/evals\/([^/]+)\/graders\/([^/]+)\/calibrations$/);
+    if (method === "GET" && match)
+        return run(() => handleCalibrationList(c, seg(match[1]), seg(match[2])));
+    match = m(/^\/api\/evals\/([^/]+)\/graders\/([^/]+)\/suggest-examples$/);
+    if (method === "POST" && match)
+        return run(() => handleSuggestExamples(c, seg(match[1]), seg(match[2])));
+    match = m(/^\/api\/evals\/([^/]+)\/graders\/([^/]+)\/calibrate$/);
+    if (method === "POST" && match)
+        return run(() => handleCalibrate(c, seg(match[1]), seg(match[2])));
     match = m(/^\/api\/evals\/([^/]+)\/failure-types$/);
     if (method === "GET" && match)
         return run(() => handleFailureTypes(c, seg(match[1])));
@@ -1401,6 +1633,13 @@ export async function handleEvalRoutes(req, res, deps) {
     if (method === "PUT" && match) {
         return run(() => handleReviewPut(c, seg(match[1]), seg(match[2]), Number(match[3])));
     }
+    match = m(/^\/api\/eval-runs\/([^/]+)\/results\/([^/]+)\/(\d+)\/graders\/([^/]+)\/label$/);
+    if (method === "PUT" && match) {
+        return run(() => handleGraderLabelPut(c, seg(match[1]), seg(match[2]), Number(match[3]), seg(match[4])));
+    }
+    match = m(/^\/api\/eval-runs\/([^/]+)\/graders\/([^/]+)\/label-queue$/);
+    if (method === "GET" && match)
+        return run(() => handleLabelQueue(c, seg(match[1]), seg(match[2])));
     const triageRoute = matchEvalTriageRoute(c, seg);
     if (triageRoute)
         return run(triageRoute);

package/dist/server/guideRoutes.d.ts

@@ -0,0 +1,5 @@
+import type { IncomingMessage, ServerResponse } from "node:http";
+import type { OperatorAgentHost } from "../operatorAgent/index.js";
+type Json = (res: ServerResponse, status: number, body: unknown) => void;
+export declare function handleGuideRoutes(req: IncomingMessage, method: string, pathname: string, host: OperatorAgentHost, res: ServerResponse, json: Json): Promise<boolean>;
+export {};

package/dist/server/guideRoutes.js

@@ -0,0 +1,166 @@
+import { abortGuideTurn, GuideError, guideAgentInfo, isGuideThreadRunning, streamGuideTurn, } from "../guide/chat.js";
+import { createGuideThread, listGuideThreads, readGuideThread, setGuideThreadModel, } from "../guide/threadStore.js";
+async function readBody(req) {
+    const chunks = [];
+    let size = 0;
+    for await (const chunk of req) {
+        const buffer = typeof chunk === "string" ? Buffer.from(chunk) : chunk;
+        size += buffer.length;
+        if (size > 1_500_000)
+            return { __tooLarge: true };
+        chunks.push(buffer);
+    }
+    if (chunks.length === 0)
+        return {};
+    try {
+        const parsed = JSON.parse(Buffer.concat(chunks).toString("utf8"));
+        return typeof parsed === "object" && parsed !== null && !Array.isArray(parsed)
+            ? parsed
+            : { __invalid: true };
+    }
+    catch {
+        return { __invalid: true };
+    }
+}
+function threadSummary(record) {
+    return { id: record.id, title: record.title, updatedAt: record.updatedAt };
+}
+function canWrite(res) {
+    return !res.writableEnded && !res.destroyed;
+}
+export async function handleGuideRoutes(req, method, pathname, host, res, json) {
+    const root = pathname === "/api/guide";
+    const threads = pathname === "/api/guide/threads";
+    const threadMatch = /^\/api\/guide\/threads\/([^/]+)$/.exec(pathname);
+    const messageMatch = /^\/api\/guide\/threads\/([^/]+)\/messages$/.exec(pathname);
+    const stopMatch = /^\/api\/guide\/threads\/([^/]+)\/stop$/.exec(pathname);
+    if (!root && !threads && !threadMatch && !messageMatch && !stopMatch)
+        return false;
+    if (root && method === "GET") {
+        json(res, 200, { agent: guideAgentInfo(host) });
+        return true;
+    }
+    if (threads && method === "GET") {
+        json(res, 200, {
+            threads: listGuideThreads().map((thread) => ({
+                ...thread,
+                status: isGuideThreadRunning(thread.id) ? "running" : "open",
+            })),
+        });
+        return true;
+    }
+    if (threads && method === "POST") {
+        const record = createGuideThread();
+        json(res, 201, { thread: threadSummary(record) });
+        return true;
+    }
+    if (stopMatch && method === "POST") {
+        const threadId = decodeURIComponent(stopMatch[1] ?? "");
+        const record = readGuideThread(threadId);
+        if (!record) {
+            json(res, 404, { code: "guide.not_found", message: "" });
+            return true;
+        }
+        json(res, 200, { ok: true, stopped: abortGuideTurn(threadId) });
+        return true;
+    }
+    if (threadMatch && method === "GET") {
+        const threadId = decodeURIComponent(threadMatch[1] ?? "");
+        const record = readGuideThread(threadId);
+        if (!record) {
+            json(res, 404, { code: "guide.not_found", message: "" });
+            return true;
+        }
+        json(res, 200, {
+            thread: threadSummary(record),
+            status: isGuideThreadRunning(record.id) ? "running" : "open",
+            messages: record.messages,
+            model: record.model ?? null,
+        });
+        return true;
+    }
+    if (threadMatch && method === "PATCH") {
+        const threadId = decodeURIComponent(threadMatch[1] ?? "");
+        const body = await readBody(req);
+        if (body.__tooLarge || body.__invalid) {
+            json(res, 400, { code: "guide.invalid", message: "" });
+            return true;
+        }
+        let record = readGuideThread(threadId);
+        if (!record) {
+            json(res, 404, { code: "guide.not_found", message: "" });
+            return true;
+        }
+        if ("model" in body) {
+            if (typeof body.model === "string") {
+                const next = setGuideThreadModel(threadId, body.model.trim() || null);
+                if (!next) {
+                    json(res, 404, { code: "guide.not_found", message: "" });
+                    return true;
+                }
+                record = next;
+            }
+            else if (body.model === null) {
+                const next = setGuideThreadModel(threadId, null);
+                if (!next) {
+                    json(res, 404, { code: "guide.not_found", message: "" });
+                    return true;
+                }
+                record = next;
+            }
+            else {
+                json(res, 400, { code: "guide.invalid", message: "" });
+                return true;
+            }
+        }
+        json(res, 200, { thread: threadSummary(record), model: record.model ?? null });
+        return true;
+    }
+    if (messageMatch && method === "POST") {
+        const threadId = decodeURIComponent(messageMatch[1] ?? "");
+        const body = await readBody(req);
+        if (body.__tooLarge || body.__invalid) {
+            json(res, 400, { code: "guide.invalid", message: "" });
+            return true;
+        }
+        const text = typeof body.text === "string" ? body.text : "";
+        let started = false;
+        try {
+            for await (const frame of streamGuideTurn({
+                host,
+                threadId,
+                text,
+                attachments: body.attachments,
+                model: typeof body.model === "string" ? body.model : undefined,
+            })) {
+                if (!canWrite(res))
+                    continue;
+                if (!started) {
+                    started = true;
+                    res.writeHead(200, {
+                        "Content-Type": "application/x-ndjson; charset=utf-8",
+                        "Cache-Control": "no-store",
+                    });
+                }
+                res.write(`${JSON.stringify(frame)}\n`);
+            }
+        }
+        catch (err) {
+            const code = err instanceof GuideError ? err.code : "guide.failed";
+            const status = err instanceof GuideError ? err.status : 500;
+            if (!started && canWrite(res))
+                json(res, status, { code, message: "" });
+            else if (started && canWrite(res))
+                res.write(`${JSON.stringify({ type: "error", code, message: "" })}\n`);
+        }
+        finally {
+            if (!res.headersSent && canWrite(res))
+                json(res, 200, { code: "guide.failed", message: "" });
+            else if (canWrite(res))
+                res.end();
+        }
+        return true;
+    }
+    json(res, 405, { code: "guide.invalid", message: "" });
+    return true;
+}

package/dist/server/http.d.ts

@@ -1,6 +1,8 @@
 import type { StagePort } from "../agent/port.js";
 import { type ProviderAuthContext } from "../agent/providerAuth.js";
+import type { ChannelService } from "../channels/service.js";
 import type { EvalRunManager } from "../evals/manager.js";
+import type { MemoryHost } from "../memory/host.js";
 import { type LiveViewRoutesOptions } from "./liveViewRoutes.js";
 import { WorkshopChatSessionRegistry } from "../workshop/chatTurn.js";
 import { type AgentPort } from "../operatorAgent/index.js";
@@ -70,9 +72,17 @@ export type OperatorRouteDeps = {
     agentConversationBackend?: ConversationBackend;
     /** Shared with MCP when set by bootstrap. */
     agentHost?: AgentHostContext;
+    /** Bootstrap fills this once the operator host exists, so evals can run against it. */
+    operatorHostSlot?: {
+        current?: import("../operatorAgent/types.js").OperatorAgentHost;
+    };
     evalRunManager?: EvalRunManager;
     /** Wired by the eval manager; until then the internal eval callback answers 404. */
     evalServices?: EvalServices;
+    /** Host connections. Guide and MCP use the same service the console does. */
+    channels?: ChannelService;
+    /** Host memories. Guide and MCP use the same host the console does. */
+    memories?: MemoryHost;
 };
 /**
  * The full operator-console route surface: REST API (runs, stages,

package/dist/server/http.js

@@ -8,6 +8,7 @@ import { mapProviderAuthError } from "../agent/providerInspect.js";
 import { handleProviderRoutes } from "./providerRoutes.js";
 import { handleProjectMcpRoutes } from "./projectMcpRoutes.js";
 import { handleTriggerRoutes } from "./triggerRoutes.js";
+import { guideWriteRoot } from "../guide/writeRoot.js";
 import { handleEvalRoutes } from "./evalRoutes.js";
 import { defaultLoadJudgePrices } from "../evals/engine/promptfoo/judgeServices.js";
 import { engineStatus as promptfooEngineStatus } from "../evals/engine/promptfoo/status.js";
@@ -62,6 +63,8 @@ import { buildModelsResponse } from "../models/response.js";
 import { createAgentHostContext } from "../agents/hostContext.js";
 import { AgentScheduleSource, agentScheduleTickIntervalMsFromEnv, } from "../agents/scheduleSource.js";
 import { handleAgentAuthorRoutes } from "./agentAuthorRoutes.js";
+import { handleGuideRoutes } from "./guideRoutes.js";
+import { handleInternalAgentRoutes } from "./internalAgentRoutes.js";
 import { handleAgentRoutes } from "./agentRoutes.js";
 import { handleChannelRoutes } from "./channelRoutes.js";
 import { handleEvalInternalRoutes, isEvalInternalPath } from "./evalInternalRoutes.js";
@@ -217,7 +220,27 @@ export function createOperatorRoutes(deps) {
         resolveAgentPort({
             cwd,
             projectRoot: cwd,
+            guideDeps: {
+                manager,
+                store,
+                cwd,
+                agentDir,
+                projectRoot: guideWriteRoot(rootDir),
+                ...(providerAuthContext ? { providerAuthContext } : {}),
+                ...(deps.evalRunManager ? { evalRunManager: deps.evalRunManager } : {}),
+                ...(deps.channels ? { channels: deps.channels } : {}),
+                ...(deps.memories ? { memories: deps.memories } : {}),
+                ...(agentHost
+                    ? {
+                        agentService: agentHost.service,
+                        agentChat: agentHost.chat,
+                        agentConversationBackend: agentHost.backend,
+                    }
+                    : {}),
+            },
         });
+    if (deps.operatorHostSlot)
+        deps.operatorHostSlot.current = operatorHost;
     const workshopChatRegistry = deps.workshopChatRegistry ?? new WorkshopChatSessionRegistry(operatorHost);
     const workshopSessionStoreRoot = () => resolveWorkshopSessionStoreRoot();
     const workshopBuildStoreRoot = () => resolveWorkshopBuildStoreRoot();
@@ -263,6 +286,12 @@ export function createOperatorRoutes(deps) {
                     if (await handleAgentAuthorRoutes(req, method, pathname, cwd, operatorHost, res, json))
                         return true;
                 }
+                if (pathname === "/api/guide" || pathname.startsWith("/api/guide/")) {
+                    if (await handleGuideRoutes(req, method, pathname, operatorHost, res, json))
+                        return true;
+                }
+                if (handleInternalAgentRoutes(method, pathname, operatorHost, res, json))
+                    return true;
                 if (PublicWebChat.owns(pathname)) {
                     if (!agentHost) {
                         json(res, 503, { code: "web_chat.unavailable" });
@@ -1064,7 +1093,9 @@ export function createOperatorRoutes(deps) {
                     await readJsonBody(req);
                     const runId = decodeURIComponent(retryMatch[1] ?? "");
                     const stageId = decodeURIComponent(retryMatch[2] ?? "");
-                    const result = await manager.retryStage(runId, stageId);
+                    const result = await manager.retryStage(runId, stageId, {
+                        awaitRoot: false,
+                    });
                     if (!result.ok) {
                         json(res, result.status ?? 500, mapRetryStageFailure(result));
                         return true;
@@ -3178,8 +3209,11 @@ export async function startUiServer(options) {
                 ? { workshopOperatorHost: options.workshopOperatorHost }
                 : {}),
             ...(boot.agentHost ? { agentHost: boot.agentHost } : {}),
+            ...(boot.operatorHostSlot ? { operatorHostSlot: boot.operatorHostSlot } : {}),
             ...(boot.evalRunManager ? { evalRunManager: boot.evalRunManager } : {}),
             ...(boot.evalServices ? { evalServices: boot.evalServices } : {}),
+            ...(boot.channels ? { channels: boot.channels } : {}),
+            ...(boot.memories ? { memories: boot.memories } : {}),
             ...(options.agentConversationBackend
                 ? { agentConversationBackend: options.agentConversationBackend }
                 : {}),

package/dist/server/internalAgentRoutes.d.ts

@@ -0,0 +1,6 @@
+import type { ServerResponse } from "node:http";
+import type { OperatorAgentHost } from "../operatorAgent/index.js";
+type Json = (res: ServerResponse, status: number, body: unknown) => void;
+export declare const INTERNAL_AGENTS_PATH = "/api/internal-agents";
+export declare function handleInternalAgentRoutes(method: string, pathname: string, host: OperatorAgentHost, res: ServerResponse, json: Json, env?: NodeJS.ProcessEnv): boolean;
+export {};