stageflow 0.40.0 → 0.42.0

package/dist/runstore/sqlite/migrations/015-evals.js

@@ -0,0 +1,157 @@
+import { PACKAGE_VERSION } from "../../../package-meta.js";
+/**
+ * Eval runs, their cases, results, grades and the author's reviews, plus `runs.origin` so stage runs
+ * started by an eval can be hidden from normal run views. `eval_run_counters` keeps `run_number`
+ * from being reused after an eval run is deleted.
+ */
+export const MIGRATION_015 = {
+    version: 15,
+    name: "015_evals",
+    minStageflowVersion: PACKAGE_VERSION,
+    up(db) {
+        db.exec(`
+ALTER TABLE runs ADD COLUMN origin TEXT;
+CREATE INDEX runs_origin ON runs (origin);
+
+CREATE TABLE eval_runs (
+  eval_run_id TEXT PRIMARY KEY,
+  project_root TEXT NOT NULL,
+  eval_id TEXT NOT NULL,
+  run_number INTEGER NOT NULL,
+  eval_path TEXT NOT NULL,
+  eval_hash TEXT NOT NULL,
+  target_kind TEXT NOT NULL CHECK (target_kind IN ('stage','agent')),
+  target_ref_json TEXT NOT NULL,
+  target_version_hash TEXT NOT NULL,
+  dataset_path TEXT NOT NULL,
+  dataset_hash TEXT NOT NULL,
+  config_json TEXT NOT NULL,
+  config_fingerprint TEXT NOT NULL,
+  config_components_json TEXT NOT NULL,
+  status TEXT NOT NULL CHECK (status IN ('queued','running','completed','failed','cancelled','budget_stopped')),
+  status_reason TEXT,
+  passed INTEGER,
+  repeats INTEGER NOT NULL,
+  pass_threshold REAL NOT NULL,
+  max_cost_usd REAL NOT NULL,
+  cases_total INTEGER NOT NULL,
+  results_total INTEGER NOT NULL,
+  results_done INTEGER NOT NULL DEFAULT 0,
+  cost_usd REAL NOT NULL DEFAULT 0,
+  engine_name TEXT NOT NULL,
+  engine_version TEXT NOT NULL,
+  stageflow_version TEXT NOT NULL,
+  label TEXT,
+  started_via TEXT NOT NULL CHECK (started_via IN ('console','cli','mcp')),
+  caller_id TEXT,
+  summary_json TEXT,
+  created_at TEXT NOT NULL,
+  started_at TEXT,
+  finished_at TEXT
+);
+CREATE UNIQUE INDEX eval_runs_number ON eval_runs (project_root, eval_id, run_number);
+CREATE INDEX eval_runs_eval ON eval_runs (project_root, eval_id, created_at DESC);
+CREATE INDEX eval_runs_status ON eval_runs (status);
+
+CREATE TABLE eval_run_counters (
+  project_root TEXT NOT NULL,
+  eval_id TEXT NOT NULL,
+  last_number INTEGER NOT NULL,
+  PRIMARY KEY (project_root, eval_id)
+);
+
+CREATE TABLE eval_cases (
+  eval_run_id TEXT NOT NULL REFERENCES eval_runs (eval_run_id) ON DELETE CASCADE,
+  case_id TEXT NOT NULL,
+  position INTEGER NOT NULL,
+  case_json TEXT NOT NULL,
+  PRIMARY KEY (eval_run_id, case_id)
+);
+
+CREATE TABLE eval_results (
+  eval_run_id TEXT NOT NULL REFERENCES eval_runs (eval_run_id) ON DELETE CASCADE,
+  case_id TEXT NOT NULL,
+  repeat_index INTEGER NOT NULL,
+  status TEXT NOT NULL CHECK (status IN ('completed','stage_failed','needs_input','error','skipped_budget','cancelled')),
+  error_code TEXT,
+  error_message TEXT,
+  stage_run_id TEXT,
+  output_json TEXT,
+  latency_ms INTEGER,
+  cost_usd REAL,
+  tokens_in INTEGER,
+  tokens_out INTEGER,
+  score REAL,
+  passed INTEGER NOT NULL DEFAULT 0,
+  has_grader_errors INTEGER NOT NULL DEFAULT 0,
+  budget_warnings_json TEXT,
+  started_at TEXT,
+  finished_at TEXT,
+  PRIMARY KEY (eval_run_id, case_id, repeat_index),
+  FOREIGN KEY (eval_run_id, case_id) REFERENCES eval_cases (eval_run_id, case_id) ON DELETE CASCADE
+);
+CREATE INDEX eval_results_stage_run ON eval_results (stage_run_id);
+
+CREATE TABLE eval_grades (
+  eval_run_id TEXT NOT NULL,
+  case_id TEXT NOT NULL,
+  repeat_index INTEGER NOT NULL,
+  grader_id TEXT NOT NULL,
+  grader_type TEXT NOT NULL,
+  status TEXT NOT NULL CHECK (status IN ('graded','error')),
+  passed INTEGER,
+  score REAL,
+  raw_value REAL,
+  reason TEXT,
+  details_json TEXT,
+  error_code TEXT,
+  judge_model TEXT,
+  judge_cost_usd REAL,
+  judge_tokens_in INTEGER,
+  judge_tokens_out INTEGER,
+  PRIMARY KEY (eval_run_id, case_id, repeat_index, grader_id),
+  FOREIGN KEY (eval_run_id, case_id, repeat_index) REFERENCES eval_results (eval_run_id, case_id, repeat_index) ON DELETE CASCADE
+);
+
+CREATE TABLE eval_grader_versions (
+  project_root TEXT NOT NULL,
+  eval_id TEXT NOT NULL,
+  grader_id TEXT NOT NULL,
+  grader_hash TEXT NOT NULL,
+  ordinal INTEGER NOT NULL,
+  first_seen_at TEXT NOT NULL,
+  PRIMARY KEY (project_root, eval_id, grader_id, grader_hash)
+);
+
+CREATE TABLE eval_failure_types (
+  project_root TEXT NOT NULL,
+  eval_id TEXT NOT NULL,
+  name TEXT NOT NULL,
+  created_at TEXT NOT NULL,
+  PRIMARY KEY (project_root, eval_id, name COLLATE NOCASE)
+);
+
+CREATE TABLE eval_result_reviews (
+  eval_run_id TEXT NOT NULL,
+  case_id TEXT NOT NULL,
+  repeat_index INTEGER NOT NULL,
+  label TEXT NOT NULL CHECK (label IN ('pass','fail')),
+  note TEXT,
+  created_at TEXT NOT NULL,
+  updated_at TEXT NOT NULL,
+  PRIMARY KEY (eval_run_id, case_id, repeat_index),
+  FOREIGN KEY (eval_run_id, case_id, repeat_index) REFERENCES eval_results (eval_run_id, case_id, repeat_index) ON DELETE CASCADE
+);
+
+CREATE TABLE eval_result_tags (
+  eval_run_id TEXT NOT NULL,
+  case_id TEXT NOT NULL,
+  repeat_index INTEGER NOT NULL,
+  tag TEXT NOT NULL,
+  created_at TEXT NOT NULL,
+  PRIMARY KEY (eval_run_id, case_id, repeat_index, tag COLLATE NOCASE),
+  FOREIGN KEY (eval_run_id, case_id, repeat_index) REFERENCES eval_results (eval_run_id, case_id, repeat_index) ON DELETE CASCADE
+);
+`);
+    },
+};

package/dist/runstore/sqlite/migrations/016-eval-subject-ref.d.ts

@@ -0,0 +1,12 @@
+import type Database from "better-sqlite3";
+/**
+ * Agent eval results point at an agent session, not a stage run: `subject_ref` is the root subject id
+ * (stage run id for stage results, agent session id for agent results). `turns` and `tool_call_count`
+ * are agent summaries. Additive; stage rows keep `stage_run_id` and are backfilled.
+ */
+export declare const MIGRATION_016: {
+    version: number;
+    name: string;
+    minStageflowVersion: string;
+    up(db: Database.Database): void;
+};

package/dist/runstore/sqlite/migrations/016-eval-subject-ref.js

@@ -0,0 +1,20 @@
+import { PACKAGE_VERSION } from "../../../package-meta.js";
+/**
+ * Agent eval results point at an agent session, not a stage run: `subject_ref` is the root subject id
+ * (stage run id for stage results, agent session id for agent results). `turns` and `tool_call_count`
+ * are agent summaries. Additive; stage rows keep `stage_run_id` and are backfilled.
+ */
+export const MIGRATION_016 = {
+    version: 16,
+    name: "016_eval_subject_ref",
+    minStageflowVersion: PACKAGE_VERSION,
+    up(db) {
+        db.exec(`
+ALTER TABLE eval_results ADD COLUMN subject_ref TEXT;
+ALTER TABLE eval_results ADD COLUMN turns INTEGER;
+ALTER TABLE eval_results ADD COLUMN tool_call_count INTEGER;
+UPDATE eval_results SET subject_ref = stage_run_id WHERE stage_run_id IS NOT NULL;
+CREATE INDEX eval_results_subject_ref ON eval_results (subject_ref);
+`);
+    },
+};

package/dist/runstore/sqlite/migrations/017-eval-result-suggestions.d.ts

@@ -0,0 +1,11 @@
+import type Database from "better-sqlite3";
+/**
+ * Suggested bucket and failure type per failed eval result, produced on request by a classifier
+ * (`jev` or `judge`). A suggestion becomes a tag only when the author accepts it. Additive.
+ */
+export declare const MIGRATION_017: {
+    version: number;
+    name: string;
+    minStageflowVersion: string;
+    up(db: Database.Database): void;
+};

package/dist/runstore/sqlite/migrations/017-eval-result-suggestions.js

@@ -0,0 +1,36 @@
+import { PACKAGE_VERSION } from "../../../package-meta.js";
+/**
+ * Suggested bucket and failure type per failed eval result, produced on request by a classifier
+ * (`jev` or `judge`). A suggestion becomes a tag only when the author accepts it. Additive.
+ */
+export const MIGRATION_017 = {
+    version: 17,
+    name: "017_eval_result_suggestions",
+    minStageflowVersion: PACKAGE_VERSION,
+    up(db) {
+        db.exec(`
+CREATE TABLE eval_result_suggestions (
+  id TEXT PRIMARY KEY,
+  eval_run_id TEXT NOT NULL,
+  case_id TEXT NOT NULL,
+  repeat_index INTEGER NOT NULL,
+  source TEXT NOT NULL CHECK (source IN ('jev','judge')),
+  bucket TEXT NOT NULL CHECK (bucket IN ('agent_wrong','grader_wrong','case_wrong','unclear')),
+  failure_type TEXT,
+  confidence REAL NOT NULL,
+  probabilities_json TEXT NOT NULL,
+  grader_false_alarms_json TEXT NOT NULL,
+  model TEXT NOT NULL,
+  prompt_version INTEGER NOT NULL,
+  status TEXT NOT NULL DEFAULT 'pending' CHECK (status IN ('pending','accepted','dismissed')),
+  batch_id TEXT NOT NULL,
+  cost_usd REAL NOT NULL DEFAULT 0,
+  state_truncated INTEGER NOT NULL DEFAULT 0,
+  created_at TEXT NOT NULL,
+  UNIQUE (eval_run_id, case_id, repeat_index, source),
+  FOREIGN KEY (eval_run_id, case_id, repeat_index) REFERENCES eval_results (eval_run_id, case_id, repeat_index) ON DELETE CASCADE
+);
+CREATE INDEX eval_result_suggestions_run ON eval_result_suggestions (eval_run_id, status);
+`);
+    },
+};

package/dist/runstore/sqlite/migrations/index.js

@@ -13,6 +13,9 @@ import { MIGRATION_011 } from "./011-email-dispatch-key.js";
 import { MIGRATION_012 } from "./012-trigger-fires.js";
 import { MIGRATION_013 } from "./013-agents.js";
 import { MIGRATION_014 } from "./014-agent-deliveries.js";
+import { MIGRATION_015 } from "./015-evals.js";
+import { MIGRATION_016 } from "./016-eval-subject-ref.js";
+import { MIGRATION_017 } from "./017-eval-result-suggestions.js";
 const DEFAULT_MIGRATIONS = [
     MIGRATION_001,
     MIGRATION_002,
@@ -28,6 +31,9 @@ const DEFAULT_MIGRATIONS = [
     MIGRATION_012,
     MIGRATION_013,
     MIGRATION_014,
+    MIGRATION_015,
+    MIGRATION_016,
+    MIGRATION_017,
 ];
 export const CURRENT_SCHEMA_VERSION = DEFAULT_MIGRATIONS[DEFAULT_MIGRATIONS.length - 1].version;
 function throwStoreSchemaTooNew(db, version, maxKnown) {

package/dist/runstore/sqlite/suggestionStore.d.ts

@@ -0,0 +1,14 @@
+import type Database from "better-sqlite3";
+import type { SuggestionInput, SuggestionRow } from "../../evals/triage/types.js";
+/** Inserts or replaces pending suggestions; accepted and dismissed ones are kept. Returns the rows this call wrote. */
+export declare function upsertEvalSuggestions(db: Database.Database, evalRunId: string, inputs: SuggestionInput[]): SuggestionRow[];
+export declare function listEvalSuggestions(db: Database.Database, evalRunId: string, status?: SuggestionRow["status"]): SuggestionRow[];
+export declare function dismissEvalSuggestions(db: Database.Database, evalRunId: string, ids: string[]): number;
+/**
+ * Marks pending suggestions accepted and adds their failure type and `bucket:<bucket>` tags to the result
+ * through `putEvalReview`, keeping tags already there. Returns how many were accepted and how many results gained a tag.
+ */
+export declare function acceptEvalSuggestions(db: Database.Database, evalRunId: string, ids: string[]): {
+    accepted: number;
+    tagged: number;
+};

package/dist/runstore/sqlite/suggestionStore.js

@@ -0,0 +1,132 @@
+import { randomBytes } from "node:crypto";
+import { EvalStoreError } from "../../evals/storeError.js";
+import { BUCKET_TAG_PREFIX } from "../../evals/triage/constants.js";
+import { getEvalReviews, putEvalReview } from "./evalStore.js";
+const now = () => new Date().toISOString();
+function suggestionFromDb(row) {
+    return {
+        id: row.id,
+        evalRunId: row.eval_run_id,
+        caseId: row.case_id,
+        repeatIndex: row.repeat_index,
+        source: row.source,
+        bucket: row.bucket,
+        failureType: row.failure_type,
+        confidence: row.confidence,
+        bucketProbabilities: JSON.parse(row.probabilities_json),
+        graderFalseAlarms: JSON.parse(row.grader_false_alarms_json),
+        model: row.model,
+        promptVersion: row.prompt_version,
+        status: row.status,
+        batchId: row.batch_id,
+        costUsd: row.cost_usd,
+        stateTruncated: row.state_truncated === 1,
+        createdAt: row.created_at,
+    };
+}
+/** Inserts or replaces pending suggestions; accepted and dismissed ones are kept. Returns the rows this call wrote. */
+export function upsertEvalSuggestions(db, evalRunId, inputs) {
+    const written = [];
+    const at = now();
+    db.transaction(() => {
+        for (const input of inputs) {
+            const info = db
+                .prepare(`INSERT INTO eval_result_suggestions
+             (id, eval_run_id, case_id, repeat_index, source, bucket, failure_type, confidence, probabilities_json,
+              grader_false_alarms_json, model, prompt_version, status, batch_id, cost_usd, state_truncated, created_at)
+           VALUES (@id, @eval_run_id, @case_id, @repeat_index, @source, @bucket, @failure_type, @confidence, @probabilities_json,
+              @grader_false_alarms_json, @model, @prompt_version, 'pending', @batch_id, @cost_usd, @state_truncated, @at)
+           ON CONFLICT (eval_run_id, case_id, repeat_index, source) DO UPDATE SET
+             bucket = excluded.bucket, failure_type = excluded.failure_type, confidence = excluded.confidence,
+             probabilities_json = excluded.probabilities_json, grader_false_alarms_json = excluded.grader_false_alarms_json,
+             model = excluded.model, prompt_version = excluded.prompt_version, batch_id = excluded.batch_id,
+             cost_usd = excluded.cost_usd, state_truncated = excluded.state_truncated, created_at = excluded.created_at
+           WHERE eval_result_suggestions.status = 'pending'`)
+                .run({
+                id: `sg_${randomBytes(8).toString("hex")}`,
+                eval_run_id: evalRunId,
+                case_id: input.caseId,
+                repeat_index: input.repeatIndex,
+                source: input.source,
+                bucket: input.bucket,
+                failure_type: input.failureType,
+                confidence: input.confidence,
+                probabilities_json: JSON.stringify(input.bucketProbabilities),
+                grader_false_alarms_json: JSON.stringify(input.graderFalseAlarms),
+                model: input.model,
+                prompt_version: input.promptVersion,
+                batch_id: input.batchId,
+                cost_usd: input.costUsd,
+                state_truncated: input.stateTruncated ? 1 : 0,
+                at,
+            });
+            if (info.changes === 0)
+                continue;
+            const row = db
+                .prepare(`SELECT * FROM eval_result_suggestions WHERE eval_run_id = ? AND case_id = ? AND repeat_index = ? AND source = ?`)
+                .get(evalRunId, input.caseId, input.repeatIndex, input.source);
+            written.push(suggestionFromDb(row));
+        }
+    })();
+    return written;
+}
+export function listEvalSuggestions(db, evalRunId, status) {
+    const rows = (status !== undefined
+        ? db
+            .prepare(`SELECT * FROM eval_result_suggestions WHERE eval_run_id = ? AND status = ? ORDER BY case_id, repeat_index, source`)
+            .all(evalRunId, status)
+        : db
+            .prepare(`SELECT * FROM eval_result_suggestions WHERE eval_run_id = ? ORDER BY case_id, repeat_index, source`)
+            .all(evalRunId));
+    return rows.map(suggestionFromDb);
+}
+export function dismissEvalSuggestions(db, evalRunId, ids) {
+    let changed = 0;
+    db.transaction(() => {
+        for (const id of ids) {
+            changed += db
+                .prepare(`UPDATE eval_result_suggestions SET status = 'dismissed' WHERE eval_run_id = ? AND id = ? AND status = 'pending'`)
+                .run(evalRunId, id).changes;
+        }
+    })();
+    return changed;
+}
+/**
+ * Marks pending suggestions accepted and adds their failure type and `bucket:<bucket>` tags to the result
+ * through `putEvalReview`, keeping tags already there. Returns how many were accepted and how many results gained a tag.
+ */
+export function acceptEvalSuggestions(db, evalRunId, ids) {
+    let accepted = 0;
+    let tagged = 0;
+    db.transaction(() => {
+        for (const id of ids) {
+            const found = db
+                .prepare(`SELECT * FROM eval_result_suggestions WHERE eval_run_id = ? AND id = ? AND status = 'pending'`)
+                .get(evalRunId, id);
+            if (found === undefined)
+                continue;
+            const suggestion = suggestionFromDb(found);
+            const add = [
+                ...(suggestion.failureType !== null ? [suggestion.failureType] : []),
+                `${BUCKET_TAG_PREFIX}${suggestion.bucket}`,
+            ];
+            const have = getEvalReviews(db, evalRunId).find((r) => r.caseId === suggestion.caseId && r.repeatIndex === suggestion.repeatIndex)?.tags ?? [];
+            const haveKeys = new Set(have.map((t) => t.toLowerCase()));
+            const fresh = add.filter((t) => !haveKeys.has(t.toLowerCase()));
+            if (fresh.length > 0) {
+                try {
+                    putEvalReview(db, evalRunId, suggestion.caseId, suggestion.repeatIndex, { tags: [...have, ...fresh] });
+                    tagged += 1;
+                }
+                catch (err) {
+                    if (err instanceof EvalStoreError && err.code === "EVAL_TAGS_ON_PASSED_RESULT")
+                        continue;
+                    throw err;
+                }
+            }
+            db.prepare(`UPDATE eval_result_suggestions SET status = 'accepted' WHERE id = ?`).run(id);
+            accepted += 1;
+        }
+    })();
+    return { accepted, tagged };
+}

package/dist/runtime/pipelineRunner.d.ts

@@ -1,7 +1,7 @@
 import type { RunSubmission } from "../runstore/submission.js";
 import type { StagePort } from "../agent/port.js";
 import { type ValidationFinding } from "../config/validateCatalog.js";
-import type { RunStore } from "../runstore/port.js";
+import type { RunOrigin, RunStore } from "../runstore/port.js";
 import type { StageEnvelope } from "../types/envelope.js";
 import type { StageHitlController } from "./stageHitl.js";
 import type { InlinePipelineDefinition, LoadedPipeline } from "../types/pipeline.js";
@@ -130,6 +130,7 @@ export declare function startPipeline(options: {
         halted: boolean;
     };
     callerId?: string | null;
+    origin?: RunOrigin;
     skills?: SkillsPayload;
 }): Promise<StartedPipeline>;
 export declare function resolveTaskPath(taskArg: string, cwd?: string): string;

package/dist/runtime/pipelineRunner.js

@@ -224,6 +224,7 @@ async function preparePipeline(options) {
                 ? { pipelineBody: persistence.fields.pipelineBody }
                 : {}),
             ...(callerId !== null ? { callerId } : {}),
+            ...(options.origin !== undefined ? { origin: options.origin } : {}),
             runManifest,
             skipGates: options.skipGates,
         });
@@ -329,6 +330,7 @@ export async function startPipeline(options) {
         operatorCatalog: options.operatorCatalog,
         skipGates: options.skipGates,
         callerId: options.callerId,
+        ...(options.origin !== undefined ? { origin: options.origin } : {}),
         ...(options.skills !== undefined ? { skills: options.skills } : {}),
     });
     if (prepared.run.created === false) {

package/dist/runtime/runManager.d.ts

@@ -2,7 +2,7 @@ import { type RunSubmission } from "../runstore/submission.js";
 import type { StagePort, OpaqueAnswer } from "../agent/port.js";
 import type { InlinePipelineDefinition } from "../types/pipeline.js";
 import { type DiskBreakdown, type FreeSpaceReader } from "../runstore/diskUsage.js";
-import { type RunStore } from "../runstore/port.js";
+import { type RunOrigin, type RunStore } from "../runstore/port.js";
 import type { TaskFile } from "../types/task.js";
 import { type PipelineRunResult } from "./pipelineRunner.js";
 import { type SkillsPayload } from "./runSkills.js";
@@ -314,6 +314,7 @@ export declare class RunManager {
         projectRoot?: string;
         /** Attribution from auth only — never from request body/query. */
         callerId?: string | null;
+        origin?: RunOrigin;
         skills?: SkillsPayload;
     }, submission?: RunSubmission): Promise<StartRunResult>;
     rerun(runId: string, options?: {

package/dist/runtime/runManager.js

@@ -530,7 +530,7 @@ export class RunManager {
      */
     async attachWaitingStages() {
         const found = [];
-        const runs = await this.options.store.listRuns();
+        const runs = await this.options.store.listRuns({ origin: "any" });
         for (const summary of runs) {
             if (summary.status !== "running")
                 continue;
@@ -627,7 +627,7 @@ export class RunManager {
     }
     async reconcileOrphanedStages() {
         const reconciled = [];
-        const runs = await this.options.store.listRuns();
+        const runs = await this.options.store.listRuns({ origin: "any" });
         for (const summary of runs) {
             let meta;
             try {
@@ -806,7 +806,7 @@ export class RunManager {
     }
     async resumeStalledSchedules() {
         const resumed = [];
-        const runs = await this.options.store.listRuns();
+        const runs = await this.options.store.listRuns({ origin: "any" });
         for (const summary of runs) {
             if (summary.status !== "running")
                 continue;
@@ -828,7 +828,7 @@ export class RunManager {
     }
     /** Rebuild in-memory admission queue from persisted `queued` rows (R27). */
     async reenqueuePersistedQueuedRuns() {
-        const queued = await this.options.store.listRuns({ status: "queued" });
+        const queued = await this.options.store.listRuns({ status: "queued", origin: "any" });
         const ordered = [...queued].sort((a, b) => a.created_at.localeCompare(b.created_at));
         this.admissionQueue.clear();
         for (const row of ordered) {
@@ -1273,7 +1273,7 @@ export class RunManager {
             gitSha: input.gitSha,
             ciPrUrl: input.ciPrUrl,
             ciJobUrl: input.ciJobUrl,
-        }, derivedProjectRoot, resolved.kind === "path" ? resolved.taskPath : undefined, submission, undefined, input.callerId, input.skills, input.dispatchKey);
+        }, derivedProjectRoot, resolved.kind === "path" ? resolved.taskPath : undefined, submission, undefined, input.callerId, input.skills, input.dispatchKey, input.origin);
     }
     async rerun(runId, options) {
         if (!this.acceptingWork) {
@@ -2219,7 +2219,7 @@ export class RunManager {
             this.attachedWaiting.delete(`${runId}\0${stageId}`);
         }
     }
-    async reserveAndStartPipeline(taskYaml, pipeline, taskLabel, cwd, checkoutOverride, skipGates, ciIdentity, projectRoot, taskPath, submission, pinned, callerId, skills, dispatchKey) {
+    async reserveAndStartPipeline(taskYaml, pipeline, taskLabel, cwd, checkoutOverride, skipGates, ciIdentity, projectRoot, taskPath, submission, pinned, callerId, skills, dispatchKey, origin) {
         const persistence = pipelinePersistenceForStart(pipeline);
         if (!persistence.ok) {
             return {
@@ -2419,6 +2419,7 @@ export class RunManager {
                 ...(callerId !== undefined && callerId !== null
                     ? { callerId }
                     : {}),
+                ...(origin !== undefined ? { origin } : {}),
                 runManifest,
                 skipGates,
             });
@@ -2506,6 +2507,7 @@ export class RunManager {
                 skipGates,
                 schedulingHalt,
                 callerId,
+                ...(origin !== undefined ? { origin } : {}),
                 ...(hasSkills ? { skills: validatedSkills } : {}),
             });
             if (started.created === false)

package/dist/runtime/runRetentionSweep.js

@@ -252,7 +252,7 @@ export async function runRetentionSweep(store, a2aStore, options = {}) {
     const windows = options.windows ?? retentionWindowsFromEnv(env);
     const artifactMaxBytes = options.artifactMaxBytes ?? slimArtifactMaxBytesFromEnv(env);
     const bareCacheTtlMs = options.bareCacheTtlMs ?? bareCacheTtlFromEnv(env);
-    const runs = await store.listRuns();
+    const runs = await store.listRuns({ origin: "any" });
     const slimmed = [];
     const purged = [];
     const metas = new Map();
@@ -305,7 +305,7 @@ export async function runRetentionSweep(store, a2aStore, options = {}) {
         }
     }
     // Re-list for bare-cache decisions after PURGE removed rows when execute.
-    const runsForBare = execute ? await store.listRuns() : runs;
+    const runsForBare = execute ? await store.listRuns({ origin: "any" }) : runs;
     const bareCachesEvicted = await evictBareCaches({
         runs: runsForBare,
         now,

package/dist/server/agentAuthorRoutes.js

@@ -1,4 +1,6 @@
-import { parseAgentAuthorDraft, runAgentAuthorTurn, undoAgentAuthorTurn } from "../agents/authorChat.js";
+import { authorSessionForApi, parseAgentAuthorDraft, runAgentAuthorTurn, undoAgentAuthorTurn, } from "../agents/authorChat.js";
+import { createAuthorSession, listAuthorDrafts, readAuthorSession, saveAuthorSession } from "../agents/authorSessionStore.js";
+import { parseWorkshopModel } from "../workshop/modelSettings.js";
 async function readBody(req) {
     const chunks = [];
     let size = 0;
@@ -24,6 +26,51 @@ async function readBody(req) {
 export async function handleAgentAuthorRoutes(req, method, pathname, projectRoot, host, res, json) {
     const chat = pathname === "/api/agent-author/chat";
     const undo = pathname === "/api/agent-author/undo";
+    const sessions = pathname === "/api/agent-author/sessions";
+    const sessionMatch = /^\/api\/agent-author\/sessions\/([^/]+)$/.exec(pathname);
+    if (!chat && !undo && !sessions && !sessionMatch)
+        return false;
+    if (sessions && method === "POST") {
+        const body = await readBody(req);
+        if (body.__tooLarge || body.__invalid) {
+            json(res, 400, { code: "agent.invalid" });
+            return true;
+        }
+        json(res, 200, createAuthorSession({
+            draft: parseAgentAuthorDraft(body.draft),
+            ...(parseWorkshopModel(body.model) ? { model: parseWorkshopModel(body.model) } : {}),
+        }));
+        return true;
+    }
+    if (sessions && method === "GET") {
+        json(res, 200, { drafts: listAuthorDrafts() });
+        return true;
+    }
+    if (sessionMatch && (method === "GET" || method === "PUT")) {
+        const sessionId = decodeURIComponent(sessionMatch[1] ?? "");
+        try {
+            if (method === "GET") {
+                json(res, 200, authorSessionForApi(sessionId));
+                return true;
+            }
+            const body = await readBody(req);
+            if (body.__tooLarge || body.__invalid) {
+                json(res, 400, { code: "agent.invalid" });
+                return true;
+            }
+            const current = readAuthorSession(sessionId);
+            if (!current) {
+                json(res, 404, { code: "agent.session_not_found" });
+                return true;
+            }
+            json(res, 200, saveAuthorSession({ ...current, draft: parseAgentAuthorDraft(body.draft) }));
+        }
+        catch (err) {
+            const code = err instanceof Error ? err.message : "agent.internal";
+            json(res, code === "agent.session_not_found" ? 404 : 400, { code: code.startsWith("agent.") ? code : "agent.internal" });
+        }
+        return true;
+    }
     if (!chat && !undo)
         return false;
     if (method !== "POST") {
@@ -46,12 +93,14 @@ export async function handleAgentAuthorRoutes(req, method, pathname, projectRoot
             return true;
         }
         const message = typeof body.message === "string" ? body.message : "";
+        const model = parseWorkshopModel(body.model);
         const result = await runAgentAuthorTurn({
             host,
             projectRoot,
             message,
             ...(typeof body.sessionId === "string" ? { sessionId: body.sessionId } : {}),
             draft: parseAgentAuthorDraft(body.draft),
+            ...(model ? { model } : {}),
         });
         json(res, 200, result);
     }

package/dist/server/bootstrap.d.ts

@@ -7,6 +7,8 @@ import type { RunStore } from "../runstore/port.js";
 import type { StoreFilesystemClassification } from "../runstore/storeFilesystem.js";
 import type { StageBrowserSupport } from "../browser/browserHost.js";
 import { RunManager } from "../runtime/runManager.js";
+import { EvalRunManager } from "../evals/manager.js";
+import type { EvalInternalServices } from "./evalInternalRoutes.js";
 import { emailHostFor } from "../email/host.js";
 import { EmailTriggers } from "../email/triggers.js";
 import { type ChannelService } from "../channels/service.js";
@@ -48,6 +50,12 @@ export type StageflowHostBootstrap = {
     store?: RunStore;
     /** Absent when `serveBlocked` — no RunManager lifecycle/resume. */
     manager?: RunManager;
+    /** Eval runs: shared with HTTP routes and MCP. Absent when `manager` is. */
+    evalRunManager?: EvalRunManager;
+    /** Services for the internal eval callback route; absent when `evalRunManager` is. */
+    evalServices?: EvalInternalServices;
+    /** Set once the HTTP host is listening so the promptfoo provider knows where to call back. */
+    setEvalCallbackBase?: (baseUrl: string) => void;
     runChangeBus: RunChangeBus;
     mcpStateless: boolean;
     providerAuthContext: ProviderAuthContext | undefined;