stageflow 0.40.0 → 0.42.0

package/dist/runstore/sqlite/SqliteRunStore.d.ts

@@ -1,7 +1,11 @@
 import Database from "better-sqlite3";
+import type { CreateEvalRunInput, EvalCaseInput, EvalCaseRow, EvalGraderVersionInput, EvalResultInput, EvalResultWithGrades, EvalReviewPatch, EvalReviewRow, EvalRunPage, EvalRunProgress, EvalRunRow, FinishEvalRunInput, ListEvalRunsQuery } from "../../evals/storeTypes.js";
+import type { EvalTargetKind } from "../../evals/constants.js";
 import type { StageEnvelope } from "../../types/envelope.js";
 import type { StageLogLine } from "../../agent/activity.js";
 import { type CreateFeedbackLoopInput, type CreateFeedbackReplayInput, type CreateFeedbackReplayStagePassInput, type CreateForkGenerationInput, type CreateRunInput, type CreatedRun, type FeedbackLoopPatch, type FeedbackLoopRecord, type FeedbackLoopState, type FeedbackReplayPatch, type FeedbackReplayRecord, type FeedbackReplayStagePassPatch, type FeedbackReplayStagePassRecord, type ForkGenerationPatch, type ForkGenerationRecord, type ListRunsFilter, type RunDetail, type RunMeta, type RunPipelineDagSnapshot, type RunStatus, type RunStore, type RunSummary, type StageExecution, type StageExecutionPatch, type StageLogEvent, type TriggerRecord, type UpsertTriggerInput, type VerificationCheckResult, type VerificationCheckResultPatch, type ConfigOriginRecord } from "../port.js";
+import type { SuggestionStatus } from "../../evals/triage/constants.js";
+import type { SuggestionInput, SuggestionRow } from "../../evals/triage/types.js";
 import type { OverviewRunRow } from "../overviewSummary.js";
 import { type RunSubmissionRecord } from "../submission.js";
 type SqliteRunStoreOpenerMode = "assert" | "migrate";
@@ -78,6 +82,30 @@ export declare class SqliteRunStore implements RunStore {
     listStageEvents(runId: string, stageId: string, attempt?: number): Promise<StageLogEvent[]>;
     listRuns(filter?: ListRunsFilter): Promise<RunSummary[]>;
     listRunOverviewRows(since: string): Promise<OverviewRunRow[]>;
+    createEvalRun(input: CreateEvalRunInput): Promise<EvalRunRow>;
+    insertEvalCases(evalRunId: string, cases: EvalCaseInput[]): Promise<void>;
+    upsertEvalResult(result: EvalResultInput): Promise<void>;
+    updateEvalRunProgress(evalRunId: string, progress: EvalRunProgress): Promise<void>;
+    finishEvalRun(evalRunId: string, input: FinishEvalRunInput): Promise<void>;
+    getEvalRun(evalRunId: string): Promise<EvalRunRow | null>;
+    getEvalRunByNumber(projectRoot: string, evalId: string, runNumber: number): Promise<EvalRunRow | null>;
+    listEvalRuns(query: ListEvalRunsQuery): Promise<EvalRunPage>;
+    listEvalResults(evalRunId: string): Promise<EvalResultWithGrades[]>;
+    getEvalResultsForCase(evalRunId: string, caseId: string): Promise<EvalResultWithGrades[]>;
+    getEvalCases(evalRunId: string): Promise<EvalCaseRow[]>;
+    putEvalReview(evalRunId: string, caseId: string, repeatIndex: number, patch: EvalReviewPatch): Promise<EvalReviewRow>;
+    getEvalReviews(evalRunId: string): Promise<EvalReviewRow[]>;
+    listFailureTypes(projectRoot: string, evalId: string): Promise<string[]>;
+    recordGraderVersions(projectRoot: string, evalId: string, graders: EvalGraderVersionInput[]): Promise<Record<string, number>>;
+    listEvalSubjectRefs(evalRunId: string, targetKind: EvalTargetKind): Promise<string[]>;
+    upsertEvalSuggestions(evalRunId: string, inputs: SuggestionInput[]): Promise<SuggestionRow[]>;
+    listEvalSuggestions(evalRunId: string, status?: SuggestionStatus): Promise<SuggestionRow[]>;
+    dismissEvalSuggestions(evalRunId: string, ids: string[]): Promise<number>;
+    acceptEvalSuggestions(evalRunId: string, ids: string[]): Promise<{
+        accepted: number;
+        tagged: number;
+    }>;
+    deleteEvalRun(evalRunId: string): Promise<string[]>;
     ensureProject(absPath: string): Promise<string>;
     listRegisteredProjects(): Promise<string[]>;
     upsertTrigger(input: UpsertTriggerInput): Promise<TriggerRecord>;

package/dist/runstore/sqlite/SqliteRunStore.js

@@ -9,6 +9,8 @@ import { parsePipelineDagSnapshot } from "../pipelineDagSnapshot.js";
 import { newRunId, runWorkspaceDir } from "../paths.js";
 import { attemptAgentDir, attemptArtifactsDir, stageDir, } from "../workspaceLayout.js";
 import { importDiskRunsIfEmpty } from "./migrateFromDisk.js";
+import * as evalStore from "./evalStore.js";
+import * as suggestionStore from "./suggestionStore.js";
 import { applyPendingMigrations, assertSchemaVersion, } from "./migrations/index.js";
 import { StoreSchemaError } from "./storeSchemaError.js";
 import { applyStorePragmas } from "./applyStorePragmas.js";
@@ -220,9 +222,9 @@ export class SqliteRunStore {
             }
             const inserted = this.db
                 .prepare(`INSERT INTO runs
-          (run_id, pipeline_id, task_id, task_yaml, status, created_at, updated_at, checkout_root, pipeline_dag_json, git_sha, ci_pr_url, ci_job_url, pipeline_path, task_path, project_root, repository, ref, resolved_sha, run_branch, git_author_name, git_author_email, pipeline_source, pipeline_body, caller_id, run_manifest, skip_gates, dispatch_key)
+          (run_id, pipeline_id, task_id, task_yaml, status, created_at, updated_at, checkout_root, pipeline_dag_json, git_sha, ci_pr_url, ci_job_url, pipeline_path, task_path, project_root, repository, ref, resolved_sha, run_branch, git_author_name, git_author_email, pipeline_source, pipeline_body, caller_id, origin, run_manifest, skip_gates, dispatch_key)
          VALUES
-          (@run_id, @pipeline_id, @task_id, @task_yaml, @status, @created_at, @updated_at, @checkout_root, @pipeline_dag_json, @git_sha, @ci_pr_url, @ci_job_url, @pipeline_path, @task_path, @project_root, @repository, @ref, @resolved_sha, @run_branch, @git_author_name, @git_author_email, @pipeline_source, @pipeline_body, @caller_id, @run_manifest, @skip_gates, @dispatch_key)
+          (@run_id, @pipeline_id, @task_id, @task_yaml, @status, @created_at, @updated_at, @checkout_root, @pipeline_dag_json, @git_sha, @ci_pr_url, @ci_job_url, @pipeline_path, @task_path, @project_root, @repository, @ref, @resolved_sha, @run_branch, @git_author_name, @git_author_email, @pipeline_source, @pipeline_body, @caller_id, @origin, @run_manifest, @skip_gates, @dispatch_key)
           ON CONFLICT(dispatch_key) DO NOTHING`)
                 .run({
                 run_id: runId,
@@ -252,6 +254,7 @@ export class SqliteRunStore {
                 pipeline_source: pipelineSource,
                 pipeline_body: pipelineBody,
                 caller_id: input.callerId ?? null,
+                origin: input.origin ?? null,
                 run_manifest: input.runManifest !== undefined
                     ? JSON.stringify(input.runManifest)
                     : null,
@@ -854,9 +857,15 @@ export class SqliteRunStore {
             clauses.push("caller_id = ?");
             params.push(filter.caller_id);
         }
+        if (filter?.origin === "eval") {
+            clauses.push("origin = 'eval'");
+        }
+        else if (filter?.origin !== "any") {
+            clauses.push("origin IS NOT 'eval'");
+        }
         const where = clauses.length > 0 ? `WHERE ${clauses.join(" AND ")}` : "";
         const rows = this.db
-            .prepare(`SELECT run_id, pipeline_id, task_id, task_yaml, status, created_at, updated_at, checkout_root, pipeline_dag_json, git_sha, ci_pr_url, ci_job_url, pipeline_path, task_path, project_root, repository, ref, resolved_sha, run_branch, git_author_name, git_author_email, cancel_reason, finished_at, slimmed_at, disk_bytes, disk_measured_at, config_origins_json, pipeline_source, pipeline_body, caller_id, run_manifest, skip_gates
+            .prepare(`SELECT run_id, pipeline_id, task_id, task_yaml, status, created_at, updated_at, checkout_root, pipeline_dag_json, git_sha, ci_pr_url, ci_job_url, pipeline_path, task_path, project_root, repository, ref, resolved_sha, run_branch, git_author_name, git_author_email, cancel_reason, finished_at, slimmed_at, disk_bytes, disk_measured_at, config_origins_json, pipeline_source, pipeline_body, caller_id, origin, run_manifest, skip_gates
          FROM runs ${where} ORDER BY created_at DESC`)
             .all(...params);
         const summaries = [];
@@ -876,7 +885,7 @@ export class SqliteRunStore {
             .prepare(`SELECT r.run_id, r.pipeline_id, r.status, r.created_at, r.finished_at,
                 (SELECT SUM(e.cost_usd) FROM stage_executions e WHERE e.run_id = r.run_id) AS cost_usd
          FROM runs r
-         WHERE r.created_at >= ?
+         WHERE r.created_at >= ? AND r.origin IS NOT 'eval'
          ORDER BY r.created_at DESC`)
             .all(since);
         return rows.map((row) => ({
@@ -888,6 +897,90 @@ export class SqliteRunStore {
             ...(row.cost_usd != null ? { cost_usd: row.cost_usd } : {}),
         }));
     }
+    async createEvalRun(input) {
+        await this.ready();
+        return evalStore.createEvalRun(this.db, input);
+    }
+    async insertEvalCases(evalRunId, cases) {
+        await this.ready();
+        evalStore.insertEvalCases(this.db, evalRunId, cases);
+    }
+    async upsertEvalResult(result) {
+        await this.ready();
+        evalStore.upsertEvalResult(this.db, result);
+    }
+    async updateEvalRunProgress(evalRunId, progress) {
+        await this.ready();
+        evalStore.updateEvalRunProgress(this.db, evalRunId, progress);
+    }
+    async finishEvalRun(evalRunId, input) {
+        await this.ready();
+        evalStore.finishEvalRun(this.db, evalRunId, input);
+    }
+    async getEvalRun(evalRunId) {
+        await this.ready();
+        return evalStore.getEvalRun(this.db, evalRunId);
+    }
+    async getEvalRunByNumber(projectRoot, evalId, runNumber) {
+        await this.ready();
+        return evalStore.getEvalRunByNumber(this.db, projectRoot, evalId, runNumber);
+    }
+    async listEvalRuns(query) {
+        await this.ready();
+        return evalStore.listEvalRuns(this.db, query);
+    }
+    async listEvalResults(evalRunId) {
+        await this.ready();
+        return evalStore.listEvalResults(this.db, evalRunId);
+    }
+    async getEvalResultsForCase(evalRunId, caseId) {
+        await this.ready();
+        return evalStore.getEvalResultsForCase(this.db, evalRunId, caseId);
+    }
+    async getEvalCases(evalRunId) {
+        await this.ready();
+        return evalStore.getEvalCases(this.db, evalRunId);
+    }
+    async putEvalReview(evalRunId, caseId, repeatIndex, patch) {
+        await this.ready();
+        return evalStore.putEvalReview(this.db, evalRunId, caseId, repeatIndex, patch);
+    }
+    async getEvalReviews(evalRunId) {
+        await this.ready();
+        return evalStore.getEvalReviews(this.db, evalRunId);
+    }
+    async listFailureTypes(projectRoot, evalId) {
+        await this.ready();
+        return evalStore.listFailureTypes(this.db, projectRoot, evalId);
+    }
+    async recordGraderVersions(projectRoot, evalId, graders) {
+        await this.ready();
+        return evalStore.recordGraderVersions(this.db, projectRoot, evalId, graders);
+    }
+    async listEvalSubjectRefs(evalRunId, targetKind) {
+        await this.ready();
+        return evalStore.listEvalSubjectRefs(this.db, evalRunId, targetKind);
+    }
+    async upsertEvalSuggestions(evalRunId, inputs) {
+        await this.ready();
+        return suggestionStore.upsertEvalSuggestions(this.db, evalRunId, inputs);
+    }
+    async listEvalSuggestions(evalRunId, status) {
+        await this.ready();
+        return suggestionStore.listEvalSuggestions(this.db, evalRunId, status);
+    }
+    async dismissEvalSuggestions(evalRunId, ids) {
+        await this.ready();
+        return suggestionStore.dismissEvalSuggestions(this.db, evalRunId, ids);
+    }
+    async acceptEvalSuggestions(evalRunId, ids) {
+        await this.ready();
+        return suggestionStore.acceptEvalSuggestions(this.db, evalRunId, ids);
+    }
+    async deleteEvalRun(evalRunId) {
+        await this.ready();
+        return evalStore.deleteEvalRun(this.db, evalRunId);
+    }
     async ensureProject(absPath) {
         await this.ready();
         const resolved = path.resolve(absPath);
@@ -1450,6 +1543,7 @@ export class SqliteRunStore {
                 ? { pipeline_source: row.pipeline_source }
                 : {}),
             ...(row.caller_id != null ? { caller_id: row.caller_id } : {}),
+            ...(row.origin === "eval" ? { origin: "eval" } : {}),
             ...(row.run_manifest
                 ? { run_manifest: JSON.parse(row.run_manifest) }
                 : {}),
@@ -1464,7 +1558,7 @@ export class SqliteRunStore {
     }
     getRunRow(runId) {
         const row = this.db
-            .prepare(`SELECT run_id, pipeline_id, task_id, task_yaml, status, created_at, updated_at, checkout_root, pipeline_dag_json, git_sha, ci_pr_url, ci_job_url, pipeline_path, task_path, project_root, repository, ref, resolved_sha, run_branch, git_author_name, git_author_email, cancel_reason, finished_at, slimmed_at, disk_bytes, disk_measured_at, config_origins_json, pipeline_source, pipeline_body, caller_id, run_manifest, skip_gates
+            .prepare(`SELECT run_id, pipeline_id, task_id, task_yaml, status, created_at, updated_at, checkout_root, pipeline_dag_json, git_sha, ci_pr_url, ci_job_url, pipeline_path, task_path, project_root, repository, ref, resolved_sha, run_branch, git_author_name, git_author_email, cancel_reason, finished_at, slimmed_at, disk_bytes, disk_measured_at, config_origins_json, pipeline_source, pipeline_body, caller_id, origin, run_manifest, skip_gates
          FROM runs WHERE run_id = ?`)
             .get(runId);
         if (!row)

package/dist/runstore/sqlite/evalStore.d.ts

@@ -0,0 +1,20 @@
+import type Database from "better-sqlite3";
+import { type EvalTargetKind } from "../../evals/constants.js";
+import type { CreateEvalRunInput, EvalCaseInput, EvalCaseRow, EvalGraderVersionInput, EvalResultInput, EvalResultWithGrades, EvalReviewPatch, EvalReviewRow, EvalRunPage, EvalRunProgress, EvalRunRow, FinishEvalRunInput, ListEvalRunsQuery } from "../../evals/storeTypes.js";
+export declare function createEvalRun(db: Database.Database, input: CreateEvalRunInput): EvalRunRow;
+export declare function insertEvalCases(db: Database.Database, evalRunId: string, cases: EvalCaseInput[]): void;
+export declare function upsertEvalResult(db: Database.Database, input: EvalResultInput): void;
+export declare function updateEvalRunProgress(db: Database.Database, evalRunId: string, progress: EvalRunProgress): void;
+export declare function finishEvalRun(db: Database.Database, evalRunId: string, input: FinishEvalRunInput): void;
+export declare function getEvalRun(db: Database.Database, evalRunId: string): EvalRunRow | null;
+export declare function getEvalRunByNumber(db: Database.Database, projectRoot: string, evalId: string, runNumber: number): EvalRunRow | null;
+export declare function listEvalRuns(db: Database.Database, query: ListEvalRunsQuery): EvalRunPage;
+export declare function listEvalResults(db: Database.Database, evalRunId: string): EvalResultWithGrades[];
+export declare function getEvalResultsForCase(db: Database.Database, evalRunId: string, caseId: string): EvalResultWithGrades[];
+export declare function getEvalCases(db: Database.Database, evalRunId: string): EvalCaseRow[];
+export declare function putEvalReview(db: Database.Database, evalRunId: string, caseId: string, repeatIndex: number, patch: EvalReviewPatch): EvalReviewRow;
+export declare function getEvalReviews(db: Database.Database, evalRunId: string): EvalReviewRow[];
+export declare function listFailureTypes(db: Database.Database, projectRoot: string, evalId: string): string[];
+export declare function recordGraderVersions(db: Database.Database, projectRoot: string, evalId: string, graders: EvalGraderVersionInput[]): Record<string, number>;
+export declare function listEvalSubjectRefs(db: Database.Database, evalRunId: string, targetKind: EvalTargetKind): string[];
+export declare function deleteEvalRun(db: Database.Database, evalRunId: string): string[];