stageflow 0.40.0 → 0.44.0

package/dist/runstore/sqlite/evalStore.js

@@ -0,0 +1,541 @@
+import { randomBytes } from "node:crypto";
+import { EVAL_LIMITS, EVAL_RUN_ID_PREFIX } from "../../evals/constants.js";
+import { EvalStoreError } from "../../evals/storeError.js";
+const now = () => new Date().toISOString();
+const bool = (value) => (value === null ? null : value !== 0);
+const json = (value) => (value === undefined || value === null ? null : JSON.stringify(value));
+const parse = (value) => (value === null ? null : JSON.parse(value));
+function runFromDb(row) {
+    return {
+        evalRunId: row.eval_run_id,
+        projectRoot: row.project_root,
+        evalId: row.eval_id,
+        runNumber: row.run_number,
+        evalPath: row.eval_path,
+        evalHash: row.eval_hash,
+        targetKind: row.target_kind,
+        targetRef: JSON.parse(row.target_ref_json),
+        targetVersionHash: row.target_version_hash,
+        datasetPath: row.dataset_path,
+        datasetHash: row.dataset_hash,
+        config: JSON.parse(row.config_json),
+        configFingerprint: row.config_fingerprint,
+        configComponents: JSON.parse(row.config_components_json),
+        status: row.status,
+        statusReason: row.status_reason,
+        passed: bool(row.passed),
+        repeats: row.repeats,
+        passThreshold: row.pass_threshold,
+        maxCostUsd: row.max_cost_usd,
+        casesTotal: row.cases_total,
+        resultsTotal: row.results_total,
+        resultsDone: row.results_done,
+        costUsd: row.cost_usd,
+        engineName: row.engine_name,
+        engineVersion: row.engine_version,
+        stageflowVersion: row.stageflow_version,
+        label: row.label,
+        startedVia: row.started_via,
+        callerId: row.caller_id,
+        summary: parse(row.summary_json),
+        createdAt: row.created_at,
+        startedAt: row.started_at,
+        finishedAt: row.finished_at,
+    };
+}
+function resultFromDb(row) {
+    return {
+        evalRunId: row.eval_run_id,
+        caseId: row.case_id,
+        repeatIndex: row.repeat_index,
+        status: row.status,
+        errorCode: row.error_code,
+        errorMessage: row.error_message,
+        stageRunId: row.stage_run_id,
+        subjectRef: row.subject_ref,
+        turns: row.turns,
+        toolCallCount: row.tool_call_count,
+        output: parse(row.output_json),
+        latencyMs: row.latency_ms,
+        costUsd: row.cost_usd,
+        tokensIn: row.tokens_in,
+        tokensOut: row.tokens_out,
+        score: row.score,
+        passed: row.passed !== 0,
+        hasGraderErrors: row.has_grader_errors !== 0,
+        budgetWarnings: parse(row.budget_warnings_json),
+        startedAt: row.started_at,
+        finishedAt: row.finished_at,
+    };
+}
+function gradeFromDb(row) {
+    return {
+        evalRunId: row.eval_run_id,
+        caseId: row.case_id,
+        repeatIndex: row.repeat_index,
+        graderId: row.grader_id,
+        graderType: row.grader_type,
+        status: row.status,
+        passed: bool(row.passed),
+        score: row.score,
+        rawValue: row.raw_value,
+        reason: row.reason,
+        details: parse(row.details_json),
+        errorCode: row.error_code,
+        judgeModel: row.judge_model,
+        judgeCostUsd: row.judge_cost_usd,
+        judgeTokensIn: row.judge_tokens_in,
+        judgeTokensOut: row.judge_tokens_out,
+    };
+}
+function insertGrade(db, evalRunId, caseId, repeatIndex, grade) {
+    db.prepare(`INSERT INTO eval_grades
+       (eval_run_id, case_id, repeat_index, grader_id, grader_type, status, passed, score, raw_value, reason,
+        details_json, error_code, judge_model, judge_cost_usd, judge_tokens_in, judge_tokens_out)
+     VALUES
+       (@eval_run_id, @case_id, @repeat_index, @grader_id, @grader_type, @status, @passed, @score, @raw_value, @reason,
+        @details_json, @error_code, @judge_model, @judge_cost_usd, @judge_tokens_in, @judge_tokens_out)`).run({
+        eval_run_id: evalRunId,
+        case_id: caseId,
+        repeat_index: repeatIndex,
+        grader_id: grade.graderId,
+        grader_type: grade.graderType,
+        status: grade.status,
+        passed: grade.passed === undefined || grade.passed === null ? null : grade.passed ? 1 : 0,
+        score: grade.score ?? null,
+        raw_value: grade.rawValue ?? null,
+        reason: grade.reason ?? null,
+        details_json: json(grade.details),
+        error_code: grade.errorCode ?? null,
+        judge_model: grade.judgeModel ?? null,
+        judge_cost_usd: grade.judgeCostUsd ?? null,
+        judge_tokens_in: grade.judgeTokensIn ?? null,
+        judge_tokens_out: grade.judgeTokensOut ?? null,
+    });
+}
+export function createEvalRun(db, input) {
+    const evalRunId = `${EVAL_RUN_ID_PREFIX}${randomBytes(6).toString("hex")}`;
+    db.transaction(() => {
+        const counter = db
+            .prepare(`INSERT INTO eval_run_counters (project_root, eval_id, last_number) VALUES (?, ?, 1)
+         ON CONFLICT (project_root, eval_id) DO UPDATE SET last_number = last_number + 1
+         RETURNING last_number`)
+            .get(input.projectRoot, input.evalId);
+        db.prepare(`INSERT INTO eval_runs
+         (eval_run_id, project_root, eval_id, run_number, eval_path, eval_hash, target_kind, target_ref_json,
+          target_version_hash, dataset_path, dataset_hash, config_json, config_fingerprint, config_components_json,
+          status, repeats, pass_threshold, max_cost_usd, cases_total, results_total, engine_name, engine_version,
+          stageflow_version, label, started_via, caller_id, created_at)
+       VALUES
+         (@eval_run_id, @project_root, @eval_id, @run_number, @eval_path, @eval_hash, @target_kind, @target_ref_json,
+          @target_version_hash, @dataset_path, @dataset_hash, @config_json, @config_fingerprint, @config_components_json,
+          @status, @repeats, @pass_threshold, @max_cost_usd, @cases_total, @results_total, @engine_name, @engine_version,
+          @stageflow_version, @label, @started_via, @caller_id, @created_at)`).run({
+            eval_run_id: evalRunId,
+            project_root: input.projectRoot,
+            eval_id: input.evalId,
+            run_number: counter.last_number,
+            eval_path: input.evalPath,
+            eval_hash: input.evalHash,
+            target_kind: input.targetKind,
+            target_ref_json: JSON.stringify(input.targetRef),
+            target_version_hash: input.targetVersionHash,
+            dataset_path: input.datasetPath,
+            dataset_hash: input.datasetHash,
+            config_json: JSON.stringify(input.config),
+            config_fingerprint: input.configFingerprint,
+            config_components_json: JSON.stringify(input.configComponents),
+            status: input.status ?? "queued",
+            repeats: input.repeats,
+            pass_threshold: input.passThreshold,
+            max_cost_usd: input.maxCostUsd,
+            cases_total: input.casesTotal,
+            results_total: input.resultsTotal,
+            engine_name: input.engineName,
+            engine_version: input.engineVersion,
+            stageflow_version: input.stageflowVersion,
+            label: input.label ?? null,
+            started_via: input.startedVia,
+            caller_id: input.callerId ?? null,
+            created_at: now(),
+        });
+    })();
+    return getEvalRun(db, evalRunId);
+}
+export function insertEvalCases(db, evalRunId, cases) {
+    const insert = db.prepare(`INSERT INTO eval_cases (eval_run_id, case_id, position, case_json) VALUES (?, ?, ?, ?)`);
+    db.transaction(() => {
+        for (const c of cases)
+            insert.run(evalRunId, c.caseId, c.position, JSON.stringify(c.case));
+    })();
+}
+export function upsertEvalResult(db, input) {
+    db.transaction(() => {
+        db.prepare(`INSERT INTO eval_results
+         (eval_run_id, case_id, repeat_index, status, error_code, error_message, stage_run_id, subject_ref, turns, tool_call_count, output_json, latency_ms,
+          cost_usd, tokens_in, tokens_out, score, passed, has_grader_errors, budget_warnings_json, started_at, finished_at)
+       VALUES
+         (@eval_run_id, @case_id, @repeat_index, @status, @error_code, @error_message, @stage_run_id, @subject_ref, @turns, @tool_call_count, @output_json, @latency_ms,
+          @cost_usd, @tokens_in, @tokens_out, @score, @passed, @has_grader_errors, @budget_warnings_json, @started_at, @finished_at)
+       ON CONFLICT (eval_run_id, case_id, repeat_index) DO UPDATE SET
+         status = excluded.status, error_code = excluded.error_code, error_message = excluded.error_message,
+         stage_run_id = excluded.stage_run_id, subject_ref = excluded.subject_ref, turns = excluded.turns,
+         tool_call_count = excluded.tool_call_count, output_json = excluded.output_json, latency_ms = excluded.latency_ms,
+         cost_usd = excluded.cost_usd, tokens_in = excluded.tokens_in, tokens_out = excluded.tokens_out,
+         score = excluded.score, passed = excluded.passed, has_grader_errors = excluded.has_grader_errors,
+         budget_warnings_json = excluded.budget_warnings_json, started_at = excluded.started_at,
+         finished_at = excluded.finished_at`).run({
+            eval_run_id: input.evalRunId,
+            case_id: input.caseId,
+            repeat_index: input.repeatIndex,
+            status: input.status,
+            error_code: input.errorCode ?? null,
+            error_message: input.errorMessage ?? null,
+            stage_run_id: input.stageRunId ?? null,
+            subject_ref: input.subjectRef ?? input.stageRunId ?? null,
+            turns: input.turns ?? null,
+            tool_call_count: input.toolCallCount ?? null,
+            output_json: json(input.output),
+            latency_ms: input.latencyMs ?? null,
+            cost_usd: input.costUsd ?? null,
+            tokens_in: input.tokensIn ?? null,
+            tokens_out: input.tokensOut ?? null,
+            score: input.score ?? null,
+            passed: input.passed ? 1 : 0,
+            has_grader_errors: input.hasGraderErrors ? 1 : 0,
+            budget_warnings_json: json(input.budgetWarnings),
+            started_at: input.startedAt ?? null,
+            finished_at: input.finishedAt ?? null,
+        });
+        db.prepare(`DELETE FROM eval_grades WHERE eval_run_id = ? AND case_id = ? AND repeat_index = ?`).run(input.evalRunId, input.caseId, input.repeatIndex);
+        for (const grade of input.grades ?? []) {
+            insertGrade(db, input.evalRunId, input.caseId, input.repeatIndex, grade);
+        }
+    })();
+}
+export function updateEvalRunProgress(db, evalRunId, progress) {
+    const sets = [];
+    const params = { eval_run_id: evalRunId };
+    if (progress.resultsDone !== undefined) {
+        sets.push("results_done = @results_done");
+        params.results_done = progress.resultsDone;
+    }
+    if (progress.costUsd !== undefined) {
+        sets.push("cost_usd = @cost_usd");
+        params.cost_usd = progress.costUsd;
+    }
+    if (progress.status !== undefined) {
+        sets.push("status = @status");
+        params.status = progress.status;
+    }
+    if (progress.statusReason !== undefined) {
+        sets.push("status_reason = @status_reason");
+        params.status_reason = progress.statusReason;
+    }
+    if (progress.startedAt !== undefined) {
+        sets.push("started_at = @started_at");
+        params.started_at = progress.startedAt;
+    }
+    if (sets.length === 0)
+        return;
+    const result = db.prepare(`UPDATE eval_runs SET ${sets.join(", ")} WHERE eval_run_id = @eval_run_id`).run(params);
+    if (result.changes === 0)
+        throw notFound(evalRunId);
+}
+export function finishEvalRun(db, evalRunId, input) {
+    const result = db
+        .prepare(`UPDATE eval_runs SET status = @status, status_reason = @status_reason, passed = @passed,
+         summary_json = @summary_json, finished_at = @finished_at WHERE eval_run_id = @eval_run_id`)
+        .run({
+        eval_run_id: evalRunId,
+        status: input.status,
+        status_reason: input.statusReason ?? null,
+        passed: input.passed ? 1 : 0,
+        summary_json: JSON.stringify(input.summary),
+        finished_at: input.finishedAt,
+    });
+    if (result.changes === 0)
+        throw notFound(evalRunId);
+}
+function notFound(evalRunId) {
+    return new EvalStoreError("EVAL_RUN_NOT_FOUND", `Eval run not found: ${evalRunId}`);
+}
+export function getEvalRun(db, evalRunId) {
+    const row = db.prepare(`SELECT * FROM eval_runs WHERE eval_run_id = ?`).get(evalRunId);
+    return row ? runFromDb(row) : null;
+}
+export function getEvalRunByNumber(db, projectRoot, evalId, runNumber) {
+    const row = db
+        .prepare(`SELECT * FROM eval_runs WHERE project_root = ? AND eval_id = ? AND run_number = ?`)
+        .get(projectRoot, evalId, runNumber);
+    return row ? runFromDb(row) : null;
+}
+export function listEvalRuns(db, query) {
+    const clauses = [];
+    const params = [];
+    if (query.projectRoot !== undefined) {
+        clauses.push("project_root = ?");
+        params.push(query.projectRoot);
+    }
+    if (query.evalId !== undefined) {
+        clauses.push("eval_id = ?");
+        params.push(query.evalId);
+    }
+    if (query.status !== undefined) {
+        clauses.push("status = ?");
+        params.push(query.status);
+    }
+    if (query.stageRunId !== undefined) {
+        clauses.push("eval_run_id IN (SELECT eval_run_id FROM eval_results WHERE stage_run_id = ?)");
+        params.push(query.stageRunId);
+    }
+    if (query.subjectRef !== undefined) {
+        clauses.push("eval_run_id IN (SELECT eval_run_id FROM eval_results WHERE subject_ref = ?)");
+        params.push(query.subjectRef);
+    }
+    if (query.cursor !== undefined) {
+        const [createdAt, id] = JSON.parse(Buffer.from(query.cursor, "base64url").toString("utf8"));
+        clauses.push("(created_at < ? OR (created_at = ? AND eval_run_id < ?))");
+        params.push(createdAt, createdAt, id);
+    }
+    const where = clauses.length > 0 ? `WHERE ${clauses.join(" AND ")}` : "";
+    const rows = db
+        .prepare(`SELECT * FROM eval_runs ${where} ORDER BY created_at DESC, eval_run_id DESC LIMIT ?`)
+        .all(...params, query.limit + 1);
+    const page = rows.slice(0, query.limit).map(runFromDb);
+    const last = page[page.length - 1];
+    const nextCursor = rows.length > query.limit && last
+        ? Buffer.from(JSON.stringify([last.createdAt, last.evalRunId]), "utf8").toString("base64url")
+        : null;
+    return { items: page, nextCursor };
+}
+function attachGrades(db, results, evalRunId, caseId) {
+    const grades = db
+        .prepare(`SELECT * FROM eval_grades WHERE eval_run_id = ? ${caseId !== undefined ? "AND case_id = ?" : ""}
+       ORDER BY case_id, repeat_index, rowid`)
+        .all(...(caseId !== undefined ? [evalRunId, caseId] : [evalRunId]));
+    const byResult = new Map();
+    for (const grade of grades.map(gradeFromDb)) {
+        const key = `${grade.caseId}\u0000${grade.repeatIndex}`;
+        const list = byResult.get(key);
+        if (list)
+            list.push(grade);
+        else
+            byResult.set(key, [grade]);
+    }
+    return results.map((result) => ({
+        ...result,
+        grades: byResult.get(`${result.caseId}\u0000${result.repeatIndex}`) ?? [],
+    }));
+}
+export function listEvalResults(db, evalRunId) {
+    const rows = db
+        .prepare(`SELECT r.* FROM eval_results r JOIN eval_cases c ON c.eval_run_id = r.eval_run_id AND c.case_id = r.case_id
+       WHERE r.eval_run_id = ? ORDER BY c.position, r.repeat_index`)
+        .all(evalRunId);
+    return attachGrades(db, rows.map(resultFromDb), evalRunId);
+}
+export function getEvalResultsForCase(db, evalRunId, caseId) {
+    const rows = db
+        .prepare(`SELECT * FROM eval_results WHERE eval_run_id = ? AND case_id = ? ORDER BY repeat_index`)
+        .all(evalRunId, caseId);
+    return attachGrades(db, rows.map(resultFromDb), evalRunId, caseId);
+}
+export function getEvalCases(db, evalRunId) {
+    const rows = db
+        .prepare(`SELECT eval_run_id, case_id, position, case_json FROM eval_cases WHERE eval_run_id = ? ORDER BY position`)
+        .all(evalRunId);
+    return rows.map((row) => ({
+        evalRunId: row.eval_run_id,
+        caseId: row.case_id,
+        position: row.position,
+        case: JSON.parse(row.case_json),
+    }));
+}
+function normalizeTags(tags) {
+    const seen = new Set();
+    const out = [];
+    for (const raw of tags) {
+        const name = raw.trim();
+        if (name.length < 1 || name.length > EVAL_LIMITS.failureTypeNameMaxChars) {
+            throw new Error(`Failure type must be 1-${EVAL_LIMITS.failureTypeNameMaxChars} characters`);
+        }
+        const key = name.toLowerCase();
+        if (seen.has(key))
+            continue;
+        seen.add(key);
+        out.push(name);
+    }
+    return out;
+}
+export function putEvalReview(db, evalRunId, caseId, repeatIndex, patch) {
+    if (typeof patch.note === "string" && patch.note.length > EVAL_LIMITS.reviewNoteMaxChars) {
+        throw new Error(`Review note must be at most ${EVAL_LIMITS.reviewNoteMaxChars} characters`);
+    }
+    const tags = patch.tags === undefined ? undefined : normalizeTags(patch.tags);
+    db.transaction(() => {
+        const result = db
+            .prepare(`SELECT r.passed, e.project_root, e.eval_id FROM eval_results r JOIN eval_runs e ON e.eval_run_id = r.eval_run_id
+         WHERE r.eval_run_id = ? AND r.case_id = ? AND r.repeat_index = ?`)
+            .get(evalRunId, caseId, repeatIndex);
+        if (!result) {
+            throw new EvalStoreError("EVAL_RUN_NOT_FOUND", `Eval result not found: ${evalRunId}/${caseId}/${repeatIndex}`);
+        }
+        if (tags !== undefined && tags.length > 0 && result.passed !== 0) {
+            throw new EvalStoreError("EVAL_TAGS_ON_PASSED_RESULT", "Failure types apply only to results marked failed");
+        }
+        const at = now();
+        const existing = db
+            .prepare(`SELECT 1 AS ok FROM eval_result_reviews WHERE eval_run_id = ? AND case_id = ? AND repeat_index = ?`)
+            .get(evalRunId, caseId, repeatIndex);
+        if (patch.label !== undefined) {
+            db.prepare(`INSERT INTO eval_result_reviews (eval_run_id, case_id, repeat_index, label, note, created_at, updated_at)
+         VALUES (@eval_run_id, @case_id, @repeat_index, @label, @note, @at, @at)
+         ON CONFLICT (eval_run_id, case_id, repeat_index) DO UPDATE SET
+           label = excluded.label,
+           note = ${patch.note === undefined ? "eval_result_reviews.note" : "excluded.note"},
+           updated_at = excluded.updated_at`).run({
+                eval_run_id: evalRunId,
+                case_id: caseId,
+                repeat_index: repeatIndex,
+                label: patch.label,
+                note: patch.note ?? null,
+                at,
+            });
+        }
+        else if (patch.note !== undefined) {
+            if (existing) {
+                db.prepare(`UPDATE eval_result_reviews SET note = ?, updated_at = ? WHERE eval_run_id = ? AND case_id = ? AND repeat_index = ?`).run(patch.note, at, evalRunId, caseId, repeatIndex);
+            }
+            else if (patch.note !== null) {
+                throw new Error("A review note needs a pass or fail label");
+            }
+        }
+        if (tags !== undefined) {
+            const keep = new Set(tags.map((t) => t.toLowerCase()));
+            const current = db
+                .prepare(`SELECT tag FROM eval_result_tags WHERE eval_run_id = ? AND case_id = ? AND repeat_index = ?`)
+                .all(evalRunId, caseId, repeatIndex);
+            const have = new Set(current.map((row) => row.tag.toLowerCase()));
+            for (const row of current) {
+                if (!keep.has(row.tag.toLowerCase())) {
+                    db.prepare(`DELETE FROM eval_result_tags WHERE eval_run_id = ? AND case_id = ? AND repeat_index = ? AND tag = ?`).run(evalRunId, caseId, repeatIndex, row.tag);
+                }
+            }
+            for (const tag of tags) {
+                db.prepare(`INSERT INTO eval_failure_types (project_root, eval_id, name, created_at) VALUES (?, ?, ?, ?)
+           ON CONFLICT DO NOTHING`).run(result.project_root, result.eval_id, tag, at);
+                if (!have.has(tag.toLowerCase())) {
+                    db.prepare(`INSERT INTO eval_result_tags (eval_run_id, case_id, repeat_index, tag, created_at) VALUES (?, ?, ?, ?, ?)`).run(evalRunId, caseId, repeatIndex, tag, at);
+                }
+            }
+        }
+    })();
+    return readReview(db, evalRunId, caseId, repeatIndex);
+}
+function readReview(db, evalRunId, caseId, repeatIndex) {
+    return (getEvalReviews(db, evalRunId).find((r) => r.caseId === caseId && r.repeatIndex === repeatIndex) ?? {
+        evalRunId,
+        caseId,
+        repeatIndex,
+        label: null,
+        note: null,
+        tags: [],
+        createdAt: null,
+        updatedAt: null,
+    });
+}
+export function getEvalReviews(db, evalRunId) {
+    const reviews = db
+        .prepare(`SELECT case_id, repeat_index, label, note, created_at, updated_at FROM eval_result_reviews WHERE eval_run_id = ?`)
+        .all(evalRunId);
+    const tagRows = db
+        .prepare(`SELECT case_id, repeat_index, tag FROM eval_result_tags WHERE eval_run_id = ? ORDER BY created_at, rowid`)
+        .all(evalRunId);
+    const merged = new Map();
+    const keyOf = (caseId, repeat) => `${caseId}\u0000${repeat}`;
+    for (const r of reviews) {
+        merged.set(keyOf(r.case_id, r.repeat_index), {
+            evalRunId,
+            caseId: r.case_id,
+            repeatIndex: r.repeat_index,
+            label: r.label,
+            note: r.note,
+            tags: [],
+            createdAt: r.created_at,
+            updatedAt: r.updated_at,
+        });
+    }
+    for (const t of tagRows) {
+        const key = keyOf(t.case_id, t.repeat_index);
+        let row = merged.get(key);
+        if (!row) {
+            row = {
+                evalRunId,
+                caseId: t.case_id,
+                repeatIndex: t.repeat_index,
+                label: null,
+                note: null,
+                tags: [],
+                createdAt: null,
+                updatedAt: null,
+            };
+            merged.set(key, row);
+        }
+        row.tags.push(t.tag);
+    }
+    return [...merged.values()].sort((a, b) => a.caseId.localeCompare(b.caseId) || a.repeatIndex - b.repeatIndex);
+}
+export function listFailureTypes(db, projectRoot, evalId) {
+    const rows = db
+        .prepare(`SELECT ft.name AS name FROM eval_failure_types ft
+       WHERE ft.project_root = ? AND ft.eval_id = ?
+       ORDER BY COALESCE(
+         (SELECT MAX(t.created_at) FROM eval_result_tags t JOIN eval_runs r ON r.eval_run_id = t.eval_run_id
+          WHERE r.project_root = ft.project_root AND r.eval_id = ft.eval_id AND t.tag = ft.name COLLATE NOCASE),
+         ft.created_at) DESC, ft.rowid DESC`)
+        .all(projectRoot, evalId);
+    return rows.map((row) => row.name);
+}
+export function recordGraderVersions(db, projectRoot, evalId, graders) {
+    const out = {};
+    db.transaction(() => {
+        for (const { graderId, graderHash } of graders) {
+            const existing = db
+                .prepare(`SELECT ordinal FROM eval_grader_versions WHERE project_root = ? AND eval_id = ? AND grader_id = ? AND grader_hash = ?`)
+                .get(projectRoot, evalId, graderId, graderHash);
+            if (existing) {
+                out[graderId] = existing.ordinal;
+                continue;
+            }
+            const max = db
+                .prepare(`SELECT MAX(ordinal) AS m FROM eval_grader_versions WHERE project_root = ? AND eval_id = ? AND grader_id = ?`)
+                .get(projectRoot, evalId, graderId);
+            const ordinal = (max.m ?? 0) + 1;
+            db.prepare(`INSERT INTO eval_grader_versions (project_root, eval_id, grader_id, grader_hash, ordinal, first_seen_at)
+         VALUES (?, ?, ?, ?, ?, ?)`).run(projectRoot, evalId, graderId, graderHash, ordinal, now());
+            out[graderId] = ordinal;
+        }
+    })();
+    return out;
+}
+export function listEvalSubjectRefs(db, evalRunId, targetKind) {
+    const run = getEvalRun(db, evalRunId);
+    if (!run || run.targetKind !== targetKind)
+        return [];
+    return db
+        .prepare(`SELECT DISTINCT subject_ref FROM eval_results WHERE eval_run_id = ? AND subject_ref IS NOT NULL ORDER BY subject_ref`)
+        .all(evalRunId).map((row) => row.subject_ref);
+}
+export function deleteEvalRun(db, evalRunId) {
+    let stageRunIds = [];
+    db.transaction(() => {
+        if (!getEvalRun(db, evalRunId))
+            throw notFound(evalRunId);
+        stageRunIds = db
+            .prepare(`SELECT DISTINCT stage_run_id FROM eval_results WHERE eval_run_id = ? AND stage_run_id IS NOT NULL ORDER BY stage_run_id`)
+            .all(evalRunId).map((row) => row.stage_run_id);
+        db.prepare(`DELETE FROM eval_runs WHERE eval_run_id = ?`).run(evalRunId);
+    })();
+    return stageRunIds;
+}

package/dist/runstore/sqlite/migrations/015-evals.d.ts

@@ -0,0 +1,12 @@
+import type Database from "better-sqlite3";
+/**
+ * Eval runs, their cases, results, grades and the author's reviews, plus `runs.origin` so stage runs
+ * started by an eval can be hidden from normal run views. `eval_run_counters` keeps `run_number`
+ * from being reused after an eval run is deleted.
+ */
+export declare const MIGRATION_015: {
+    version: number;
+    name: string;
+    minStageflowVersion: string;
+    up(db: Database.Database): void;
+};