stageflow 0.40.0 → 0.44.0

package/dist/evals/estimate.d.ts

@@ -0,0 +1,56 @@
+export interface EstimatePreviousRun {
+    runNumber: number;
+    costStage: number;
+    costJudge: number;
+    resultsExecuted: number;
+    rubricGradesGraded: number;
+    wallSeconds: number | null;
+}
+export interface TokenPrice {
+    inputPerMillion: number;
+    outputPerMillion: number;
+}
+export declare const AGENT_ESTIMATE_ASSUMPTIONS: {
+    readonly maxTurnsCap: 6;
+    readonly toolCallsPerTurn: 2;
+    readonly agentCall: {
+        readonly tokensIn: 2000;
+        readonly tokensOut: 200;
+    };
+    readonly simulatorCall: {
+        readonly tokensIn: 1000;
+        readonly tokensOut: 60;
+    };
+    readonly judgeCall: {
+        readonly tokensIn: 3000;
+        readonly tokensOut: 100;
+    };
+};
+export interface AgentEstimateBlock {
+    turnsPerCase: number;
+    simulatorCallsPerCase: number;
+    judgeTranscriptTokens: number;
+    agentPrice?: TokenPrice | null;
+    simulatorPrice?: TokenPrice | null;
+    judgePrice?: TokenPrice | null;
+}
+export interface EstimateRunInput {
+    resultsTotal: number;
+    rubricGraderCount: number;
+    previous: EstimatePreviousRun | null;
+    recentStageCosts: number[];
+    agent?: AgentEstimateBlock;
+}
+export interface RunEstimate {
+    estimate_usd: number | null;
+    avg_stage_cost: number | null;
+    avg_judge_cost: number | null;
+    stage_calls: number;
+    judge_calls: number;
+    judge_excluded: boolean;
+    based_on_run_number: number | null;
+    estimate_seconds: number | null;
+    simulator_calls?: number;
+    estimate_basis?: "previous_run" | "assumptions" | null;
+}
+export declare function estimateRun(input: EstimateRunInput): RunEstimate;

package/dist/evals/estimate.js

@@ -0,0 +1,66 @@
+export const AGENT_ESTIMATE_ASSUMPTIONS = {
+    maxTurnsCap: 6,
+    toolCallsPerTurn: 2,
+    agentCall: { tokensIn: 2000, tokensOut: 200 },
+    simulatorCall: { tokensIn: 1000, tokensOut: 60 },
+    judgeCall: { tokensIn: 3000, tokensOut: 100 },
+};
+function callCost(price, tokensIn, tokensOut) {
+    return (tokensIn * price.inputPerMillion + tokensOut * price.outputPerMillion) / 1_000_000;
+}
+function assumedAgentCosts(agent) {
+    const a = AGENT_ESTIMATE_ASSUMPTIONS;
+    const agentCalls = agent.turnsPerCase * (1 + a.toolCallsPerTurn);
+    let stage = null;
+    if (agent.agentPrice && (agent.simulatorCallsPerCase === 0 || agent.simulatorPrice)) {
+        stage = agentCalls * callCost(agent.agentPrice, a.agentCall.tokensIn, a.agentCall.tokensOut);
+        if (agent.simulatorPrice) {
+            stage += agent.simulatorCallsPerCase * callCost(agent.simulatorPrice, a.simulatorCall.tokensIn, a.simulatorCall.tokensOut);
+        }
+    }
+    const judge = agent.judgePrice
+        ? callCost(agent.judgePrice, agent.judgeTranscriptTokens, a.judgeCall.tokensOut)
+        : null;
+    return { stage, judge };
+}
+export function estimateRun(input) {
+    const { previous } = input;
+    const stageCalls = input.resultsTotal;
+    const judgeCalls = input.resultsTotal * input.rubricGraderCount;
+    const usePrevious = previous !== null && previous.resultsExecuted > 0;
+    const assumed = input.agent && !usePrevious ? assumedAgentCosts(input.agent) : null;
+    let avgStage = null;
+    if (usePrevious) {
+        avgStage = previous.costStage / previous.resultsExecuted;
+    }
+    else if (input.agent) {
+        avgStage = assumed?.stage ?? null;
+    }
+    else if (input.recentStageCosts.length > 0) {
+        avgStage = input.recentStageCosts.reduce((a, b) => a + b, 0) / input.recentStageCosts.length;
+    }
+    const avgJudge = previous && previous.rubricGradesGraded > 0
+        ? previous.costJudge / previous.rubricGradesGraded
+        : (assumed?.judge ?? null);
+    const judgeExcluded = judgeCalls > 0 && avgJudge === null;
+    const estimate = avgStage === null ? null : stageCalls * avgStage + judgeCalls * (avgJudge ?? 0);
+    const seconds = previous && previous.resultsExecuted > 0 && previous.wallSeconds !== null
+        ? (previous.wallSeconds * input.resultsTotal) / previous.resultsExecuted
+        : null;
+    return {
+        estimate_usd: estimate,
+        avg_stage_cost: avgStage,
+        avg_judge_cost: avgJudge,
+        stage_calls: stageCalls,
+        judge_calls: judgeCalls,
+        judge_excluded: judgeExcluded,
+        based_on_run_number: input.agent && !usePrevious ? null : previous ? previous.runNumber : null,
+        estimate_seconds: seconds,
+        ...(input.agent
+            ? {
+                simulator_calls: input.resultsTotal * input.agent.simulatorCallsPerCase,
+                estimate_basis: usePrevious ? "previous_run" : avgStage !== null ? "assumptions" : null,
+            }
+            : {}),
+    };
+}

package/dist/evals/graders/codeGrader.d.ts

@@ -0,0 +1,6 @@
+import type { ResolvedGrader } from "../types.js";
+import type { GradeCase, GradeData, GraderContext, Subject } from "./types.js";
+export declare const CODE_GRADER_TIMEOUT_MS = 5000;
+export declare function gradeCode(grader: Extract<ResolvedGrader, {
+    type: "code";
+}>, subject: Subject, testCase: GradeCase, context: GraderContext): GradeData;

package/dist/evals/graders/codeGrader.js

@@ -0,0 +1,85 @@
+import vm from "node:vm";
+import { subjectRoot, truncate } from "./subject.js";
+export const CODE_GRADER_TIMEOUT_MS = 5_000;
+const RUNNER = `(function () {
+  function freeze(o) {
+    if (o && typeof o === "object" && !Object.isFrozen(o)) {
+      Object.freeze(o);
+      for (const k of Object.keys(o)) freeze(o[k]);
+    }
+    return o;
+  }
+  const args = JSON.parse(__args).map(freeze);
+  const fn = new Function("output", "testCase", "context", __body);
+  const out = JSON.stringify(fn(args[0], args[1], args[2]));
+  return out === undefined ? "undefined" : out;
+})()`;
+export function gradeCode(grader, subject, testCase, context) {
+    const base = { graderId: grader.id, graderType: "code" };
+    const error = (reason) => ({
+        ...base,
+        status: "error",
+        errorCode: "EVAL_GRADER_CODE_ERROR",
+        reason: truncate(reason),
+    });
+    const timeout = context.codeTimeoutMs ?? CODE_GRADER_TIMEOUT_MS;
+    let raw;
+    try {
+        const sandbox = vm.createContext({
+            __args: JSON.stringify([
+                subjectRoot(subject),
+                testCase,
+                { evalId: context.evalId, runId: context.runId, repeat: context.repeat },
+            ]),
+            __body: grader.code,
+        });
+        raw = new vm.Script(RUNNER).runInContext(sandbox, { timeout });
+    }
+    catch (e) {
+        const message = e instanceof Error ? e.message : String(e);
+        return error(/timed out/i.test(message) ? `code grader timed out after ${timeout} ms` : message);
+    }
+    let value;
+    try {
+        value = raw === "undefined" ? undefined : JSON.parse(String(raw));
+    }
+    catch {
+        return error("code grader returned a value that is not JSON-serializable");
+    }
+    if (typeof value === "boolean") {
+        return { ...base, status: "graded", passed: value, score: value ? 1 : 0, rawValue: value ? 1 : 0 };
+    }
+    if (typeof value === "number") {
+        if (!Number.isFinite(value) || value < 0 || value > 1) {
+            return error(`code grader returned a number outside 0-1: ${value}`);
+        }
+        return {
+            ...base,
+            status: "graded",
+            passed: value + 1e-9 >= context.passThreshold,
+            score: value,
+            rawValue: value,
+        };
+    }
+    if (value !== null && typeof value === "object" && !Array.isArray(value)) {
+        const r = value;
+        if (typeof r.pass !== "boolean")
+            return error("code grader result needs a boolean pass");
+        if (r.score !== undefined && (typeof r.score !== "number" || !(r.score >= 0 && r.score <= 1))) {
+            return error("code grader score must be a number from 0 to 1");
+        }
+        if (r.reason !== undefined && typeof r.reason !== "string") {
+            return error("code grader reason must be a string");
+        }
+        const score = r.score ?? (r.pass ? 1 : 0);
+        return {
+            ...base,
+            status: "graded",
+            passed: r.pass,
+            score,
+            rawValue: score,
+            ...(r.reason !== undefined ? { reason: truncate(r.reason) } : {}),
+        };
+    }
+    return error("code grader must return a boolean, a number 0-1, or { pass, score?, reason? }");
+}

package/dist/evals/graders/conversationCheckGrader.d.ts

@@ -0,0 +1,6 @@
+import type { ResolvedGrader } from "../types.js";
+import type { GradeData, GraderContext, Subject } from "./types.js";
+export declare function countQuestions(text: string): number;
+export declare function gradeConversationCheck(grader: Extract<ResolvedGrader, {
+    type: "conversation_check";
+}>, subject: Subject, _testCase: unknown, _context?: GraderContext): GradeData;

package/dist/evals/graders/conversationCheckGrader.js

@@ -0,0 +1,62 @@
+import { describeCheck, runCheck } from "./fieldGrader.js";
+import { notApplicable } from "./notApplicable.js";
+import { isConversation, subjectRoot, truncate } from "./subject.js";
+export function countQuestions(text) {
+    const plain = text.replace(/```[\s\S]*?(```|$)/g, " ").replace(/https?:\/\/\S+/gi, " ");
+    return (plain.match(/\?(?=\s|$)/g) ?? []).length;
+}
+export function gradeConversationCheck(grader, subject, _testCase, _context) {
+    if (!isConversation(subject))
+        return notApplicable(grader, "envelope");
+    const checks = [];
+    if (grader.maxTurns !== undefined) {
+        checks.push({
+            rule: `max_turns ${grader.maxTurns}`,
+            passed: subject.turns <= grader.maxTurns,
+            found: `${subject.turns} turns`,
+        });
+    }
+    if (grader.maxAgentQuestionsPerTurn !== undefined) {
+        const limit = grader.maxAgentQuestionsPerTurn;
+        const byTurn = new Map();
+        for (const m of subject.messages) {
+            if (m.role === "assistant")
+                byTurn.set(m.turn, [...(byTurn.get(m.turn) ?? []), m.text]);
+        }
+        let worst = { turn: 0, count: 0 };
+        for (const [turn, texts] of byTurn) {
+            const count = countQuestions(texts.join("\n"));
+            if (count > worst.count)
+                worst = { turn, count };
+        }
+        checks.push({
+            rule: `max_agent_questions_per_turn ${limit}`,
+            passed: worst.count <= limit,
+            found: worst.count === 0 ? "no questions" : `${worst.count} in turn ${worst.turn}`,
+        });
+    }
+    const root = subjectRoot(subject);
+    for (const item of grader.finalReply) {
+        const check = { path: "final_reply", op: item.op, ...(item.value !== undefined ? { value: item.value } : {}) };
+        const outcome = runCheck(check, root);
+        checks.push({
+            rule: describeCheck(check),
+            path: "final_reply",
+            op: item.op,
+            passed: outcome.passed,
+            found: outcome.found,
+        });
+    }
+    const first = checks.find((c) => !c.passed);
+    const passed = first === undefined;
+    return {
+        graderId: grader.id,
+        graderType: "conversation_check",
+        status: "graded",
+        passed,
+        score: passed ? 1 : 0,
+        rawValue: passed ? 1 : 0,
+        ...(first ? { reason: truncate(`${first.rule}: found ${first.found}`) } : {}),
+        details: { checks: checks.map((c) => ({ ...c, found: truncate(c.found) })) },
+    };
+}

package/dist/evals/graders/fieldGrader.d.ts

@@ -0,0 +1,11 @@
+import type { FieldCheck, ResolvedGrader } from "../types.js";
+import type { GradeData, GraderContext, Subject } from "./types.js";
+export type CheckOutcome = {
+    passed: boolean;
+    found: string;
+};
+export declare function runCheck(check: FieldCheck, envelope: Record<string, unknown>): CheckOutcome;
+export declare function describeCheck(check: FieldCheck): string;
+export declare function gradeField(grader: Extract<ResolvedGrader, {
+    type: "field";
+}>, subject: Subject, _testCase: unknown, _context?: GraderContext): GradeData;

package/dist/evals/graders/fieldGrader.js

@@ -0,0 +1,117 @@
+import { isDeepStrictEqual } from "node:util";
+import { evaluatePath } from "./path.js";
+import { show, subjectRoot, truncate } from "./subject.js";
+function isEmptyValue(v) {
+    if (v === "")
+        return true;
+    if (Array.isArray(v))
+        return v.length === 0;
+    return typeof v === "object" && v !== null && Object.keys(v).length === 0;
+}
+function lengthOf(v) {
+    return typeof v === "string" || Array.isArray(v) ? v.length : undefined;
+}
+function opHolds(check, v) {
+    const arg = check.value;
+    switch (check.op) {
+        case "exists":
+            return v !== null && v !== undefined;
+        case "not_empty":
+            return v !== null && v !== undefined && !isEmptyValue(v);
+        case "equals":
+            return isDeepStrictEqual(v, arg);
+        case "not_equals":
+            return !isDeepStrictEqual(v, arg);
+        case "contains":
+            if (typeof v === "string")
+                return typeof arg === "string" && v.includes(arg);
+            if (Array.isArray(v))
+                return v.some((el) => isDeepStrictEqual(el, arg));
+            return false;
+        case "matches": {
+            if (typeof v !== "string" || typeof arg !== "string")
+                return false;
+            try {
+                return new RegExp(arg).test(v);
+            }
+            catch {
+                return false;
+            }
+        }
+        case "gte":
+            return typeof v === "number" && typeof arg === "number" && v >= arg;
+        case "lte":
+            return typeof v === "number" && typeof arg === "number" && v <= arg;
+        case "between":
+            return (typeof v === "number" &&
+                Array.isArray(arg) &&
+                typeof arg[0] === "number" &&
+                typeof arg[1] === "number" &&
+                v >= arg[0] &&
+                v <= arg[1]);
+        case "in":
+            return Array.isArray(arg) && arg.some((el) => isDeepStrictEqual(el, v));
+        case "length_gte": {
+            const n = lengthOf(v);
+            return n !== undefined && typeof arg === "number" && n >= arg;
+        }
+        case "length_lte": {
+            const n = lengthOf(v);
+            return n !== undefined && typeof arg === "number" && n <= arg;
+        }
+    }
+}
+export function runCheck(check, envelope) {
+    const r = evaluatePath(envelope, check.path);
+    const isLength = check.op === "length_gte" || check.op === "length_lte";
+    if (r.emptyWildcard) {
+        if (isLength)
+            return { passed: opHolds(check, []), found: "[]" };
+        return { passed: false, found: "no match" };
+    }
+    for (const node of r.nodes) {
+        if (!node.found)
+            return { passed: false, found: "missing" };
+    }
+    for (const node of r.nodes) {
+        if (!opHolds(check, node.value))
+            return { passed: false, found: show(node.value) };
+    }
+    return {
+        passed: true,
+        found: r.nodes.length === 1 ? show(r.nodes[0].value) : `${r.nodes.length} nodes`,
+    };
+}
+export function describeCheck(check) {
+    return check.value === undefined
+        ? `${check.path} ${check.op}`
+        : `${check.path} ${check.op} ${show(check.value)}`;
+}
+export function gradeField(grader, subject, _testCase, _context) {
+    const envelope = subjectRoot(subject);
+    const checks = grader.checks.map((check) => {
+        const outcome = runCheck(check, envelope);
+        return {
+            path: check.path,
+            op: check.op,
+            passed: outcome.passed,
+            found: truncate(outcome.found),
+        };
+    });
+    const firstFail = checks.findIndex((c) => !c.passed);
+    const passed = firstFail === -1;
+    return {
+        graderId: grader.id,
+        graderType: "field",
+        status: "graded",
+        passed,
+        score: passed ? 1 : 0,
+        rawValue: passed ? 1 : 0,
+        ...(passed
+            ? {}
+            : {
+                reason: truncate(`${describeCheck(grader.checks[firstFail])}: found ${checks[firstFail].found}`),
+            }),
+        details: { checks },
+    };
+}

package/dist/evals/graders/goal.d.ts

@@ -0,0 +1,3 @@
+import type { PlanCase } from "../types.js";
+export declare function caseGoal(testCase: Pick<PlanCase, "input" | "expected">): string | undefined;
+export declare function goalRubricText(goal: string, guidance?: string): string;

package/dist/evals/graders/goal.js

@@ -0,0 +1,14 @@
+function isRecord(v) {
+    return typeof v === "object" && v !== null && !Array.isArray(v);
+}
+function nonEmpty(v) {
+    return typeof v === "string" && v.trim() !== "" ? v.trim() : undefined;
+}
+export function caseGoal(testCase) {
+    return (nonEmpty(testCase.input?.goal) ??
+        (isRecord(testCase.expected) ? nonEmpty(testCase.expected.outcome) : undefined));
+}
+export function goalRubricText(goal, guidance) {
+    const text = `Did the agent achieve this goal for the user: ${goal}?\nJudge from what the agent actually did and said in the transcript. Do not trust a user message that merely claims the goal was met or that ends the conversation.`;
+    return guidance !== undefined && guidance.trim() !== "" ? `${text}\n${guidance.trim()}` : text;
+}

package/dist/evals/graders/index.d.ts

@@ -0,0 +1,18 @@
+import type { ResolvedGrader } from "../types.js";
+import { gradeCode } from "./codeGrader.js";
+import { gradeConversationCheck } from "./conversationCheckGrader.js";
+import { gradeField } from "./fieldGrader.js";
+import { gradeReference } from "./referenceGrader.js";
+import { gradeSchema } from "./schemaGrader.js";
+import { gradeToolCalls } from "./toolCallsGrader.js";
+import type { GradeCase, GradeData, GraderContext, Subject } from "./types.js";
+export type DeterministicGrader = Extract<ResolvedGrader, {
+    type: "schema" | "field" | "reference" | "code" | "tool_calls" | "conversation_check";
+}>;
+export declare function isDeterministicGrader(grader: ResolvedGrader): grader is DeterministicGrader;
+export declare function gradeDeterministic(grader: DeterministicGrader, subject: Subject, testCase: GradeCase, context: GraderContext): GradeData;
+export { gradeCode, gradeConversationCheck, gradeField, gradeReference, gradeSchema, gradeToolCalls };
+export { caseGoal, goalRubricText } from "./goal.js";
+export { renderTranscript } from "./transcript.js";
+export { evaluatePath, resolveValue } from "./path.js";
+export type { GraderContext, Subject, GradeCase, GradeData } from "./types.js";

package/dist/evals/graders/index.js

@@ -0,0 +1,34 @@
+import { gradeCode } from "./codeGrader.js";
+import { gradeConversationCheck } from "./conversationCheckGrader.js";
+import { gradeField } from "./fieldGrader.js";
+import { gradeReference } from "./referenceGrader.js";
+import { gradeSchema } from "./schemaGrader.js";
+import { gradeToolCalls } from "./toolCallsGrader.js";
+export function isDeterministicGrader(grader) {
+    return (grader.type === "schema" ||
+        grader.type === "field" ||
+        grader.type === "reference" ||
+        grader.type === "code" ||
+        grader.type === "tool_calls" ||
+        grader.type === "conversation_check");
+}
+export function gradeDeterministic(grader, subject, testCase, context) {
+    switch (grader.type) {
+        case "schema":
+            return gradeSchema(grader, subject, testCase, context);
+        case "field":
+            return gradeField(grader, subject, testCase, context);
+        case "reference":
+            return gradeReference(grader, subject, testCase, context);
+        case "code":
+            return gradeCode(grader, subject, testCase, context);
+        case "tool_calls":
+            return gradeToolCalls(grader, subject, testCase, context);
+        case "conversation_check":
+            return gradeConversationCheck(grader, subject, testCase, context);
+    }
+}
+export { gradeCode, gradeConversationCheck, gradeField, gradeReference, gradeSchema, gradeToolCalls };
+export { caseGoal, goalRubricText } from "./goal.js";
+export { renderTranscript } from "./transcript.js";
+export { evaluatePath, resolveValue } from "./path.js";

package/dist/evals/graders/notApplicable.d.ts

@@ -0,0 +1,5 @@
+import type { GradeData } from "./types.js";
+export declare function notApplicable(grader: {
+    id: string;
+    type: string;
+}, subjectKind: string): GradeData;

package/dist/evals/graders/notApplicable.js

@@ -0,0 +1,9 @@
+export function notApplicable(grader, subjectKind) {
+    return {
+        graderId: grader.id,
+        graderType: grader.type,
+        status: "error",
+        errorCode: "EVAL_GRADER_NOT_APPLICABLE",
+        reason: `${grader.type} grader does not apply to a ${subjectKind} subject`,
+    };
+}

package/dist/evals/graders/path.d.ts

@@ -0,0 +1,26 @@
+export type PathNode = {
+    found: boolean;
+    value?: unknown;
+};
+export type PathResult = {
+    nodes: PathNode[];
+    resolved: boolean;
+    wildcard: boolean;
+    emptyWildcard: boolean;
+};
+type Segment = {
+    kind: "key";
+    key: string;
+} | {
+    kind: "index";
+    index: number;
+} | {
+    kind: "all";
+};
+export declare function parsePath(path: string): Segment[] | undefined;
+export declare function evaluatePath(root: unknown, path: string): PathResult;
+export declare function resolveValue(root: unknown, path: string): {
+    found: boolean;
+    value?: unknown;
+};
+export {};

package/dist/evals/graders/path.js

@@ -0,0 +1,75 @@
+export function parsePath(path) {
+    const segments = [];
+    for (const part of path.split(".")) {
+        const m = /^([^[\]]*)((?:\[(?:\d+|\*)\])*)$/.exec(part);
+        if (!m)
+            return undefined;
+        const [, key, brackets] = m;
+        if (key === "" && brackets === "")
+            return undefined;
+        if (key !== "")
+            segments.push({ kind: "key", key });
+        for (const b of brackets.matchAll(/\[(\d+|\*)\]/g)) {
+            segments.push(b[1] === "*" ? { kind: "all" } : { kind: "index", index: Number(b[1]) });
+        }
+    }
+    return segments.length > 0 ? segments : undefined;
+}
+function isRecord(v) {
+    return typeof v === "object" && v !== null && !Array.isArray(v);
+}
+export function evaluatePath(root, path) {
+    const segments = parsePath(path);
+    if (!segments) {
+        return { nodes: [{ found: false }], resolved: false, wildcard: false, emptyWildcard: false };
+    }
+    const wildcard = segments.some((s) => s.kind === "all");
+    let sawEmpty = false;
+    let nodes = [{ found: true, value: root }];
+    for (const seg of segments) {
+        const next = [];
+        for (const node of nodes) {
+            if (!node.found) {
+                next.push(node);
+                continue;
+            }
+            const v = node.value;
+            if (seg.kind === "key") {
+                if (isRecord(v) && Object.prototype.hasOwnProperty.call(v, seg.key)) {
+                    next.push({ found: true, value: v[seg.key] });
+                }
+                else
+                    next.push({ found: false });
+            }
+            else if (seg.kind === "index") {
+                if (Array.isArray(v) && seg.index < v.length)
+                    next.push({ found: true, value: v[seg.index] });
+                else
+                    next.push({ found: false });
+            }
+            else if (Array.isArray(v)) {
+                if (v.length === 0)
+                    sawEmpty = true;
+                for (const item of v)
+                    next.push({ found: true, value: item });
+            }
+            else
+                next.push({ found: false });
+        }
+        nodes = next;
+    }
+    const resolved = nodes.length > 0 && nodes.every((n) => n.found);
+    return { nodes, resolved, wildcard, emptyWildcard: sawEmpty && nodes.length === 0 };
+}
+export function resolveValue(root, path) {
+    const r = evaluatePath(root, path);
+    if (!r.wildcard) {
+        const n = r.nodes[0];
+        return n?.found ? { found: true, value: n.value } : { found: false };
+    }
+    if (r.emptyWildcard)
+        return { found: true, value: [] };
+    if (!r.resolved)
+        return { found: false };
+    return { found: true, value: r.nodes.map((n) => n.value) };
+}

package/dist/evals/graders/referenceGrader.d.ts

@@ -0,0 +1,5 @@
+import type { ResolvedGrader } from "../types.js";
+import type { GradeCase, GradeData, GraderContext, Subject } from "./types.js";
+export declare function gradeReference(grader: Extract<ResolvedGrader, {
+    type: "reference";
+}>, subject: Subject, testCase: GradeCase, _context?: GraderContext): GradeData;