vitest-evals 0.8.0 → 0.9.0-beta.1

package/README.md

@@ -1,281 +1,227 @@
 # vitest-evals
 
-End-to-end evaluation framework for AI agents, built on Vitest.
+Harness-backed AI testing on top of Vitest.
 
-## Installation
+## Install
 
-```shell
+```sh
 npm install -D vitest-evals
 ```
 
-## Quick Start
+Install a first-party harness package for the runtime you want to test:
 
-```javascript
-import { describeEval } from "vitest-evals";
-
-describeEval("deploy agent", {
-  data: async () => [
-    { input: "Deploy the latest release to production", expected: "deployed" },
-    { input: "Roll back the last deploy", expected: "rolled back" },
-  ],
-  task: async (input) => {
-    const response = await myAgent.run(input);
-    return response;
-  },
-  scorers: [
-    async ({ output, expected }) => ({
-      score: output.toLowerCase().includes(expected.toLowerCase()) ? 1.0 : 0.0,
-    }),
-  ],
-  threshold: 0.8,
-});
+```sh
+npm install -D @vitest-evals/harness-pi-ai
+# or
+npm install -D @vitest-evals/harness-ai-sdk
 ```
 
-## Tasks
-
-Tasks process inputs and return outputs. Two formats are supported:
-
-```javascript
-// Simple: just return a string
-const task = async (input) => "response";
-
-// With tool tracking: return a TaskResult
-const task = async (input) => ({
-  result: "response",
-  toolCalls: [
-    { name: "search", arguments: { query: "..." }, result: {...} }
-  ]
-});
-```
-
-## Test Data
-
-Each test case requires an `input` field. Use `name` to give tests a descriptive label:
-
-```javascript
-data: async () => [
-  { name: "simple deploy", input: "Deploy to staging" },
-  { name: "deploy with rollback", input: "Deploy to prod, roll back if errors" },
-],
-```
-
-Additional fields (like `expected`, `expectedTools`) are passed through to scorers.
-
-## Lifecycle Hooks
-
-Use `beforeEach` and `afterEach` for setup and teardown:
+## Core Model
+
+- `describeEval(...)` binds exactly one harness to a suite
+- the suite callback receives a fixture-backed Vitest `it`
+- `run(input, { metadata? })` executes the harness explicitly and returns a
+  normalized `HarnessRun`
+- the returned `result.output` is the app-facing value you assert on directly
+- the returned `result.session` is the canonical JSON-serializable trace for
+  reporting, replay, tool assertions, and judges
+- per-run judge inputs should usually live under `metadata`
+- suite-level `judges` are optional and run automatically after each `run(...)`
+- suite-level `judgeThreshold` controls fail-on-score for those automatic judges
+- every judge receives `JudgeContext`, including the configured `harness` with
+  its required `prompt` function
+- explicit judge assertions use
+  `await expect(result).toSatisfyJudge(judge, context)`
+
+## Explicit Run Example
+
+```ts
+import { expect } from "vitest";
+import { piAiHarness } from "@vitest-evals/harness-pi-ai";
+import {
+  describeEval,
+  namedJudge,
+  toolCalls,
+  type JudgeContext,
+} from "vitest-evals";
+import { createRefundAgent, judgePrompt } from "../src/refundAgent";
+
+type RefundEvalMetadata = {
+  expectedStatus: "approved" | "denied";
+  expectedTools: string[];
+};
 
-```javascript
-describeEval("agent with database", {
-  beforeEach: async () => {
-    await db.seed();
+const FactualityJudge = namedJudge(
+  "FactualityJudge",
+  async ({
+    input,
+    output,
+    metadata,
+  }: JudgeContext<string, RefundEvalMetadata>) => {
+    const verdict = await judgeFactuality({
+      question: input,
+      answer: output,
+      expectedStatus: metadata.expectedStatus,
+    });
+
+    return {
+      score: verdict.score,
+      metadata: {
+        rationale: verdict.rationale,
+      },
+    };
   },
-  afterEach: async () => {
-    await db.clean();
+);
+
+describeEval(
+  "refund agent",
+  {
+    harness: piAiHarness({
+      createAgent: () => createRefundAgent(),
+      prompt: judgePrompt,
+    }),
+    judges: [FactualityJudge],
   },
-  data: async () => [{ input: "Find recent errors" }],
-  task: myAgentTask,
-  scorers: [async ({ output }) => ({ score: output.includes("error") ? 1.0 : 0.0 })],
-});
+  (it) => {
+    it("approves a refundable invoice", async ({ run }) => {
+      const result = await run("Refund invoice inv_123", {
+        metadata: {
+          expectedStatus: "approved",
+          expectedTools: ["lookupInvoice", "createRefund"],
+        },
+      });
+
+      expect(result.output).toMatchObject({ status: "approved" });
+      expect(toolCalls(result.session).map((call) => call.name)).toEqual([
+        "lookupInvoice",
+        "createRefund",
+      ]);
+    });
+  },
+);
 ```
 
-## Scorers
-
-Scorers evaluate outputs and return a score (0-1). Use built-in scorers or create your own.
+## Table-Driven Vitest Style
 
-### ToolCallScorer
+If you want case tables, use Vitest's own `it.for(...)` and call `run(...)`
+inside the test body:
 
-Evaluates if the expected tools were called with correct arguments.
-
-```javascript
-import { ToolCallScorer } from "vitest-evals";
-
-describeEval("tool usage", {
-  data: async () => [
+```ts
+describeEval("refund agent", { harness }, (it) => {
+  it.for([
     {
-      input: "Find Italian restaurants",
-      expectedTools: [
-        { name: "search", arguments: { type: "restaurant" } },
-        { name: "filter", arguments: { cuisine: "italian" } },
-      ],
+      name: "approves refundable invoice",
+      input: "Refund invoice inv_123",
+      expectedStatus: "approved",
     },
-  ],
-  task: myTask,
-  scorers: [ToolCallScorer()],
-});
-
-// Strict order and parameters
-scorers: [ToolCallScorer({ ordered: true, params: "strict" })];
-
-// Flexible evaluation
-scorers: [ToolCallScorer({ requireAll: false, allowExtras: false })];
-```
-
-**Default behavior:**
-
-- Strict parameter matching (exact equality required)
-- Any order allowed
-- Extra tools allowed
-- All expected tools required
-
-### StructuredOutputScorer
-
-Evaluates if the output matches expected structured data (JSON).
-
-```javascript
-import { StructuredOutputScorer } from "vitest-evals";
-
-describeEval("query generation", {
-  data: async () => [
     {
-      input: "Show me errors from today",
-      expected: {
-        dataset: "errors",
-        query: "",
-        sort: "-timestamp",
-        timeRange: { statsPeriod: "24h" },
-      },
-    },
-  ],
-  task: myTask,
-  scorers: [StructuredOutputScorer()],
-});
-
-// Fuzzy matching
-scorers: [StructuredOutputScorer({ match: "fuzzy" })];
-
-// Custom validation
-scorers: [
-  StructuredOutputScorer({
-    match: (expected, actual, key) => {
-      if (key === "age") return actual >= 18 && actual <= 100;
-      return expected === actual;
+      name: "denies non-refundable invoice",
+      input: "Refund invoice inv_404",
+      expectedStatus: "denied",
     },
-  }),
-];
-```
-
-### Custom Scorers
-
-```javascript
-// Inline scorer
-const LengthScorer = async ({ output }) => ({
-  score: output.length > 50 ? 1.0 : 0.0,
-});
-
-// TypeScript scorer with custom options
-import { type ScoreFn, type BaseScorerOptions } from "vitest-evals";
-
-interface CustomOptions extends BaseScorerOptions {
-  minLength: number;
-}
-
-const TypedScorer: ScoreFn<CustomOptions> = async (opts) => ({
-  score: opts.output.length >= opts.minLength ? 1.0 : 0.0,
+  ])("$name", async ({ input, ...metadata }, { run }) => {
+    const result = await run(input, {
+      metadata,
+    });
+
+    expect(result.output).toMatchObject({
+      status: metadata.expectedStatus,
+    });
+  });
 });
 ```
 
-## AI SDK Integration
-
-See [`src/ai-sdk-integration.test.ts`](src/ai-sdk-integration.test.ts) for a complete example with the Vercel AI SDK.
-
-Transform provider responses to our format:
-
-```javascript
-const { text, steps } = await generateText({
-  model: openai("gpt-4o"),
-  prompt: input,
-  tools: { myTool: myToolDefinition },
-});
-
-return {
-  result: text,
-  toolCalls: steps
-    .flatMap((step) => step.toolCalls)
-    .map((call) => ({
-      name: call.toolName,
-      arguments: call.args,
-    })),
-};
-```
+## Existing Agents
 
-## Advanced Usage
+For an existing agent, the intended contract is:
 
-### Using autoevals
+- pass the agent instance or per-test factory through the harness
+- optionally pass `run` when the app entrypoint is not `run(input, runtime)`
+- let the harness infer native tools from the existing agent by default
+- only pass an explicit `tools` override when the agent hides its tool surface
 
-For evaluation using the autoevals library:
+The harness owns normalization, diagnostics, tool capture, replay plumbing, and
+reporter-facing artifacts. Your app just needs one runtime seam where those
+wrapped pieces can be injected.
 
-```javascript
-import { Factuality, ClosedQA } from "autoevals";
+For the Pi-specific harness, output/session/usage normalization should usually
+be inferred automatically. Treat low-level normalization callbacks as an escape
+hatch, not part of the primary authoring path.
 
-scorers: [
-  Factuality,
-  ClosedQA.partial({
-    criteria: "Does the answer mention Paris?",
-  }),
-];
-```
+## Judge Matchers
 
-### Skip Tests Conditionally
+Use the matcher when a judge should behave like a normal Vitest assertion.
+In practice, this is usually most useful for factuality, rubric, or grounded
+answer checks:
 
-```javascript
-describeEval("gpt-4 tests", {
-  skipIf: () => !process.env.OPENAI_API_KEY,
-  // ...
-});
+```ts
+await expect(result).toSatisfyJudge(FactualityJudge);
 ```
 
-### Existing Test Suites
+For lower-level cases, the matcher also accepts raw values and synthetic judge
+context:
 
-For integration with existing Vitest test suites, you can use the `.toEval()` matcher:
-
-> **Deprecated**: The `.toEval()` helper is deprecated. Use `describeEval()` instead for better test organization and multiple scorers support.
-
-```javascript
-import "vitest-evals";
-
-test("capital check", () => {
-  const simpleFactuality = async ({ output, expected }) => ({
-    score: output.toLowerCase().includes(expected.toLowerCase()) ? 1.0 : 0.0,
-  });
-
-  expect("What is the capital of France?").toEval(
-    "Paris",
-    answerQuestion,
-    simpleFactuality,
-    0.8
-  );
+```ts
+await expect({ status: "approved" }).toSatisfyJudge(MyJudge, {
+  inputValue: "Refund invoice inv_123",
 });
 ```
 
-## Configuration
-
-### Separate Eval Configuration
+If you are writing a custom judge, wrap it with `namedJudge(...)` so reporter
+output uses a stable label:
 
-Create `vitest.evals.config.ts`:
+```ts
+import { namedJudge } from "vitest-evals";
 
-```javascript
-import { defineConfig } from "vitest/config";
-import defaultConfig from "./vitest.config";
+const FactualityJudge = namedJudge(
+  "FactualityJudge",
+  async ({ output }) => {
+    const answer = output;
+    const verdict = await judgeFactuality(answer);
 
-export default defineConfig({
-  ...defaultConfig,
-  test: {
-    ...defaultConfig.test,
-    include: ["src/**/*.eval.{js,ts}"],
+    return {
+      score: verdict.score,
+      metadata: {
+        rationale: verdict.rationale,
+      },
+    };
   },
-});
+);
 ```
 
-Run evals separately:
-
-```shell
-vitest --config=vitest.evals.config.ts
-```
-
-## Development
-
-```shell
-pnpm install
-pnpm test
+LLM-backed judges can reuse the suite harness prompt by calling
+`harness.prompt(...)`. `vitest-evals` does not prescribe a rubric schema,
+scoring scale, model provider, or parser; those stay in the judge. Calling
+`harness.run(...)` from a judge executes the application again, so use that
+only when a second run is intentional.
+
+For an `EvalHarnessRun` returned by fixture `run(...)`,
+`toSatisfyJudge(...)` uses the run's canonical text output and reuses the
+registered input, metadata, and harness prompt. Inside an eval test,
+matcher calls on registered raw output or session objects reuse that exact run
+context; raw output values are serialized as the judge `output`, so
+`expect(result.output).toSatisfyJudge(judge)` stays concise. Other raw values
+fall back to the current test's most recent `run(...)` context. For
+manually-created runs or values outside an eval context, pass any required
+`inputValue`, `metadata`, or `harness` in matcher options. Structured or
+programmatic result checks should usually assert on `result.output` directly.
+When a judge needs richer normalized context or the configured suite harness,
+type it with `JudgeContext`.
+
+When you only need deterministic contract checks, built-ins such as
+`StructuredOutputJudge()` and `ToolCallJudge()` are still available. The primary
+documentation examples intentionally use factuality/rubric judges because those
+match the product's LLM-as-a-judge direction.
+
+## Legacy Compatibility
+
+The root package is harness-first and judge-first. Legacy scorer-first suites
+and `evaluate(...)` live under `vitest-evals/legacy`.
+
+```ts
+import {
+  describeEval,
+  StructuredOutputScorer,
+  ToolCallScorer,
+} from "vitest-evals/legacy";
 ```

package/dist/harness.d.mts

@@ -0,0 +1,118 @@
+type JsonPrimitive = string | number | boolean | null;
+type JsonValue = JsonPrimitive | JsonValue[] | {
+    [key: string]: JsonValue;
+};
+type ToolCallRecord = {
+    id?: string;
+    name: string;
+    arguments?: Record<string, JsonValue>;
+    result?: JsonValue;
+    error?: {
+        message: string;
+        type?: string;
+        [key: string]: JsonValue | undefined;
+    };
+    startedAt?: string;
+    finishedAt?: string;
+    durationMs?: number;
+    metadata?: Record<string, JsonValue>;
+};
+type NormalizedMessage = {
+    role: "system" | "user" | "assistant" | "tool";
+    content?: JsonValue;
+    toolCalls?: ToolCallRecord[];
+    metadata?: Record<string, JsonValue>;
+};
+type UsageSummary = {
+    provider?: string;
+    model?: string;
+    inputTokens?: number;
+    outputTokens?: number;
+    reasoningTokens?: number;
+    totalTokens?: number;
+    estimatedCost?: number;
+    toolCalls?: number;
+    retries?: number;
+    metadata?: Record<string, JsonValue>;
+};
+type TimingSummary = {
+    totalMs?: number;
+    metadata?: Record<string, JsonValue>;
+};
+type NormalizedSession = {
+    messages: NormalizedMessage[];
+    outputText?: string;
+    provider?: string;
+    model?: string;
+    metadata?: Record<string, JsonValue>;
+};
+type HarnessRun = {
+    session: NormalizedSession;
+    output?: JsonValue;
+    usage: UsageSummary;
+    timings?: TimingSummary;
+    artifacts?: Record<string, JsonValue>;
+    errors: Array<Record<string, JsonValue>>;
+};
+/** Optional provider-facing hints for harness prompt calls. */
+type HarnessPromptOptions = {
+    system?: string;
+    metadata?: Record<string, JsonValue>;
+};
+/** Provider-agnostic prompt seam that judges can reuse from a harness. */
+type HarnessPrompt = (input: string, options?: HarnessPromptOptions) => Promise<string>;
+type HarnessRunError = Error & {
+    vitestEvalsRun: HarnessRun;
+};
+type HarnessMetadata = Record<string, unknown>;
+type HarnessContext<TMetadata extends HarnessMetadata = HarnessMetadata> = {
+    metadata: Readonly<TMetadata>;
+    task: {
+        meta: Record<string, unknown>;
+    };
+    signal?: AbortSignal;
+    artifacts: Record<string, JsonValue>;
+    setArtifact: (name: string, value: JsonValue) => void;
+};
+type Harness<TInput = unknown, TMetadata extends HarnessMetadata = HarnessMetadata> = {
+    name: string;
+    /** Prompt seam reused by LLM-backed judges. */
+    prompt: HarnessPrompt;
+    run: (input: TInput, context: HarnessContext<TMetadata>) => Promise<HarnessRun>;
+};
+/** Returns true when a value exposes a callable method with the given name. */
+declare function hasCallableMethod(value: unknown, methodName: string): boolean;
+/** Normalizes an unknown value into the JSON-safe shape used by harness runs. */
+declare function toJsonValue(value: unknown): JsonValue | undefined;
+/** Drops non-JSON properties from a record while preserving valid values. */
+declare function normalizeRecord(value: Record<string, unknown>): Record<string, JsonValue>;
+/** Normalizes metadata and omits the field entirely when nothing survives. */
+declare function normalizeMetadata(value: Record<string, unknown>): Record<string, JsonValue> | undefined;
+/** Converts arbitrary content into the JSON-safe message content shape. */
+declare function normalizeContent(value: unknown): JsonValue;
+/** Flattens every recorded tool call from a normalized session. */
+declare function toolCalls(session: NormalizedSession): ToolCallRecord[];
+/** Filters normalized session messages by role. */
+declare function messagesByRole(session: NormalizedSession, role: NormalizedMessage["role"]): NormalizedMessage[];
+/** Returns every normalized system message from a session. */
+declare function systemMessages(session: NormalizedSession): NormalizedMessage[];
+/** Returns every normalized user message from a session. */
+declare function userMessages(session: NormalizedSession): NormalizedMessage[];
+/** Returns every normalized assistant message from a session. */
+declare function assistantMessages(session: NormalizedSession): NormalizedMessage[];
+/** Returns every normalized tool message from a session. */
+declare function toolMessages(session: NormalizedSession): NormalizedMessage[];
+/** Attaches a partial or complete harness run to an arbitrary thrown error. */
+declare function attachHarnessRunToError(error: unknown, run: HarnessRun): HarnessRunError;
+/** Reads an attached harness run back off a previously wrapped error value. */
+declare function getHarnessRunFromError(error: unknown): HarnessRun | undefined;
+/** Returns true when a value matches the normalized `HarnessRun` contract. */
+declare function isHarnessRun(value: unknown): value is HarnessRun;
+/** Returns true when a value matches the normalized session contract. */
+declare function isNormalizedSession(value: unknown): value is NormalizedSession;
+/** Reuses pre-normalized harness errors when a runtime already returns them. */
+declare function resolveHarnessRunErrors(result: unknown): Array<Record<string, JsonValue>>;
+/** Serializes an arbitrary thrown value into the normalized error shape. */
+declare function serializeError(error: unknown): Record<string, JsonValue>;
+
+export { type Harness, type HarnessContext, type HarnessMetadata, type HarnessPrompt, type HarnessPromptOptions, type HarnessRun, type HarnessRunError, type JsonPrimitive, type JsonValue, type NormalizedMessage, type NormalizedSession, type TimingSummary, type ToolCallRecord, type UsageSummary, assistantMessages, attachHarnessRunToError, getHarnessRunFromError, hasCallableMethod, isHarnessRun, isNormalizedSession, messagesByRole, normalizeContent, normalizeMetadata, normalizeRecord, resolveHarnessRunErrors, serializeError, systemMessages, toJsonValue, toolCalls, toolMessages, userMessages };