vitest-evals 0.8.0 → 0.9.0-beta.0

package/README.md

@@ -1,281 +1,212 @@
 # vitest-evals
 
-End-to-end evaluation framework for AI agents, built on Vitest.
+Harness-backed AI testing on top of Vitest.
 
-## Installation
+## Install
 
-```shell
+```sh
 npm install -D vitest-evals
 ```
 
-## Quick Start
+Install a first-party harness package for the runtime you want to test:
 
-```javascript
-import { describeEval } from "vitest-evals";
-
-describeEval("deploy agent", {
-  data: async () => [
-    { input: "Deploy the latest release to production", expected: "deployed" },
-    { input: "Roll back the last deploy", expected: "rolled back" },
-  ],
-  task: async (input) => {
-    const response = await myAgent.run(input);
-    return response;
-  },
-  scorers: [
-    async ({ output, expected }) => ({
-      score: output.toLowerCase().includes(expected.toLowerCase()) ? 1.0 : 0.0,
-    }),
-  ],
-  threshold: 0.8,
-});
+```sh
+npm install -D @vitest-evals/harness-pi-ai
+# or
+npm install -D @vitest-evals/harness-ai-sdk
 ```
 
-## Tasks
-
-Tasks process inputs and return outputs. Two formats are supported:
-
-```javascript
-// Simple: just return a string
-const task = async (input) => "response";
-
-// With tool tracking: return a TaskResult
-const task = async (input) => ({
-  result: "response",
-  toolCalls: [
-    { name: "search", arguments: { query: "..." }, result: {...} }
-  ]
-});
-```
-
-## Test Data
-
-Each test case requires an `input` field. Use `name` to give tests a descriptive label:
-
-```javascript
-data: async () => [
-  { name: "simple deploy", input: "Deploy to staging" },
-  { name: "deploy with rollback", input: "Deploy to prod, roll back if errors" },
-],
-```
-
-Additional fields (like `expected`, `expectedTools`) are passed through to scorers.
-
-## Lifecycle Hooks
+## Core Model
+
+- `describeEval(...)` binds exactly one harness to a suite
+- the suite callback receives a fixture-backed Vitest `it`
+- `run(input, { metadata? })` executes the harness explicitly and returns a
+  normalized `HarnessRun`
+- the returned `result.output` is the app-facing value you assert on directly
+- the returned `result.session` is the canonical JSON-serializable trace for
+  reporting, replay, tool assertions, and judges
+- per-run judge inputs should usually live under `metadata`
+- suite-level `judges` are optional and run automatically after each `run(...)`
+- suite-level `judgeThreshold` controls fail-on-score for those automatic judges
+- explicit judge assertions use
+  `await expect(result).toSatisfyJudge(judge, context)`
+
+## Explicit Run Example
+
+```ts
+import { expect } from "vitest";
+import { piAiHarness } from "@vitest-evals/harness-pi-ai";
+import {
+  describeEval,
+  namedJudge,
+  toolCalls,
+  type JudgeContext,
+} from "vitest-evals";
+import { createRefundAgent } from "../src/refundAgent";
+
+type RefundEvalMetadata = {
+  expectedStatus: "approved" | "denied";
+  expectedTools: string[];
+};
 
-Use `beforeEach` and `afterEach` for setup and teardown:
+const FactualityJudge = namedJudge(
+  "FactualityJudge",
+  async ({
+    input,
+    output,
+    metadata,
+  }: JudgeContext<string, RefundEvalMetadata>) => {
+    const verdict = await judgeFactuality({
+      question: input,
+      answer: output,
+      expectedStatus: metadata.expectedStatus,
+    });
+
+    return {
+      score: verdict.score,
+      metadata: {
+        rationale: verdict.rationale,
+      },
+    };
+  },
+);
 
-```javascript
-describeEval("agent with database", {
-  beforeEach: async () => {
-    await db.seed();
+describeEval(
+  "refund agent",
+  {
+    harness: piAiHarness({
+      createAgent: () => createRefundAgent(),
+    }),
+    judges: [FactualityJudge],
   },
-  afterEach: async () => {
-    await db.clean();
+  (it) => {
+    it("approves a refundable invoice", async ({ run }) => {
+      const result = await run("Refund invoice inv_123", {
+        metadata: {
+          expectedStatus: "approved",
+          expectedTools: ["lookupInvoice", "createRefund"],
+        },
+      });
+
+      expect(result.output).toMatchObject({ status: "approved" });
+      expect(toolCalls(result.session).map((call) => call.name)).toEqual([
+        "lookupInvoice",
+        "createRefund",
+      ]);
+    });
   },
-  data: async () => [{ input: "Find recent errors" }],
-  task: myAgentTask,
-  scorers: [async ({ output }) => ({ score: output.includes("error") ? 1.0 : 0.0 })],
-});
+);
 ```
 
-## Scorers
-
-Scorers evaluate outputs and return a score (0-1). Use built-in scorers or create your own.
-
-### ToolCallScorer
+## Table-Driven Vitest Style
 
-Evaluates if the expected tools were called with correct arguments.
+If you want case tables, use Vitest's own `it.for(...)` and call `run(...)`
+inside the test body:
 
-```javascript
-import { ToolCallScorer } from "vitest-evals";
-
-describeEval("tool usage", {
-  data: async () => [
+```ts
+describeEval("refund agent", { harness }, (it) => {
+  it.for([
     {
-      input: "Find Italian restaurants",
-      expectedTools: [
-        { name: "search", arguments: { type: "restaurant" } },
-        { name: "filter", arguments: { cuisine: "italian" } },
-      ],
+      name: "approves refundable invoice",
+      input: "Refund invoice inv_123",
+      expectedStatus: "approved",
     },
-  ],
-  task: myTask,
-  scorers: [ToolCallScorer()],
-});
-
-// Strict order and parameters
-scorers: [ToolCallScorer({ ordered: true, params: "strict" })];
-
-// Flexible evaluation
-scorers: [ToolCallScorer({ requireAll: false, allowExtras: false })];
-```
-
-**Default behavior:**
-
-- Strict parameter matching (exact equality required)
-- Any order allowed
-- Extra tools allowed
-- All expected tools required
-
-### StructuredOutputScorer
-
-Evaluates if the output matches expected structured data (JSON).
-
-```javascript
-import { StructuredOutputScorer } from "vitest-evals";
-
-describeEval("query generation", {
-  data: async () => [
     {
-      input: "Show me errors from today",
-      expected: {
-        dataset: "errors",
-        query: "",
-        sort: "-timestamp",
-        timeRange: { statsPeriod: "24h" },
-      },
+      name: "denies non-refundable invoice",
+      input: "Refund invoice inv_404",
+      expectedStatus: "denied",
     },
-  ],
-  task: myTask,
-  scorers: [StructuredOutputScorer()],
-});
-
-// Fuzzy matching
-scorers: [StructuredOutputScorer({ match: "fuzzy" })];
-
-// Custom validation
-scorers: [
-  StructuredOutputScorer({
-    match: (expected, actual, key) => {
-      if (key === "age") return actual >= 18 && actual <= 100;
-      return expected === actual;
-    },
-  }),
-];
-```
-
-### Custom Scorers
-
-```javascript
-// Inline scorer
-const LengthScorer = async ({ output }) => ({
-  score: output.length > 50 ? 1.0 : 0.0,
-});
-
-// TypeScript scorer with custom options
-import { type ScoreFn, type BaseScorerOptions } from "vitest-evals";
-
-interface CustomOptions extends BaseScorerOptions {
-  minLength: number;
-}
-
-const TypedScorer: ScoreFn<CustomOptions> = async (opts) => ({
-  score: opts.output.length >= opts.minLength ? 1.0 : 0.0,
+  ])("$name", async ({ input, ...metadata }, { run }) => {
+    const result = await run(input, {
+      metadata,
+    });
+
+    expect(result.output).toMatchObject({
+      status: metadata.expectedStatus,
+    });
+  });
 });
 ```
 
-## AI SDK Integration
-
-See [`src/ai-sdk-integration.test.ts`](src/ai-sdk-integration.test.ts) for a complete example with the Vercel AI SDK.
+## Existing Agents
 
-Transform provider responses to our format:
+For an existing agent, the intended contract is:
 
-```javascript
-const { text, steps } = await generateText({
-  model: openai("gpt-4o"),
-  prompt: input,
-  tools: { myTool: myToolDefinition },
-});
-
-return {
-  result: text,
-  toolCalls: steps
-    .flatMap((step) => step.toolCalls)
-    .map((call) => ({
-      name: call.toolName,
-      arguments: call.args,
-    })),
-};
-```
+- pass the agent instance or per-test factory through the harness
+- optionally pass `run` when the app entrypoint is not `run(input, runtime)`
+- let the harness infer native tools from the existing agent by default
+- only pass an explicit `tools` override when the agent hides its tool surface
 
-## Advanced Usage
+The harness owns normalization, diagnostics, tool capture, replay plumbing, and
+reporter-facing artifacts. Your app just needs one runtime seam where those
+wrapped pieces can be injected.
 
-### Using autoevals
+For the Pi-specific harness, output/session/usage normalization should usually
+be inferred automatically. Treat low-level normalization callbacks as an escape
+hatch, not part of the primary authoring path.
 
-For evaluation using the autoevals library:
+## Judge Matchers
 
-```javascript
-import { Factuality, ClosedQA } from "autoevals";
+Use the matcher when a judge should behave like a normal Vitest assertion.
+In practice, this is usually most useful for factuality, rubric, or grounded
+answer checks:
 
-scorers: [
-  Factuality,
-  ClosedQA.partial({
-    criteria: "Does the answer mention Paris?",
-  }),
-];
+```ts
+await expect(result).toSatisfyJudge(FactualityJudge);
 ```
 
-### Skip Tests Conditionally
+For lower-level cases, the matcher also accepts raw values and synthetic judge
+context:
 
-```javascript
-describeEval("gpt-4 tests", {
-  skipIf: () => !process.env.OPENAI_API_KEY,
-  // ...
+```ts
+await expect({ status: "approved" }).toSatisfyJudge(MyJudge, {
+  inputValue: "Refund invoice inv_123",
 });
 ```
 
-### Existing Test Suites
+If you are writing a custom judge, wrap it with `namedJudge(...)` so reporter
+output uses a stable label:
 
-For integration with existing Vitest test suites, you can use the `.toEval()` matcher:
+```ts
+import { namedJudge } from "vitest-evals";
 
-> **Deprecated**: The `.toEval()` helper is deprecated. Use `describeEval()` instead for better test organization and multiple scorers support.
+const FactualityJudge = namedJudge(
+  "FactualityJudge",
+  async ({ output }) => {
+    const answer = output;
+    const verdict = await judgeFactuality(answer);
 
-```javascript
-import "vitest-evals";
-
-test("capital check", () => {
-  const simpleFactuality = async ({ output, expected }) => ({
-    score: output.toLowerCase().includes(expected.toLowerCase()) ? 1.0 : 0.0,
-  });
-
-  expect("What is the capital of France?").toEval(
-    "Paris",
-    answerQuestion,
-    simpleFactuality,
-    0.8
-  );
-});
-```
-
-## Configuration
-
-### Separate Eval Configuration
-
-Create `vitest.evals.config.ts`:
-
-```javascript
-import { defineConfig } from "vitest/config";
-import defaultConfig from "./vitest.config";
-
-export default defineConfig({
-  ...defaultConfig,
-  test: {
-    ...defaultConfig.test,
-    include: ["src/**/*.eval.{js,ts}"],
+    return {
+      score: verdict.score,
+      metadata: {
+        rationale: verdict.rationale,
+      },
+    };
   },
-});
+);
 ```
 
-Run evals separately:
-
-```shell
-vitest --config=vitest.evals.config.ts
-```
-
-## Development
-
-```shell
-pnpm install
-pnpm test
+For a `HarnessRun`, `toSatisfyJudge(...)` passes `result.output` as `output`.
+For raw values or normalized sessions, the matcher infers the best available
+output from the received value. Structured or programmatic result checks should
+usually assert on `result.output` directly. When a judge needs richer context,
+type it with `JudgeContext` and read `inputValue`, `metadata`, `toolCalls`, or
+`session` from there.
+
+When you only need deterministic contract checks, built-ins such as
+`StructuredOutputJudge()` and `ToolCallJudge()` are still available. The primary
+documentation examples intentionally use factuality/rubric judges because those
+match the product's LLM-as-a-judge direction.
+
+## Legacy Compatibility
+
+The root package is harness-first and judge-first. Legacy scorer-first suites
+and `evaluate(...)` live under `vitest-evals/legacy`.
+
+```ts
+import {
+  describeEval,
+  StructuredOutputScorer,
+  ToolCallScorer,
+} from "vitest-evals/legacy";
 ```

package/dist/harness.d.mts

@@ -0,0 +1,118 @@
+type JsonPrimitive = string | number | boolean | null;
+type JsonValue = JsonPrimitive | JsonValue[] | {
+    [key: string]: JsonValue;
+};
+type ToolCallRecord = {
+    id?: string;
+    name: string;
+    arguments?: Record<string, JsonValue>;
+    result?: JsonValue;
+    error?: {
+        message: string;
+        type?: string;
+        [key: string]: JsonValue | undefined;
+    };
+    startedAt?: string;
+    finishedAt?: string;
+    durationMs?: number;
+    metadata?: Record<string, JsonValue>;
+};
+type NormalizedMessage = {
+    role: "system" | "user" | "assistant" | "tool";
+    content?: JsonValue;
+    toolCalls?: ToolCallRecord[];
+    metadata?: Record<string, JsonValue>;
+};
+type UsageSummary = {
+    provider?: string;
+    model?: string;
+    inputTokens?: number;
+    outputTokens?: number;
+    reasoningTokens?: number;
+    totalTokens?: number;
+    estimatedCost?: number;
+    toolCalls?: number;
+    retries?: number;
+    metadata?: Record<string, JsonValue>;
+};
+type TimingSummary = {
+    totalMs?: number;
+    metadata?: Record<string, JsonValue>;
+};
+type NormalizedSession = {
+    messages: NormalizedMessage[];
+    outputText?: string;
+    provider?: string;
+    model?: string;
+    metadata?: Record<string, JsonValue>;
+};
+type HarnessRun = {
+    session: NormalizedSession;
+    output?: JsonValue;
+    usage: UsageSummary;
+    timings?: TimingSummary;
+    artifacts?: Record<string, JsonValue>;
+    errors: Array<Record<string, JsonValue>>;
+};
+type HarnessPromptOptions = {
+    system?: string;
+    metadata?: Record<string, JsonValue>;
+};
+type HarnessPrompt = (input: string, options?: HarnessPromptOptions) => Promise<string>;
+type HarnessRuntime = {
+    prompt: HarnessPrompt;
+};
+type HarnessRunError = Error & {
+    vitestEvalsRun: HarnessRun;
+};
+type HarnessMetadata = Record<string, unknown>;
+type HarnessContext<TMetadata extends HarnessMetadata = HarnessMetadata> = {
+    metadata: Readonly<TMetadata>;
+    task: {
+        meta: Record<string, unknown>;
+    };
+    signal?: AbortSignal;
+    artifacts: Record<string, JsonValue>;
+    setArtifact: (name: string, value: JsonValue) => void;
+};
+type Harness<TInput = unknown, TMetadata extends HarnessMetadata = HarnessMetadata> = {
+    name: string;
+    prompt?: HarnessPrompt;
+    run: (input: TInput, context: HarnessContext<TMetadata>) => Promise<HarnessRun>;
+};
+/** Returns true when a value exposes a callable method with the given name. */
+declare function hasCallableMethod(value: unknown, methodName: string): boolean;
+/** Normalizes an unknown value into the JSON-safe shape used by harness runs. */
+declare function toJsonValue(value: unknown): JsonValue | undefined;
+/** Drops non-JSON properties from a record while preserving valid values. */
+declare function normalizeRecord(value: Record<string, unknown>): Record<string, JsonValue>;
+/** Normalizes metadata and omits the field entirely when nothing survives. */
+declare function normalizeMetadata(value: Record<string, unknown>): Record<string, JsonValue> | undefined;
+/** Converts arbitrary content into the JSON-safe message content shape. */
+declare function normalizeContent(value: unknown): JsonValue;
+/** Flattens every recorded tool call from a normalized session. */
+declare function toolCalls(session: NormalizedSession): ToolCallRecord[];
+/** Filters normalized session messages by role. */
+declare function messagesByRole(session: NormalizedSession, role: NormalizedMessage["role"]): NormalizedMessage[];
+/** Returns every normalized system message from a session. */
+declare function systemMessages(session: NormalizedSession): NormalizedMessage[];
+/** Returns every normalized user message from a session. */
+declare function userMessages(session: NormalizedSession): NormalizedMessage[];
+/** Returns every normalized assistant message from a session. */
+declare function assistantMessages(session: NormalizedSession): NormalizedMessage[];
+/** Returns every normalized tool message from a session. */
+declare function toolMessages(session: NormalizedSession): NormalizedMessage[];
+/** Attaches a partial or complete harness run to an arbitrary thrown error. */
+declare function attachHarnessRunToError(error: unknown, run: HarnessRun): HarnessRunError;
+/** Reads an attached harness run back off a previously wrapped error value. */
+declare function getHarnessRunFromError(error: unknown): HarnessRun | undefined;
+/** Returns true when a value matches the normalized `HarnessRun` contract. */
+declare function isHarnessRun(value: unknown): value is HarnessRun;
+/** Returns true when a value matches the normalized session contract. */
+declare function isNormalizedSession(value: unknown): value is NormalizedSession;
+/** Reuses pre-normalized harness errors when a runtime already returns them. */
+declare function resolveHarnessRunErrors(result: unknown): Array<Record<string, JsonValue>>;
+/** Serializes an arbitrary thrown value into the normalized error shape. */
+declare function serializeError(error: unknown): Record<string, JsonValue>;
+
+export { type Harness, type HarnessContext, type HarnessMetadata, type HarnessPrompt, type HarnessPromptOptions, type HarnessRun, type HarnessRunError, type HarnessRuntime, type JsonPrimitive, type JsonValue, type NormalizedMessage, type NormalizedSession, type TimingSummary, type ToolCallRecord, type UsageSummary, assistantMessages, attachHarnessRunToError, getHarnessRunFromError, hasCallableMethod, isHarnessRun, isNormalizedSession, messagesByRole, normalizeContent, normalizeMetadata, normalizeRecord, resolveHarnessRunErrors, serializeError, systemMessages, toJsonValue, toolCalls, toolMessages, userMessages };

package/dist/harness.d.ts

@@ -0,0 +1,118 @@
+type JsonPrimitive = string | number | boolean | null;
+type JsonValue = JsonPrimitive | JsonValue[] | {
+    [key: string]: JsonValue;
+};
+type ToolCallRecord = {
+    id?: string;
+    name: string;
+    arguments?: Record<string, JsonValue>;
+    result?: JsonValue;
+    error?: {
+        message: string;
+        type?: string;
+        [key: string]: JsonValue | undefined;
+    };
+    startedAt?: string;
+    finishedAt?: string;
+    durationMs?: number;
+    metadata?: Record<string, JsonValue>;
+};
+type NormalizedMessage = {
+    role: "system" | "user" | "assistant" | "tool";
+    content?: JsonValue;
+    toolCalls?: ToolCallRecord[];
+    metadata?: Record<string, JsonValue>;
+};
+type UsageSummary = {
+    provider?: string;
+    model?: string;
+    inputTokens?: number;
+    outputTokens?: number;
+    reasoningTokens?: number;
+    totalTokens?: number;
+    estimatedCost?: number;
+    toolCalls?: number;
+    retries?: number;
+    metadata?: Record<string, JsonValue>;
+};
+type TimingSummary = {
+    totalMs?: number;
+    metadata?: Record<string, JsonValue>;
+};
+type NormalizedSession = {
+    messages: NormalizedMessage[];
+    outputText?: string;
+    provider?: string;
+    model?: string;
+    metadata?: Record<string, JsonValue>;
+};
+type HarnessRun = {
+    session: NormalizedSession;
+    output?: JsonValue;
+    usage: UsageSummary;
+    timings?: TimingSummary;
+    artifacts?: Record<string, JsonValue>;
+    errors: Array<Record<string, JsonValue>>;
+};
+type HarnessPromptOptions = {
+    system?: string;
+    metadata?: Record<string, JsonValue>;
+};
+type HarnessPrompt = (input: string, options?: HarnessPromptOptions) => Promise<string>;
+type HarnessRuntime = {
+    prompt: HarnessPrompt;
+};
+type HarnessRunError = Error & {
+    vitestEvalsRun: HarnessRun;
+};
+type HarnessMetadata = Record<string, unknown>;
+type HarnessContext<TMetadata extends HarnessMetadata = HarnessMetadata> = {
+    metadata: Readonly<TMetadata>;
+    task: {
+        meta: Record<string, unknown>;
+    };
+    signal?: AbortSignal;
+    artifacts: Record<string, JsonValue>;
+    setArtifact: (name: string, value: JsonValue) => void;
+};
+type Harness<TInput = unknown, TMetadata extends HarnessMetadata = HarnessMetadata> = {
+    name: string;
+    prompt?: HarnessPrompt;
+    run: (input: TInput, context: HarnessContext<TMetadata>) => Promise<HarnessRun>;
+};
+/** Returns true when a value exposes a callable method with the given name. */
+declare function hasCallableMethod(value: unknown, methodName: string): boolean;
+/** Normalizes an unknown value into the JSON-safe shape used by harness runs. */
+declare function toJsonValue(value: unknown): JsonValue | undefined;
+/** Drops non-JSON properties from a record while preserving valid values. */
+declare function normalizeRecord(value: Record<string, unknown>): Record<string, JsonValue>;
+/** Normalizes metadata and omits the field entirely when nothing survives. */
+declare function normalizeMetadata(value: Record<string, unknown>): Record<string, JsonValue> | undefined;
+/** Converts arbitrary content into the JSON-safe message content shape. */
+declare function normalizeContent(value: unknown): JsonValue;
+/** Flattens every recorded tool call from a normalized session. */
+declare function toolCalls(session: NormalizedSession): ToolCallRecord[];
+/** Filters normalized session messages by role. */
+declare function messagesByRole(session: NormalizedSession, role: NormalizedMessage["role"]): NormalizedMessage[];
+/** Returns every normalized system message from a session. */
+declare function systemMessages(session: NormalizedSession): NormalizedMessage[];
+/** Returns every normalized user message from a session. */
+declare function userMessages(session: NormalizedSession): NormalizedMessage[];
+/** Returns every normalized assistant message from a session. */
+declare function assistantMessages(session: NormalizedSession): NormalizedMessage[];
+/** Returns every normalized tool message from a session. */
+declare function toolMessages(session: NormalizedSession): NormalizedMessage[];
+/** Attaches a partial or complete harness run to an arbitrary thrown error. */
+declare function attachHarnessRunToError(error: unknown, run: HarnessRun): HarnessRunError;
+/** Reads an attached harness run back off a previously wrapped error value. */
+declare function getHarnessRunFromError(error: unknown): HarnessRun | undefined;
+/** Returns true when a value matches the normalized `HarnessRun` contract. */
+declare function isHarnessRun(value: unknown): value is HarnessRun;
+/** Returns true when a value matches the normalized session contract. */
+declare function isNormalizedSession(value: unknown): value is NormalizedSession;
+/** Reuses pre-normalized harness errors when a runtime already returns them. */
+declare function resolveHarnessRunErrors(result: unknown): Array<Record<string, JsonValue>>;
+/** Serializes an arbitrary thrown value into the normalized error shape. */
+declare function serializeError(error: unknown): Record<string, JsonValue>;
+
+export { type Harness, type HarnessContext, type HarnessMetadata, type HarnessPrompt, type HarnessPromptOptions, type HarnessRun, type HarnessRunError, type HarnessRuntime, type JsonPrimitive, type JsonValue, type NormalizedMessage, type NormalizedSession, type TimingSummary, type ToolCallRecord, type UsageSummary, assistantMessages, attachHarnessRunToError, getHarnessRunFromError, hasCallableMethod, isHarnessRun, isNormalizedSession, messagesByRole, normalizeContent, normalizeMetadata, normalizeRecord, resolveHarnessRunErrors, serializeError, systemMessages, toJsonValue, toolCalls, toolMessages, userMessages };