vitest-evals 0.11.0 → 0.13.0

package/README.md

@@ -30,8 +30,8 @@ workflow.
 
 - `describeEval(...)` binds exactly one harness to a suite
 - the suite callback receives a fixture-backed Vitest `it`
-- `run(input, { metadata? })` executes the harness explicitly and returns a
-  normalized `HarnessRun`
+- `run(input)` executes the harness explicitly and returns a normalized
+  `HarnessRun`
 - the returned `result.output` is the app-facing value you assert on directly
 - the returned `result.session` is the canonical JSON-serializable transcript for
   reporting, replay, tool assertions, and judges
@@ -41,19 +41,18 @@ workflow.
   that do not return traces themselves. Span attributes include typed
   OpenTelemetry GenAI semantic keys while still allowing provider-specific
   metadata
-- scenario-specific judge criteria can live in `input`; use `metadata` for
-  per-run expectations or harness configuration that are not part of the
-  scenario payload
+- scenario-specific judge criteria should live in `input` or explicit matcher
+  options, depending on whether the app or only the judge needs them
 - suite-level `judges` are optional and run automatically after each `run(...)`
 - suite-level `judgeThreshold` controls fail-on-score for those automatic judges
 - every judge is a named object with `assess(ctx)`
 - every judge receives `JudgeContext` with typed `input`, typed `output`, the
-  normalized run/session, tool calls, and metadata; `output` is only optional
+  normalized run/session, and tool calls; `output` is only optional
   when the harness output type includes `undefined`
 - judges own their prompt, rubric, and parsing; LLM-backed judges use
   `ctx.runJudge(...)` from a configured `judgeHarness`
 - explicit judge assertions use
-  `await expect(result).toSatisfyJudge(judge, context)`
+  `await expect(result).toSatisfyJudge(judge, options)`
 
 ## Explicit Run Example
 
@@ -80,18 +79,16 @@ describeEval(
       agent: () => createRefundAgent(),
     }),
     judgeHarness,
-    judges: [FactualityJudge()],
+    judges: [
+      FactualityJudge({
+        expected: "The refund request is approved.",
+      }),
+    ],
     judgeThreshold: 0.6,
   },
   (it) => {
     it("approves a refundable invoice", async ({ run }) => {
-      const result = await run("Refund invoice inv_123", {
-        metadata: {
-          expected: "The refund request is approved.",
-          expectedStatus: "approved",
-          expectedTools: ["lookupInvoice", "createRefund"],
-        },
-      });
+      const result = await run("Refund invoice inv_123");
 
       expect(result.output).toMatchObject({ status: "approved" });
       expect(toolCalls(result.session).map((call) => call.name)).toEqual([
@@ -121,13 +118,11 @@ describeEval("refund agent", { harness }, (it) => {
       input: "Refund invoice inv_404",
       expectedStatus: "denied",
     },
-  ])("$name", async ({ input, ...metadata }, { run }) => {
-    const result = await run(input, {
-      metadata,
-    });
+  ])("$name", async ({ input, expectedStatus }, { run }) => {
+    const result = await run(input);
 
     expect(result.output).toMatchObject({
-      status: metadata.expectedStatus,
+      status: expectedStatus,
     });
   });
 });
@@ -144,6 +139,18 @@ compatibility.
 
 Full transcripts and spans are preserved in the Vitest JSON report metadata.
 
+## Local Report UI
+
+The local report UI reads the same Vitest JSON artifacts and serves a React SPA
+for drilling into runs, eval cases, harness output, sessions, tool calls,
+scores, and trace spans.
+
+```sh
+pnpm exec vitest-evals serve vitest-results.json
+pnpm exec vitest-evals serve "eval-results/*.json"
+pnpm exec vitest-evals serve eval-results/
+```
+
 ## GitHub Actions Reporting
 
 Use Vitest JSON as the eval report artifact. It preserves the `meta` field that
@@ -201,7 +208,7 @@ First-party harness packages are conveniences, not the only supported path. If
 you need to test a full application flow, use `createHarness(...)` to run your
 app through its normal entrypoint and return the app-facing output. Judges own
 their prompt/rubric text separately from the system under test.
-When generics are needed, use `createHarness<Input, Output, Metadata>(...)`.
+When generics are needed, use `createHarness<Input, Output>(...)`.
 
 ```ts
 import {
@@ -209,7 +216,6 @@ import {
   createJudge,
   createJudgeHarness,
   describeEval,
-  type JudgeContext,
 } from "vitest-evals";
 
 type AppEvent = {
@@ -226,14 +232,12 @@ type AppEvalInput = {
   };
 };
 
-type AppEvalMetadata = Record<string, never>;
-
 type AppOutput = {
   replies: Array<{ text: string }>;
   sideEffects: string[];
 };
 
-const appHarness = createHarness<AppEvalInput, AppOutput, AppEvalMetadata>({
+const appHarness = createHarness<AppEvalInput, AppOutput>({
   name: "custom-app",
   run: async ({ input, signal }) => {
     const result = await replayAppEvents(input.events, {
@@ -259,9 +263,9 @@ const judgeHarness = createJudgeHarness({
     promptJudgeModel({ prompt, signal }),
 });
 
-const AppRubricJudge = createJudge(
+const AppRubricJudge = createJudge<AppEvalInput, AppOutput>(
   "AppRubricJudge",
-  async (ctx: JudgeContext<AppEvalInput, AppOutput, AppEvalMetadata>) => {
+  async (ctx) => {
     if (!ctx.runJudge) {
       throw new Error("AppRubricJudge requires a configured judgeHarness.");
     }
@@ -311,11 +315,11 @@ describeEval(
 Use `Harness.run(...)` for the application under test. Calling
 `ctx.harness.run(...)` from inside a judge runs the application a second time,
 so reserve that for judges that intentionally need a second execution. Put
-criteria on `input` when they are part of the scenario itself; use per-run
-`metadata` for harness configuration or expectations that are not part of the
-scenario payload. `createHarness(...)` builds a default user/assistant session
-from `input` and typed `output`; return a full `HarnessRun` only when you need
-exact session control.
+criteria on `input` when they are part of the scenario itself; pass
+case-specific judge criteria through matcher options, or configure suite-wide
+criteria on the judge instance. `createHarness(...)` builds a default
+user/assistant session from `input` and typed `output`; return a full
+`HarnessRun` only when you need exact session control.
 
 Provider setup and rubric parsing stay in your judge. The core
 package only requires the judge to return a `JudgeResult` with a score and
@@ -437,7 +441,7 @@ so use that only when a second run is intentional.
 
 For an `EvalHarnessRun` returned by fixture `run(...)`,
 `toSatisfyJudge(...)` uses the run's typed `output` and reuses the registered
-input and metadata. It requires any custom judge params and rejects judges whose
+input. It requires any custom judge params and rejects judges whose
 output type cannot assess the received value. Inside an eval test,
 matcher calls on registered output objects or session objects reuse that exact
 run context when the value can be registered by reference, so
@@ -445,10 +449,10 @@ run context when the value can be registered by reference, so
 outputs. Other raw values fall back to the current test's most recent
 `run(...)` context. For
 manually-created runs or values outside an eval context, pass any required
-`input`, `metadata`, or `harness` in matcher options. Structured or
+`input` or `harness` in matcher options. Structured or
 programmatic result checks should usually assert on `result.output` directly.
 When a judge needs richer normalized context or the configured suite harness,
-type it with `JudgeContext`.
+type it with `createJudge<Input, Output>(...)` or `JudgeContext<Input, Output>`.
 
 When you only need deterministic contract checks, built-ins such as
 `StructuredOutputJudge()` and `ToolCallJudge()` are still available.

package/bin/vitest-evals.js

@@ -0,0 +1,8 @@
+#!/usr/bin/env node
+
+import("../dist/cli.mjs")
+  .then(({ runVitestEvalsCli }) => runVitestEvalsCli(process.argv.slice(2)))
+  .catch((error) => {
+    console.error(error instanceof Error ? error.message : String(error));
+    process.exitCode = 1;
+  });

package/dist/cli.d.mts

@@ -0,0 +1,13 @@
+#!/usr/bin/env node
+/** Output streams used by the `vitest-evals` CLI runner. */
+type VitestEvalsCliIo = {
+    stdout?: Pick<NodeJS.WriteStream, "write">;
+};
+/** Options for running the `vitest-evals` CLI. */
+type RunVitestEvalsCliOptions = VitestEvalsCliIo & {
+    cwd?: string;
+};
+/** Runs the product-facing `vitest-evals` CLI. */
+declare function runVitestEvalsCli(args?: string[], options?: RunVitestEvalsCliOptions): Promise<void>;
+
+export { type RunVitestEvalsCliOptions, type VitestEvalsCliIo, runVitestEvalsCli };

package/dist/cli.d.ts

@@ -0,0 +1,13 @@
+#!/usr/bin/env node
+/** Output streams used by the `vitest-evals` CLI runner. */
+type VitestEvalsCliIo = {
+    stdout?: Pick<NodeJS.WriteStream, "write">;
+};
+/** Options for running the `vitest-evals` CLI. */
+type RunVitestEvalsCliOptions = VitestEvalsCliIo & {
+    cwd?: string;
+};
+/** Runs the product-facing `vitest-evals` CLI. */
+declare function runVitestEvalsCli(args?: string[], options?: RunVitestEvalsCliOptions): Promise<void>;
+
+export { type RunVitestEvalsCliOptions, type VitestEvalsCliIo, runVitestEvalsCli };

package/dist/cli.js

@@ -0,0 +1,83 @@
+#!/usr/bin/env node
+"use strict";
+var __create = Object.create;
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getProtoOf = Object.getPrototypeOf;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
+  // If the importer is in node compatibility mode or this is not an ESM
+  // file that has been converted to a CommonJS file using a Babel-
+  // compatible transform (i.e. "__esModule" has not been set), then set
+  // "default" to the CommonJS "module.exports" for node compatibility.
+  isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
+  mod
+));
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+
+// src/cli.ts
+var cli_exports = {};
+__export(cli_exports, {
+  runVitestEvalsCli: () => runVitestEvalsCli
+});
+module.exports = __toCommonJS(cli_exports);
+async function runVitestEvalsCli(args = process.argv.slice(2), options = {}) {
+  const [command, ...commandArgs] = args;
+  if (!command || command === "help" || command === "--help" || command === "-h") {
+    writeLine(options.stdout, usage());
+    return;
+  }
+  switch (command) {
+    case "serve": {
+      const { runReportUiCli } = await import("@vitest-evals/report-ui");
+      await runReportUiCli(commandArgs, {
+        commandName: "vitest-evals serve",
+        cwd: options.cwd,
+        stdout: options.stdout
+      });
+      return;
+    }
+    default:
+      throw new Error(`Unknown command: ${command}
+
+${usage()}`);
+  }
+}
+function usage() {
+  return [
+    "Usage: vitest-evals <command>",
+    "",
+    "Commands:",
+    "  serve [json | dir | glob]   Serve the local report UI",
+    "",
+    "Run `vitest-evals serve --help` for report UI options."
+  ].join("\n");
+}
+function writeLine(stdout, message) {
+  (stdout ?? process.stdout).write(`${message}
+`);
+}
+if (typeof require !== "undefined" && typeof module !== "undefined" && require.main === module) {
+  runVitestEvalsCli().catch((error) => {
+    console.error(error instanceof Error ? error.message : String(error));
+    process.exitCode = 1;
+  });
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  runVitestEvalsCli
+});
+//# sourceMappingURL=cli.js.map

package/dist/cli.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/cli.ts"],"sourcesContent":["#!/usr/bin/env node\n\n/** Output streams used by the `vitest-evals` CLI runner. */\nexport type VitestEvalsCliIo = {\n  stdout?: Pick<NodeJS.WriteStream, \"write\">;\n};\n\n/** Options for running the `vitest-evals` CLI. */\nexport type RunVitestEvalsCliOptions = VitestEvalsCliIo & {\n  cwd?: string;\n};\n\n/** Runs the product-facing `vitest-evals` CLI. */\nexport async function runVitestEvalsCli(\n  args = process.argv.slice(2),\n  options: RunVitestEvalsCliOptions = {},\n) {\n  const [command, ...commandArgs] = args;\n\n  if (\n    !command ||\n    command === \"help\" ||\n    command === \"--help\" ||\n    command === \"-h\"\n  ) {\n    writeLine(options.stdout, usage());\n    return;\n  }\n\n  switch (command) {\n    case \"serve\": {\n      const { runReportUiCli } = await import(\"@vitest-evals/report-ui\");\n      await runReportUiCli(commandArgs, {\n        commandName: \"vitest-evals serve\",\n        cwd: options.cwd,\n        stdout: options.stdout,\n      });\n      return;\n    }\n    default:\n      throw new Error(`Unknown command: ${command}\\n\\n${usage()}`);\n  }\n}\n\nfunction usage() {\n  return [\n    \"Usage: vitest-evals <command>\",\n    \"\",\n    \"Commands:\",\n    \"  serve [json | dir | glob]   Serve the local report UI\",\n    \"\",\n    \"Run `vitest-evals serve --help` for report UI options.\",\n  ].join(\"\\n\");\n}\n\nfunction writeLine(\n  stdout: Pick<NodeJS.WriteStream, \"write\"> | undefined,\n  message: string,\n) {\n  (stdout ?? process.stdout).write(`${message}\\n`);\n}\n\ndeclare const require: NodeJS.Require | undefined;\ndeclare const module: NodeJS.Module | undefined;\n\nif (\n  typeof require !== \"undefined\" &&\n  typeof module !== \"undefined\" &&\n  require.main === module\n) {\n  runVitestEvalsCli().catch((error) => {\n    console.error(error instanceof Error ? error.message : String(error));\n    process.exitCode = 1;\n  });\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAaA,eAAsB,kBACpB,OAAO,QAAQ,KAAK,MAAM,CAAC,GAC3B,UAAoC,CAAC,GACrC;AACA,QAAM,CAAC,SAAS,GAAG,WAAW,IAAI;AAElC,MACE,CAAC,WACD,YAAY,UACZ,YAAY,YACZ,YAAY,MACZ;AACA,cAAU,QAAQ,QAAQ,MAAM,CAAC;AACjC;AAAA,EACF;AAEA,UAAQ,SAAS;AAAA,IACf,KAAK,SAAS;AACZ,YAAM,EAAE,eAAe,IAAI,MAAM,OAAO,yBAAyB;AACjE,YAAM,eAAe,aAAa;AAAA,QAChC,aAAa;AAAA,QACb,KAAK,QAAQ;AAAA,QACb,QAAQ,QAAQ;AAAA,MAClB,CAAC;AACD;AAAA,IACF;AAAA,IACA;AACE,YAAM,IAAI,MAAM,oBAAoB,OAAO;AAAA;AAAA,EAAO,MAAM,CAAC,EAAE;AAAA,EAC/D;AACF;AAEA,SAAS,QAAQ;AACf,SAAO;AAAA,IACL;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF,EAAE,KAAK,IAAI;AACb;AAEA,SAAS,UACP,QACA,SACA;AACA,GAAC,UAAU,QAAQ,QAAQ,MAAM,GAAG,OAAO;AAAA,CAAI;AACjD;AAKA,IACE,OAAO,YAAY,eACnB,OAAO,WAAW,eAClB,QAAQ,SAAS,QACjB;AACA,oBAAkB,EAAE,MAAM,CAAC,UAAU;AACnC,YAAQ,MAAM,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK,CAAC;AACpE,YAAQ,WAAW;AAAA,EACrB,CAAC;AACH;","names":[]}

package/dist/cli.mjs

@@ -0,0 +1,55 @@
+#!/usr/bin/env node
+var __require = /* @__PURE__ */ ((x) => typeof require !== "undefined" ? require : typeof Proxy !== "undefined" ? new Proxy(x, {
+  get: (a, b) => (typeof require !== "undefined" ? require : a)[b]
+}) : x)(function(x) {
+  if (typeof require !== "undefined") return require.apply(this, arguments);
+  throw Error('Dynamic require of "' + x + '" is not supported');
+});
+
+// src/cli.ts
+async function runVitestEvalsCli(args = process.argv.slice(2), options = {}) {
+  const [command, ...commandArgs] = args;
+  if (!command || command === "help" || command === "--help" || command === "-h") {
+    writeLine(options.stdout, usage());
+    return;
+  }
+  switch (command) {
+    case "serve": {
+      const { runReportUiCli } = await import("@vitest-evals/report-ui");
+      await runReportUiCli(commandArgs, {
+        commandName: "vitest-evals serve",
+        cwd: options.cwd,
+        stdout: options.stdout
+      });
+      return;
+    }
+    default:
+      throw new Error(`Unknown command: ${command}
+
+${usage()}`);
+  }
+}
+function usage() {
+  return [
+    "Usage: vitest-evals <command>",
+    "",
+    "Commands:",
+    "  serve [json | dir | glob]   Serve the local report UI",
+    "",
+    "Run `vitest-evals serve --help` for report UI options."
+  ].join("\n");
+}
+function writeLine(stdout, message) {
+  (stdout ?? process.stdout).write(`${message}
+`);
+}
+if (typeof __require !== "undefined" && typeof module !== "undefined" && __require.main === module) {
+  runVitestEvalsCli().catch((error) => {
+    console.error(error instanceof Error ? error.message : String(error));
+    process.exitCode = 1;
+  });
+}
+export {
+  runVitestEvalsCli
+};
+//# sourceMappingURL=cli.mjs.map

package/dist/cli.mjs.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/cli.ts"],"sourcesContent":["#!/usr/bin/env node\n\n/** Output streams used by the `vitest-evals` CLI runner. */\nexport type VitestEvalsCliIo = {\n  stdout?: Pick<NodeJS.WriteStream, \"write\">;\n};\n\n/** Options for running the `vitest-evals` CLI. */\nexport type RunVitestEvalsCliOptions = VitestEvalsCliIo & {\n  cwd?: string;\n};\n\n/** Runs the product-facing `vitest-evals` CLI. */\nexport async function runVitestEvalsCli(\n  args = process.argv.slice(2),\n  options: RunVitestEvalsCliOptions = {},\n) {\n  const [command, ...commandArgs] = args;\n\n  if (\n    !command ||\n    command === \"help\" ||\n    command === \"--help\" ||\n    command === \"-h\"\n  ) {\n    writeLine(options.stdout, usage());\n    return;\n  }\n\n  switch (command) {\n    case \"serve\": {\n      const { runReportUiCli } = await import(\"@vitest-evals/report-ui\");\n      await runReportUiCli(commandArgs, {\n        commandName: \"vitest-evals serve\",\n        cwd: options.cwd,\n        stdout: options.stdout,\n      });\n      return;\n    }\n    default:\n      throw new Error(`Unknown command: ${command}\\n\\n${usage()}`);\n  }\n}\n\nfunction usage() {\n  return [\n    \"Usage: vitest-evals <command>\",\n    \"\",\n    \"Commands:\",\n    \"  serve [json | dir | glob]   Serve the local report UI\",\n    \"\",\n    \"Run `vitest-evals serve --help` for report UI options.\",\n  ].join(\"\\n\");\n}\n\nfunction writeLine(\n  stdout: Pick<NodeJS.WriteStream, \"write\"> | undefined,\n  message: string,\n) {\n  (stdout ?? process.stdout).write(`${message}\\n`);\n}\n\ndeclare const require: NodeJS.Require | undefined;\ndeclare const module: NodeJS.Module | undefined;\n\nif (\n  typeof require !== \"undefined\" &&\n  typeof module !== \"undefined\" &&\n  require.main === module\n) {\n  runVitestEvalsCli().catch((error) => {\n    console.error(error instanceof Error ? error.message : String(error));\n    process.exitCode = 1;\n  });\n}\n"],"mappings":";;;;;;;;;AAaA,eAAsB,kBACpB,OAAO,QAAQ,KAAK,MAAM,CAAC,GAC3B,UAAoC,CAAC,GACrC;AACA,QAAM,CAAC,SAAS,GAAG,WAAW,IAAI;AAElC,MACE,CAAC,WACD,YAAY,UACZ,YAAY,YACZ,YAAY,MACZ;AACA,cAAU,QAAQ,QAAQ,MAAM,CAAC;AACjC;AAAA,EACF;AAEA,UAAQ,SAAS;AAAA,IACf,KAAK,SAAS;AACZ,YAAM,EAAE,eAAe,IAAI,MAAM,OAAO,yBAAyB;AACjE,YAAM,eAAe,aAAa;AAAA,QAChC,aAAa;AAAA,QACb,KAAK,QAAQ;AAAA,QACb,QAAQ,QAAQ;AAAA,MAClB,CAAC;AACD;AAAA,IACF;AAAA,IACA;AACE,YAAM,IAAI,MAAM,oBAAoB,OAAO;AAAA;AAAA,EAAO,MAAM,CAAC,EAAE;AAAA,EAC/D;AACF;AAEA,SAAS,QAAQ;AACf,SAAO;AAAA,IACL;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF,EAAE,KAAK,IAAI;AACb;AAEA,SAAS,UACP,QACA,SACA;AACA,GAAC,UAAU,QAAQ,QAAQ,MAAM,GAAG,OAAO;AAAA,CAAI;AACjD;AAKA,IACE,OAAO,cAAY,eACnB,OAAO,WAAW,eAClB,UAAQ,SAAS,QACjB;AACA,oBAAkB,EAAE,MAAM,CAAC,UAAU;AACnC,YAAQ,MAAM,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK,CAAC;AACpE,YAAQ,WAAW;AAAA,EACrB,CAAC;AACH;","names":[]}