vitest-evals 0.9.0-beta.1 → 0.9.0-beta.2

package/README.md

@@ -14,6 +14,8 @@ Install a first-party harness package for the runtime you want to test:
 npm install -D @vitest-evals/harness-pi-ai
 # or
 npm install -D @vitest-evals/harness-ai-sdk
+# or
+npm install -D @vitest-evals/harness-openai-agents
 ```
 
 ## Core Model
@@ -25,7 +27,9 @@ npm install -D @vitest-evals/harness-ai-sdk
 - the returned `result.output` is the app-facing value you assert on directly
 - the returned `result.session` is the canonical JSON-serializable trace for
   reporting, replay, tool assertions, and judges
-- per-run judge inputs should usually live under `metadata`
+- scenario-specific judge criteria can live in `inputValue`; use `metadata` for
+  per-run expectations or harness configuration that are not part of the
+  scenario payload
 - suite-level `judges` are optional and run automatically after each `run(...)`
 - suite-level `judgeThreshold` controls fail-on-score for those automatic judges
 - every judge receives `JudgeContext`, including the configured `harness` with
@@ -144,10 +148,173 @@ The harness owns normalization, diagnostics, tool capture, replay plumbing, and
 reporter-facing artifacts. Your app just needs one runtime seam where those
 wrapped pieces can be injected.
 
+Replay opt-in belongs on the harness, via `toolReplay`, while replay mode and
+recording directory can live in Vitest environment config. Tool definitions
+should stay free of VCR policy.
+
 For the Pi-specific harness, output/session/usage normalization should usually
 be inferred automatically. Treat low-level normalization callbacks as an escape
 hatch, not part of the primary authoring path.
 
+For OpenAI Agents SDK apps, use
+`@vitest-evals/harness-openai-agents` with an existing `Agent` or
+`createAgent()` factory and a `Runner` / `createRunner()` callback. The harness
+calls `Runner.run(agent, input, options)` by default and exposes the same
+normalization and replay hooks when the app needs a custom entrypoint or
+structured domain output mapping.
+
+## Custom App Harnesses
+
+First-party harness packages are conveniences, not the only supported path. If
+you need to test a full application flow, define a harness that runs your app
+through its normal entrypoint and returns a normalized `HarnessRun`. The same
+harness should also expose `prompt`, which LLM-backed judges can reuse through
+`JudgeContext.harness.prompt`.
+
+```ts
+import {
+  describeEval,
+  namedJudge,
+  type JudgeContext,
+} from "vitest-evals";
+import {
+  normalizeContent,
+  normalizeMetadata,
+  toJsonValue,
+  type Harness,
+  type HarnessRun,
+} from "vitest-evals/harness";
+
+type AppEvent = {
+  type: string;
+  payload: Record<string, unknown>;
+};
+
+type AppEvalInput = {
+  events: AppEvent[];
+  criteria: {
+    contract: string;
+    pass: string[];
+    fail?: string[];
+  };
+};
+
+const appHarness: Harness<AppEvalInput> = {
+  name: "custom-app",
+  prompt: (input, options) => promptJudgeModel(input, options),
+  run: async (input, context): Promise<HarnessRun> => {
+    const result = await replayAppEvents(input.events, {
+      signal: context.signal,
+    });
+    const output = {
+      replies: result.replies,
+      sideEffects: result.sideEffects,
+    };
+
+    return {
+      output: toJsonValue(output),
+      session: {
+        messages: [
+          ...input.events.map((event) => ({
+            role: "user" as const,
+            content: normalizeContent(event),
+          })),
+          ...result.replies.map((reply) => ({
+            role: "assistant" as const,
+            content: normalizeContent(reply.text),
+            metadata: normalizeMetadata({
+              target: reply.target,
+            }),
+          })),
+        ],
+        outputText: result.replies.map((reply) => reply.text).join("\n\n"),
+        metadata: normalizeMetadata({
+          replyCount: result.replies.length,
+        }),
+      },
+      usage: {},
+      artifacts:
+        Object.keys(context.artifacts).length > 0
+          ? context.artifacts
+          : undefined,
+      errors: [],
+    };
+  },
+};
+
+const AppRubricJudge = namedJudge(
+  "AppRubricJudge",
+  async (
+    ctx: JudgeContext<AppEvalInput, Record<string, unknown>, typeof appHarness>,
+  ) => {
+    const verdict = await ctx.harness.prompt(
+      formatRubricPrompt({
+        output: ctx.output,
+        criteria: ctx.inputValue.criteria,
+      }),
+      {
+        metadata: {
+          judge: "AppRubricJudge",
+        },
+      },
+    );
+
+    return parseRubricVerdict(verdict);
+  },
+);
+
+describeEval(
+  "app behavior",
+  {
+    harness: appHarness,
+    judges: [AppRubricJudge],
+    judgeThreshold: 0.75,
+  },
+  (it) => {
+    it("handles an event flow", async ({ run }) => {
+      await run({
+        events: [
+          {
+            type: "message.created",
+            payload: {
+              text: "Summarize the current incident.",
+            },
+          },
+        ],
+        criteria: {
+          contract: "The app posts one user-visible incident summary.",
+          pass: ["The reply names the incident status."],
+          fail: ["The reply exposes internal metadata."],
+        },
+      });
+    });
+  },
+);
+```
+
+Use `Harness.run(...)` for the application under test and `Harness.prompt(...)`
+for judge model calls. Calling `ctx.harness.run(...)` from inside a judge runs
+the application a second time, so reserve that for judges that intentionally
+need a second execution. Put criteria on `inputValue` when they are part of the
+scenario itself; use per-run `metadata` for harness configuration or
+expectations that are not part of the scenario payload. `session.outputText` is
+the canonical text sent to judges, so define it deliberately when your app
+returns structured artifacts.
+
+Provider setup and rubric parsing stay in your harness and judge. The core
+package only requires the judge to return a `JudgeResult` with a score and
+optional metadata.
+
+Automatic suite-level judges are a good fit when every `run(...)` should get
+the same scoring. For cases where only some runs need an LLM judge, keep the
+suite free of automatic judges and use an explicit matcher:
+
+```ts
+await expect(result).toSatisfyJudge(AppRubricJudge, {
+  threshold: 0.75,
+});
+```
+
 ## Judge Matchers
 
 Use the matcher when a judge should behave like a normal Vitest assertion.

package/dist/judges/types.d.mts

@@ -11,7 +11,9 @@ type JudgeResult = {
 /**
  * Full normalized context passed to every judge.
  *
- * Per-run judge parameters should generally live under `metadata`.
+ * Scenario-owned judge criteria should live on `inputValue`. Use `metadata`
+ * for per-run expectations or harness configuration that are not part of the
+ * scenario payload.
  */
 interface JudgeContext<TInput = unknown, TMetadata extends HarnessMetadata = HarnessMetadata, THarness extends Harness<TInput, TMetadata> | undefined = Harness<TInput, TMetadata> | undefined> {
     /** Canonical text input passed to judges for plain prompt evaluation. */

package/dist/judges/types.d.ts

@@ -11,7 +11,9 @@ type JudgeResult = {
 /**
  * Full normalized context passed to every judge.
  *
- * Per-run judge parameters should generally live under `metadata`.
+ * Scenario-owned judge criteria should live on `inputValue`. Use `metadata`
+ * for per-run expectations or harness configuration that are not part of the
+ * scenario payload.
  */
 interface JudgeContext<TInput = unknown, TMetadata extends HarnessMetadata = HarnessMetadata, THarness extends Harness<TInput, TMetadata> | undefined = Harness<TInput, TMetadata> | undefined> {
     /** Canonical text input passed to judges for plain prompt evaluation. */

package/dist/judges/types.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/judges/types.ts"],"sourcesContent":["import type {\n  Harness,\n  HarnessMetadata,\n  HarnessRun,\n  ToolCallRecord,\n} from \"../harness\";\n\n/** Score payload returned by a judge. */\nexport type JudgeResult = {\n  score: number | null;\n  metadata?: {\n    rationale?: string;\n    output?: unknown;\n  } & Record<string, unknown>;\n};\n\n/**\n * Full normalized context passed to every judge.\n *\n * Per-run judge parameters should generally live under `metadata`.\n */\nexport interface JudgeContext<\n  TInput = unknown,\n  TMetadata extends HarnessMetadata = HarnessMetadata,\n  THarness extends Harness<TInput, TMetadata> | undefined =\n    | Harness<TInput, TMetadata>\n    | undefined,\n> {\n  /** Canonical text input passed to judges for plain prompt evaluation. */\n  input: string;\n  /** Canonical text response passed to judges for plain output evaluation. */\n  output: string;\n  /** Original non-string input value when the judge needs more than `input`. */\n  inputValue: TInput;\n  toolCalls: ToolCallRecord[];\n  metadata: Readonly<TMetadata>;\n  run: HarnessRun;\n  session: HarnessRun[\"session\"];\n  /** Harness associated with this judge context. */\n  harness: THarness;\n}\n\n/** Convenience helper for judges that accept explicit per-call params. */\nexport type JudgeOptions<\n  TParams extends Record<string, unknown> = Record<string, never>,\n  TInput = unknown,\n  TMetadata extends HarnessMetadata = HarnessMetadata,\n  THarness extends Harness<TInput, TMetadata> | undefined =\n    | Harness<TInput, TMetadata>\n    | undefined,\n> = JudgeContext<TInput, TMetadata, THarness> & TParams;\n\n/** Judge function over the normalized judge context. */\nexport type JudgeFn<\n  TOptions extends JudgeContext<any, any, any> = JudgeContext,\n> = (opts: TOptions) => Promise<JudgeResult> | JudgeResult;\n"],"mappings":";;;;;;;;;;;;;;;;AAAA;AAAA;","names":[]}
+{"version":3,"sources":["../../src/judges/types.ts"],"sourcesContent":["import type {\n  Harness,\n  HarnessMetadata,\n  HarnessRun,\n  ToolCallRecord,\n} from \"../harness\";\n\n/** Score payload returned by a judge. */\nexport type JudgeResult = {\n  score: number | null;\n  metadata?: {\n    rationale?: string;\n    output?: unknown;\n  } & Record<string, unknown>;\n};\n\n/**\n * Full normalized context passed to every judge.\n *\n * Scenario-owned judge criteria should live on `inputValue`. Use `metadata`\n * for per-run expectations or harness configuration that are not part of the\n * scenario payload.\n */\nexport interface JudgeContext<\n  TInput = unknown,\n  TMetadata extends HarnessMetadata = HarnessMetadata,\n  THarness extends Harness<TInput, TMetadata> | undefined =\n    | Harness<TInput, TMetadata>\n    | undefined,\n> {\n  /** Canonical text input passed to judges for plain prompt evaluation. */\n  input: string;\n  /** Canonical text response passed to judges for plain output evaluation. */\n  output: string;\n  /** Original non-string input value when the judge needs more than `input`. */\n  inputValue: TInput;\n  toolCalls: ToolCallRecord[];\n  metadata: Readonly<TMetadata>;\n  run: HarnessRun;\n  session: HarnessRun[\"session\"];\n  /** Harness associated with this judge context. */\n  harness: THarness;\n}\n\n/** Convenience helper for judges that accept explicit per-call params. */\nexport type JudgeOptions<\n  TParams extends Record<string, unknown> = Record<string, never>,\n  TInput = unknown,\n  TMetadata extends HarnessMetadata = HarnessMetadata,\n  THarness extends Harness<TInput, TMetadata> | undefined =\n    | Harness<TInput, TMetadata>\n    | undefined,\n> = JudgeContext<TInput, TMetadata, THarness> & TParams;\n\n/** Judge function over the normalized judge context. */\nexport type JudgeFn<\n  TOptions extends JudgeContext<any, any, any> = JudgeContext,\n> = (opts: TOptions) => Promise<JudgeResult> | JudgeResult;\n"],"mappings":";;;;;;;;;;;;;;;;AAAA;AAAA;","names":[]}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vitest-evals",
-  "version": "0.9.0-beta.1",
+  "version": "0.9.0-beta.2",
   "sideEffects": false,
   "types": "./dist/index.d.ts",
   "main": "./dist/index.js",