vitest-evals 0.12.0 → 0.13.0

package/dist/index.d.ts

@@ -1,9 +1,9 @@
 import * as vitest from 'vitest';
 import { TestAPI } from 'vitest';
-import { HarnessMetadata, Harness } from './harness.js';
-export { CreateHarnessOptions, CreateHarnessRunArgs, CreateToolCallSpansOptions, EnsureRunTraceOptions, HarnessContext, HarnessResultLike, MaybePromise, SimpleHarnessResult, SimpleSpanEvent, SimpleSpanRecord, SimpleToolCallRecord, SimpleTraceRecord, attachHarnessRunToError, createFailedHarnessRun, createGenAiUsageAttributes, createHarness, createToolCallSpans, ensureRunTrace, getHarnessRunFromError, normalizeHarnessRun, normalizeSpanAttributes, normalizeSpanError, toJsonValue } from './harness.js';
-import { JudgeContext, Judge, JudgeResult, JudgeAssessFn, JudgeAssessor, JudgeAssessWithAssessorFn } from './judges/types.js';
-export { BoundJudgeAssessor, JudgeAssessorOptions, JudgeOptions } from './judges/types.js';
+import { Harness } from './harness.js';
+export { CreateHarnessOptions, CreateHarnessRunArgs, CreateToolCallSpansOptions, EnsureRunTraceOptions, HarnessContext, HarnessMetadata, HarnessResultLike, MaybePromise, SimpleHarnessResult, SimpleSpanEvent, SimpleSpanRecord, SimpleToolCallRecord, SimpleTraceRecord, attachHarnessRunToError, createFailedHarnessRun, createGenAiUsageAttributes, createHarness, createToolCallSpans, ensureRunTrace, getHarnessRunFromError, normalizeHarnessRun, normalizeSpanAttributes, normalizeSpanError, toJsonValue } from './harness.js';
+import { JudgeContext, Judge, JudgeResult, JudgeAssessFn, JudgeOptions, JudgeAssessor, JudgeAssessWithAssessorFn } from './judges/types.js';
+export { BoundJudgeAssessor, JudgeAssessorOptions } from './judges/types.js';
 import { JudgeHarness } from './judges/judgeHarness.js';
 export { CreateJudgeHarnessOptions, CreateJudgeHarnessRunOptions, JudgeHarnessInput, JudgeHarnessOutput, RunJudge, RunJudgeOptions, createJudgeHarness, runJudgeHarness } from './judges/judgeHarness.js';
 export { wrapText } from './wrapText.js';
@@ -32,14 +32,14 @@ type EvalTaskMeta = {
         run: HarnessRun;
     };
 };
-type HarnessInput<THarness extends Harness<any, any, any>> = THarness extends Harness<infer TInput, any, any> ? TInput : unknown;
-type HarnessMetadataFor<THarness extends Harness<any, any, any>> = THarness extends Harness<any, any, infer TMetadata> ? TMetadata : HarnessMetadata;
-type HarnessOutput<THarness extends Harness<any, any, any>> = THarness extends Harness<any, infer TOutput, any> ? TOutput : JsonValue | undefined;
-type CreateJudgeConfig<TOptions extends JudgeContext<any, any, any, any> = JudgeContext> = {
+type HarnessInput<THarness extends Harness<any, any>> = THarness extends Harness<infer TInput, any> ? TInput : unknown;
+type HarnessOutput<THarness extends Harness<any, any>> = THarness extends Harness<any, infer TOutput> ? TOutput : JsonValue | undefined;
+type CreateJudgeConfig<TOptions extends JudgeContext<any, any, any> = JudgeContext> = {
     name: string;
     judgeHarness?: JudgeHarness;
     assess: JudgeAssessFn<TOptions>;
 };
+type CreateJudgeContext<TInput, TOutput extends JsonValue | undefined, TOptions extends object, THarness extends Harness<TInput, TOutput> | undefined = Harness<TInput, TOutput> | undefined> = JudgeOptions<TInput, TOutput, TOptions, THarness>;
 declare const evalHarnessRunBrand: unique symbol;
 /**
  * Harness run returned by the fixture-backed `run(...)` API.
@@ -53,42 +53,22 @@ declare const evalHarnessRunBrand: unique symbol;
  * });
  * ```
  */
-type EvalHarnessRun<TInput = unknown, TOutput extends JsonValue | undefined = JsonValue | undefined, TMetadata extends HarnessMetadata = HarnessMetadata, THarness extends Harness<TInput, TOutput, TMetadata> = Harness<TInput, TOutput, TMetadata>> = HarnessRun<TOutput> & {
+type EvalHarnessRun<TInput = unknown, TOutput extends JsonValue | undefined = JsonValue | undefined, THarness extends Harness<TInput, TOutput> = Harness<TInput, TOutput>> = HarnessRun<TOutput> & {
     readonly [evalHarnessRunBrand]: {
         readonly input: TInput;
-        readonly metadata: TMetadata;
         readonly output: TOutput;
         readonly harness: THarness;
     };
 };
-/**
- * Per-run metadata forwarded to the harness alongside the test input.
- *
- * @example
- * ```ts
- * await run("Refund invoice inv_123", {
- *   metadata: {
- *     expected: { status: "approved" },
- *     expectedTools: ["lookupInvoice", "createRefund"],
- *   },
- * });
- * ```
- */
-interface EvalRunOptions<TMetadata extends HarnessMetadata = HarnessMetadata> {
-    /** Per-run expectations or configuration forwarded to harnesses and judges. */
-    metadata?: TMetadata;
-}
 /**
  * Explicit harness execution primitive exposed to each eval test.
  *
  * @example
  * ```ts
- * const result = await run("Refund invoice inv_123", {
- *   metadata: { expected: { status: "approved" } },
- * });
+ * const result = await run("Refund invoice inv_123");
  * ```
  */
-type EvalRun<TInput = unknown, TOutput extends JsonValue | undefined = JsonValue | undefined, TMetadata extends HarnessMetadata = HarnessMetadata, THarness extends Harness<TInput, TOutput, TMetadata> = Harness<TInput, TOutput, TMetadata>> = (input: TInput, options?: EvalRunOptions<TMetadata>) => Promise<EvalHarnessRun<TInput, TOutput, TMetadata, THarness>>;
+type EvalRun<TInput = unknown, TOutput extends JsonValue | undefined = JsonValue | undefined, THarness extends Harness<TInput, TOutput> = Harness<TInput, TOutput>> = (input: TInput) => Promise<EvalHarnessRun<TInput, TOutput, THarness>>;
 /**
  * Fixture-backed Vitest context exposed inside `describeEval(...)` tests.
  *
@@ -103,11 +83,11 @@ type EvalRun<TInput = unknown, TOutput extends JsonValue | undefined = JsonValue
  * });
  * ```
  */
-interface EvalTestContext<TInput = unknown, TOutput extends JsonValue | undefined = JsonValue | undefined, TMetadata extends HarnessMetadata = HarnessMetadata, THarness extends Harness<TInput, TOutput, TMetadata> = Harness<TInput, TOutput, TMetadata>> {
-    run: EvalRun<TInput, TOutput, TMetadata, THarness>;
+interface EvalTestContext<TInput = unknown, TOutput extends JsonValue | undefined = JsonValue | undefined, THarness extends Harness<TInput, TOutput> = Harness<TInput, TOutput>> {
+    run: EvalRun<TInput, TOutput, THarness>;
 }
 /** Fixture-backed Vitest test API exposed inside `describeEval(...)`. */
-type EvalTestAPI<TInput = unknown, TOutput extends JsonValue | undefined = JsonValue | undefined, TMetadata extends HarnessMetadata = HarnessMetadata, THarness extends Harness<TInput, TOutput, TMetadata> = Harness<TInput, TOutput, TMetadata>> = TestAPI<EvalTestContext<TInput, TOutput, TMetadata, THarness>>;
+type EvalTestAPI<TInput = unknown, TOutput extends JsonValue | undefined = JsonValue | undefined, THarness extends Harness<TInput, TOutput> = Harness<TInput, TOutput>> = TestAPI<EvalTestContext<TInput, TOutput, THarness>>;
 /**
  * Suite-level configuration for a harness-backed eval block.
  *
@@ -124,11 +104,11 @@ type EvalTestAPI<TInput = unknown, TOutput extends JsonValue | undefined = JsonV
  * };
  * ```
  */
-interface DescribeEvalOptions<TInput = unknown, TOutput extends JsonValue | undefined = JsonValue | undefined, TMetadata extends HarnessMetadata = HarnessMetadata, THarness extends Harness<TInput, TOutput, TMetadata> = Harness<TInput, TOutput, TMetadata>> {
+interface DescribeEvalOptions<TInput = unknown, TOutput extends JsonValue | undefined = JsonValue | undefined, THarness extends Harness<TInput, TOutput> = Harness<TInput, TOutput>> {
     /** Harness used for every explicit `run(...)` call in the suite. */
     harness: THarness;
     /** Automatic judges applied after each successful `run(...)`. */
-    judges?: Array<Judge<JudgeContext<TInput, TOutput, TMetadata, THarness>>>;
+    judges?: Array<Judge<JudgeContext<TInput, TOutput, THarness>>>;
     /** Optional judge-side harness used only by judges that call `ctx.runJudge(...)`. */
     judgeHarness?: JudgeHarness;
     /** Passing threshold for automatic suite-level judges. `null` disables fail-on-score. */
@@ -136,26 +116,23 @@ interface DescribeEvalOptions<TInput = unknown, TOutput extends JsonValue | unde
     /** Skips the entire eval suite when the predicate returns true. */
     skipIf?: () => boolean;
 }
-type JudgeAssertionInput<TJudgeOptions extends JudgeContext<any, any, any, any>> = TJudgeOptions extends {
+type JudgeAssertionInput<TJudgeOptions extends JudgeContext<any, any, any>> = TJudgeOptions extends {
     input: infer TInput;
 } ? TInput : unknown;
-type JudgeAssertionOutput<TJudgeOptions extends JudgeContext<any, any, any, any>> = TJudgeOptions extends {
+type JudgeAssertionOutput<TJudgeOptions extends JudgeContext<any, any, any>> = TJudgeOptions extends {
     output: infer TOutput;
 } ? TOutput : JsonValue | undefined;
-type JudgeAssertionMetadata<TJudgeOptions extends JudgeContext<any, any, any, any>> = TJudgeOptions extends {
-    metadata: infer TMetadata;
-} ? TMetadata : HarnessMetadata;
-type JudgeAssertionHarness<TJudgeOptions extends JudgeContext<any, any, any, any>> = TJudgeOptions extends {
+type JudgeAssertionHarness<TJudgeOptions extends JudgeContext<any, any, any>> = TJudgeOptions extends {
     harness: infer THarness;
-} ? Exclude<THarness, undefined> : Harness<JudgeAssertionInput<TJudgeOptions>, JudgeAssertionOutput<TJudgeOptions>, JudgeAssertionMetadata<TJudgeOptions>>;
-type JudgeAssertionReservedKey = keyof JudgeContext<any, any, any, any> | "judgeHarness" | "signal" | "threshold";
-type JudgeAssertionParams<TJudgeOptions extends JudgeContext<any, any, any, any>> = Omit<TJudgeOptions, JudgeAssertionReservedKey>;
+} ? Exclude<THarness, undefined> : Harness<JudgeAssertionInput<TJudgeOptions>, JudgeAssertionOutput<TJudgeOptions>>;
+type JudgeAssertionReservedKey = keyof JudgeContext<any, any, any> | "judgeHarness" | "signal" | "threshold";
+type JudgeAssertionParams<TJudgeOptions extends JudgeContext<any, any, any>> = Omit<TJudgeOptions, JudgeAssertionReservedKey>;
 type RequiredKeys<T> = {
     [K in keyof T]-?: Record<string, never> extends Pick<T, K> ? never : K;
 }[keyof T];
-type JudgeAssertionArgs<TJudgeOptions extends JudgeContext<any, any, any, any>> = RequiredKeys<JudgeAssertionParams<TJudgeOptions>> extends never ? [options?: JudgeAssertionOptions<TJudgeOptions>] : [options: JudgeAssertionOptions<TJudgeOptions>];
-type MatcherOutput<TReceived> = TReceived extends EvalHarnessRun<any, infer TOutput, any, any> ? TOutput : TReceived extends HarnessRun<infer TOutput> ? TOutput : TReceived extends NormalizedSession ? JsonValue | undefined : TReceived extends JsonValue ? TReceived : JsonValue | undefined;
-type JudgeForReceived<TReceived, TJudgeOptions extends JudgeContext<any, any, any, any>> = MatcherOutput<TReceived> extends JudgeAssertionOutput<TJudgeOptions> ? Judge<TJudgeOptions> : never;
+type JudgeAssertionArgs<TJudgeOptions extends JudgeContext<any, any, any>> = RequiredKeys<JudgeAssertionParams<TJudgeOptions>> extends never ? [options?: JudgeAssertionOptions<TJudgeOptions>] : [options: JudgeAssertionOptions<TJudgeOptions>];
+type MatcherOutput<TReceived> = TReceived extends EvalHarnessRun<any, infer TOutput, any> ? TOutput : TReceived extends HarnessRun<infer TOutput> ? TOutput : TReceived extends NormalizedSession ? JsonValue | undefined : TReceived extends JsonValue ? TReceived : JsonValue | undefined;
+type JudgeForReceived<TReceived, TJudgeOptions extends JudgeContext<any, any, any>> = MatcherOutput<TReceived> extends JudgeAssertionOutput<TJudgeOptions> ? Judge<TJudgeOptions> : never;
 /**
  * Optional overrides passed to `expect(...).toSatisfyJudge(...)`.
  *
@@ -166,13 +143,11 @@ type JudgeForReceived<TReceived, TJudgeOptions extends JudgeContext<any, any, an
  * });
  * ```
  */
-type JudgeAssertionOptions<TJudgeOptions extends JudgeContext<any, any, any, any> = JudgeContext> = JudgeAssertionParams<TJudgeOptions> & {
+type JudgeAssertionOptions<TJudgeOptions extends JudgeContext<any, any, any> = JudgeContext> = JudgeAssertionParams<TJudgeOptions> & {
     /** Override or provide the original eval input for the judge. */
     input?: JudgeAssertionInput<TJudgeOptions>;
     /** Override or provide the app-facing output for the judge. */
     output?: JudgeAssertionOutput<TJudgeOptions>;
-    /** Override or provide per-run judge metadata. */
-    metadata?: JudgeAssertionMetadata<TJudgeOptions>;
     /** Override or provide flattened tool calls for the judge. */
     toolCalls?: ToolCallRecord[];
     /** Override or provide the complete normalized harness run. */
@@ -187,7 +162,7 @@ type JudgeAssertionOptions<TJudgeOptions extends JudgeContext<any, any, any, any
     threshold?: number | null;
 };
 /** Function type installed as the `toSatisfyJudge(...)` matcher. */
-type ToSatisfyJudge<TReceived = unknown> = <TJudgeOptions extends JudgeContext<any, any, any, any> = JudgeContext>(judge: JudgeForReceived<TReceived, TJudgeOptions>, ...args: JudgeAssertionArgs<TJudgeOptions>) => Promise<TReceived>;
+type ToSatisfyJudge<TReceived = unknown> = <TJudgeOptions extends JudgeContext<any, any, any> = JudgeContext>(judge: JudgeForReceived<TReceived, TJudgeOptions>, ...args: JudgeAssertionArgs<TJudgeOptions>) => Promise<TReceived>;
 /**
  * Vitest matcher extension surface added by `vitest-evals`.
  *
@@ -241,22 +216,19 @@ declare module "vitest" {
  *   judges: [ToolCallJudge()],
  * }, (it) => {
  *   it("approves a refundable invoice", async ({ run }) => {
- *     const result = await run("Refund invoice inv_123", {
- *       metadata: {
- *         expected: "Invoice inv_123 should be refunded.",
- *       },
- *     });
+ *     const result = await run("Refund invoice inv_123");
  *
  *     expect(result.output).toMatchObject({ status: "approved" });
  *     expect(toolCalls(result.session)).toHaveLength(2);
  *     await expect(result).toSatisfyJudge(FactualityJudge(), {
+ *       expected: "Invoice inv_123 should be refunded.",
  *       threshold: 0.6,
  *     });
  *   });
  * });
  * ```
  */
-declare function describeEval<THarness extends Harness<any, any, any>>(name: string, options: DescribeEvalOptions<HarnessInput<THarness>, HarnessOutput<THarness>, HarnessMetadataFor<THarness>, THarness>, define: (it: EvalTestAPI<HarnessInput<THarness>, HarnessOutput<THarness>, HarnessMetadataFor<THarness>, THarness>) => void): vitest.SuiteCollector<object>;
+declare function describeEval<THarness extends Harness<any, any>>(name: string, options: DescribeEvalOptions<HarnessInput<THarness>, HarnessOutput<THarness>, THarness>, define: (it: EvalTestAPI<HarnessInput<THarness>, HarnessOutput<THarness>, THarness>) => void): vitest.SuiteCollector<object>;
 /**
  * Formats judge results for reporter and assertion output.
  *
@@ -284,17 +256,20 @@ declare function formatScores(scores: (JudgeResult & {
  *
  * @example
  * ```ts
- * import { createJudge, type JudgeContext } from "vitest-evals";
+ * import { createJudge } from "vitest-evals";
  *
  * type RefundOutput = { status: "approved" | "denied" };
- * type RefundMetadata = { expected: { status: RefundOutput["status"] } };
  *
- * export const RefundStatusJudge = createJudge(
+ * export const RefundStatusJudge = createJudge<
+ *   string,
+ *   RefundOutput,
+ *   { expectedStatus: RefundOutput["status"] }
+ * >(
  *   "RefundStatusJudge",
- *   async ({ output, metadata }: JudgeContext<string, RefundOutput, RefundMetadata>) => ({
- *     score: output.status === metadata.expected.status ? 1 : 0,
+ *   async ({ output, expectedStatus }) => ({
+ *     score: output.status === expectedStatus ? 1 : 0,
  *     metadata: {
- *       rationale: `Expected ${metadata.expected.status}, got ${output.status}`,
+ *       rationale: `Expected ${expectedStatus}, got ${output.status}`,
  *     },
  *   }),
  * );
@@ -303,12 +278,14 @@ declare function formatScores(scores: (JudgeResult & {
  * For LLM-backed judges, prefer the object form with `ctx.runJudge(...)` so
  * provider-specific model configuration stays in the judge harness.
  */
-declare function createJudge<TOptions extends JudgeContext<any, any, any, any>>(name: string, assess: JudgeAssessFn<TOptions>): Judge<TOptions>;
-declare function createJudge<TOptions extends JudgeContext<any, any, any, any>>(config: CreateJudgeConfig<TOptions>): Judge<TOptions>;
+declare function createJudge<TOptions extends JudgeContext<any, any, any>>(name: string, assess: JudgeAssessFn<TOptions>): Judge<TOptions>;
+declare function createJudge<TOptions extends JudgeContext<any, any, any>>(config: CreateJudgeConfig<TOptions>): Judge<TOptions>;
+declare function createJudge<TInput, TOutput extends JsonValue | undefined, TOptions extends object = Record<never, never>, THarness extends Harness<TInput, TOutput> | undefined = Harness<TInput, TOutput> | undefined>(name: string, assess: JudgeAssessFn<CreateJudgeContext<TInput, TOutput, TOptions, THarness>>): Judge<CreateJudgeContext<TInput, TOutput, TOptions, THarness>>;
+declare function createJudge<TInput, TOutput extends JsonValue | undefined, TOptions extends object = Record<never, never>, THarness extends Harness<TInput, TOutput> | undefined = Harness<TInput, TOutput> | undefined>(config: CreateJudgeConfig<CreateJudgeContext<TInput, TOutput, TOptions, THarness>>): Judge<CreateJudgeContext<TInput, TOutput, TOptions, THarness>>;
 /**
  * @deprecated Prefer `createJudge({ name, judgeHarness, assess })` and call
  * `ctx.runJudge(...)` from LLM-backed judges.
  */
-declare function createJudge<TOptions extends JudgeContext<any, any, any, any>, TInput, TOutput>(name: string, assessor: JudgeAssessor<TInput, TOutput>, assess: JudgeAssessWithAssessorFn<TOptions, TInput, TOutput>): Judge<TOptions>;
+declare function createJudge<TOptions extends JudgeContext<any, any, any>, TInput, TOutput>(name: string, assessor: JudgeAssessor<TInput, TOutput>, assess: JudgeAssessWithAssessorFn<TOptions, TInput, TOutput>): Judge<TOptions>;
 
-export { type DescribeEvalOptions, type EvalHarnessRun, type EvalMatchers, type EvalRun, type EvalRunOptions, type EvalTestAPI, type EvalTestContext, Harness, HarnessMetadata, Judge, type JudgeAssertionOptions, JudgeAssessFn, JudgeAssessWithAssessorFn, JudgeAssessor, JudgeContext, JudgeHarness, JudgeResult, type ToSatisfyJudge, createJudge, describeEval, formatScores };
+export { type DescribeEvalOptions, type EvalHarnessRun, type EvalMatchers, type EvalRun, type EvalTestAPI, type EvalTestContext, Harness, Judge, type JudgeAssertionOptions, JudgeAssessFn, JudgeAssessWithAssessorFn, JudgeAssessor, JudgeContext, JudgeHarness, JudgeOptions, JudgeResult, type ToSatisfyJudge, createJudge, describeEval, formatScores };

package/dist/index.js

@@ -131,7 +131,6 @@ function createHarness(options) {
       try {
         const result = await options.run({
           input,
-          metadata: context.metadata,
           signal: context.signal,
           artifacts: context.artifacts,
           setArtifact: context.setArtifact
@@ -522,17 +521,14 @@ function serializeError(error) {
 function createJudgeHarness(options) {
   return createHarness({
     name: options.name ?? "judge-harness",
-    run: async ({ input, signal, metadata }) => {
-      return normalizeJudgeHarnessResult(
-        await options.run(input, { signal, metadata })
-      );
+    run: async ({ input, signal }) => {
+      return normalizeJudgeHarnessResult(await options.run(input, { signal }));
     }
   });
 }
 async function runJudgeHarness(judgeHarness, input, options = {}) {
   const artifacts = {};
   const run = await judgeHarness.run(input, {
-    metadata: options.metadata ?? {},
     signal: options.signal,
     artifacts,
     setArtifact: (name, value) => {
@@ -546,8 +542,7 @@ function createRunJudge(judgeHarness, signal) {
     return void 0;
   }
   return (input, options) => runJudgeHarness(judgeHarness, input, {
-    metadata: options?.metadata,
-    signal
+    signal: options?.signal ?? signal
   });
 }
 function normalizeJudgeHarnessResult(result) {
@@ -625,22 +620,24 @@ function FactualityJudge(config = {}) {
   return {
     name: config.name ?? "FactualityJudge",
     judgeHarness,
-    assess: (opts) => assessFactuality(opts, judgeHarness)
+    assess: (opts) => assessFactuality(opts, {
+      expected: config.expected,
+      judgeHarness
+    })
   };
 }
-async function assessFactuality(opts, configuredJudgeHarness) {
-  const metadata = opts.metadata;
-  const expected = opts.expected === void 0 ? metadata.expected : opts.expected;
+async function assessFactuality(opts, config) {
+  const expected = opts.expected ?? config.expected;
   if (isMissingExpectedAnswer(expected)) {
     return {
       score: 0,
       metadata: {
-        rationale: "FactualityJudge requires a non-empty expert answer in `expected` or `metadata.expected`."
+        rationale: "FactualityJudge requires a non-empty expert answer in `expected` or FactualityJudge(...) config."
       }
     };
   }
   const runJudge = opts.runJudge ?? createRunJudge(
-    configuredJudgeHarness,
+    config.judgeHarness,
     opts.signal
   );
   if (!runJudge) {
@@ -1048,15 +1045,15 @@ function StructuredOutputScorer(config = {}) {
 
 // src/judges/structuredOutputJudge.ts
 function StructuredOutputJudge(config = {}) {
-  const scorer = StructuredOutputScorer(config);
+  const { expected, ...scorerConfig } = config;
+  const scorer = StructuredOutputScorer(scorerConfig);
   return {
     name: "StructuredOutputJudge",
     assess: (opts) => {
-      const metadata = opts.metadata;
       return scorer({
         ...opts,
         input: formatStructuredOutput(opts.input),
-        expected: opts.expected ?? metadata.expected,
+        expected: opts.expected ?? expected,
         output: formatStructuredOutput(opts.output)
       });
     }
@@ -1266,17 +1263,17 @@ function evaluateUnorderedTools(expected, actual, options) {
 
 // src/judges/toolCallJudge.ts
 function ToolCallJudge(config = {}) {
-  const scorer = ToolCallScorer(config);
+  const { expectedTools, ...scorerConfig } = config;
+  const scorer = ToolCallScorer(scorerConfig);
   return {
     name: "ToolCallJudge",
     assess: (opts) => {
-      const metadata = opts.metadata;
       return scorer({
         ...opts,
         input: formatJudgeValue2(opts.input),
         output: formatJudgeValue2(opts.output),
         expectedTools: normalizeExpectedTools(
-          opts.expectedTools ?? metadata.expectedTools
+          opts.expectedTools ?? expectedTools
         )
       });
     }
@@ -1307,10 +1304,7 @@ var evalTest = import_vitest.test.extend("harness", async () => {
   throw new Error(
     "describeEval must override the harness fixture before running tests."
   );
-}).extend(
-  "automaticJudges",
-  []
-).extend("judgeThreshold", void 0).extend("judgeHarness", void 0).extend("explicitJudgeHarness", void 0).extend(
+}).extend("automaticJudges", []).extend("judgeThreshold", void 0).extend("judgeHarness", void 0).extend("explicitJudgeHarness", void 0).extend(
   "run",
   async ({
     automaticJudges,
@@ -1321,12 +1315,10 @@ var evalTest = import_vitest.test.extend("harness", async () => {
     signal,
     task
   }) => {
-    return async (input, options) => {
+    return async (input) => {
       const resolvedHarness = harness;
-      const metadata = createMetadata(options?.metadata);
       const artifacts = {};
       const context = {
-        metadata,
         signal,
         artifacts,
         setArtifact: (artifactName, value) => {
@@ -1356,7 +1348,6 @@ var evalTest = import_vitest.test.extend("harness", async () => {
             resolvedHarness,
             input,
             explicitJudgeHarness,
-            metadata,
             signal
           );
         }
@@ -1375,7 +1366,6 @@ var evalTest = import_vitest.test.extend("harness", async () => {
             resolvedHarness,
             input,
             explicitJudgeHarness,
-            metadata,
             signal
           );
         }
@@ -1395,7 +1385,6 @@ var evalTest = import_vitest.test.extend("harness", async () => {
         resolvedHarness,
         input,
         explicitJudgeHarness,
-        metadata,
         signal
       );
       if (automaticJudges.length > 0) {
@@ -1406,7 +1395,6 @@ var evalTest = import_vitest.test.extend("harness", async () => {
           resolvedHarness,
           input,
           judgeHarness,
-          metadata,
           run,
           signal
         );
@@ -1473,10 +1461,7 @@ function describeEval(name, options, define) {
     define(it);
   });
 }
-function createMetadata(metadata) {
-  return { ...metadata ?? {} };
-}
-async function applyAutomaticJudges(task, judges, threshold, harness, input, judgeHarness, metadata, run, signal) {
+async function applyAutomaticJudges(task, judges, threshold, harness, input, judgeHarness, run, signal) {
   const runToolCalls = (0, import_core2.toolCalls)(run.session);
   const scores = await Promise.all(
     judges.map((judge) => {
@@ -1488,7 +1473,6 @@ async function applyAutomaticJudges(task, judges, threshold, harness, input, jud
         input,
         output: run.output,
         toolCalls: runToolCalls,
-        metadata,
         run,
         session: run.session,
         signal,
@@ -1533,12 +1517,11 @@ function setHarnessMeta(task, name, run) {
     run
   };
 }
-function recordJudgeRunContext(run, harness, input, judgeHarness, metadata, signal) {
+function recordJudgeRunContext(run, harness, input, judgeHarness, signal) {
   const context = {
     harness,
     input,
     judgeHarness,
-    metadata,
     run,
     signal
   };
@@ -1608,7 +1591,6 @@ function buildJudgeAssertionOptions(received, judge, options, task) {
   const judgeHarness = options.judgeHarness ?? resolveJudgeHarnessForJudge(judge, registeredContext?.judgeHarness);
   const runJudge = createRunJudge(judgeHarness, registeredContext?.signal);
   const signal = registeredContext?.signal;
-  const metadata = options.metadata ?? registeredContext?.metadata ?? {};
   const input = options.input ?? registeredContext?.input ?? void 0;
   const contextualOptions = {
     ...options,
@@ -1631,7 +1613,6 @@ function buildJudgeAssertionOptions(received, judge, options, task) {
     ...judgeParams,
     input: resolvedInput,
     output,
-    metadata,
     run,
     session: options.session ?? run.session,
     signal,