vitest-evals 0.12.0 → 0.13.0

package/dist/index.mjs

@@ -92,7 +92,6 @@ function createHarness(options) {
       try {
         const result = await options.run({
           input,
-          metadata: context.metadata,
           signal: context.signal,
           artifacts: context.artifacts,
           setArtifact: context.setArtifact
@@ -483,17 +482,14 @@ function serializeError(error) {
 function createJudgeHarness(options) {
   return createHarness({
     name: options.name ?? "judge-harness",
-    run: async ({ input, signal, metadata }) => {
-      return normalizeJudgeHarnessResult(
-        await options.run(input, { signal, metadata })
-      );
+    run: async ({ input, signal }) => {
+      return normalizeJudgeHarnessResult(await options.run(input, { signal }));
     }
   });
 }
 async function runJudgeHarness(judgeHarness, input, options = {}) {
   const artifacts = {};
   const run = await judgeHarness.run(input, {
-    metadata: options.metadata ?? {},
     signal: options.signal,
     artifacts,
     setArtifact: (name, value) => {
@@ -507,8 +503,7 @@ function createRunJudge(judgeHarness, signal) {
     return void 0;
   }
   return (input, options) => runJudgeHarness(judgeHarness, input, {
-    metadata: options?.metadata,
-    signal
+    signal: options?.signal ?? signal
   });
 }
 function normalizeJudgeHarnessResult(result) {
@@ -586,22 +581,24 @@ function FactualityJudge(config = {}) {
   return {
     name: config.name ?? "FactualityJudge",
     judgeHarness,
-    assess: (opts) => assessFactuality(opts, judgeHarness)
+    assess: (opts) => assessFactuality(opts, {
+      expected: config.expected,
+      judgeHarness
+    })
   };
 }
-async function assessFactuality(opts, configuredJudgeHarness) {
-  const metadata = opts.metadata;
-  const expected = opts.expected === void 0 ? metadata.expected : opts.expected;
+async function assessFactuality(opts, config) {
+  const expected = opts.expected ?? config.expected;
   if (isMissingExpectedAnswer(expected)) {
     return {
       score: 0,
       metadata: {
-        rationale: "FactualityJudge requires a non-empty expert answer in `expected` or `metadata.expected`."
+        rationale: "FactualityJudge requires a non-empty expert answer in `expected` or FactualityJudge(...) config."
       }
     };
   }
   const runJudge = opts.runJudge ?? createRunJudge(
-    configuredJudgeHarness,
+    config.judgeHarness,
     opts.signal
   );
   if (!runJudge) {
@@ -1009,15 +1006,15 @@ function StructuredOutputScorer(config = {}) {
 
 // src/judges/structuredOutputJudge.ts
 function StructuredOutputJudge(config = {}) {
-  const scorer = StructuredOutputScorer(config);
+  const { expected, ...scorerConfig } = config;
+  const scorer = StructuredOutputScorer(scorerConfig);
   return {
     name: "StructuredOutputJudge",
     assess: (opts) => {
-      const metadata = opts.metadata;
       return scorer({
         ...opts,
         input: formatStructuredOutput(opts.input),
-        expected: opts.expected ?? metadata.expected,
+        expected: opts.expected ?? expected,
         output: formatStructuredOutput(opts.output)
       });
     }
@@ -1227,17 +1224,17 @@ function evaluateUnorderedTools(expected, actual, options) {
 
 // src/judges/toolCallJudge.ts
 function ToolCallJudge(config = {}) {
-  const scorer = ToolCallScorer(config);
+  const { expectedTools, ...scorerConfig } = config;
+  const scorer = ToolCallScorer(scorerConfig);
   return {
     name: "ToolCallJudge",
     assess: (opts) => {
-      const metadata = opts.metadata;
       return scorer({
         ...opts,
         input: formatJudgeValue2(opts.input),
         output: formatJudgeValue2(opts.output),
         expectedTools: normalizeExpectedTools(
-          opts.expectedTools ?? metadata.expectedTools
+          opts.expectedTools ?? expectedTools
         )
       });
     }
@@ -1268,10 +1265,7 @@ var evalTest = test.extend("harness", async () => {
   throw new Error(
     "describeEval must override the harness fixture before running tests."
   );
-}).extend(
-  "automaticJudges",
-  []
-).extend("judgeThreshold", void 0).extend("judgeHarness", void 0).extend("explicitJudgeHarness", void 0).extend(
+}).extend("automaticJudges", []).extend("judgeThreshold", void 0).extend("judgeHarness", void 0).extend("explicitJudgeHarness", void 0).extend(
   "run",
   async ({
     automaticJudges,
@@ -1282,12 +1276,10 @@ var evalTest = test.extend("harness", async () => {
     signal,
     task
   }) => {
-    return async (input, options) => {
+    return async (input) => {
       const resolvedHarness = harness;
-      const metadata = createMetadata(options?.metadata);
       const artifacts = {};
       const context = {
-        metadata,
         signal,
         artifacts,
         setArtifact: (artifactName, value) => {
@@ -1317,7 +1309,6 @@ var evalTest = test.extend("harness", async () => {
             resolvedHarness,
             input,
             explicitJudgeHarness,
-            metadata,
             signal
           );
         }
@@ -1336,7 +1327,6 @@ var evalTest = test.extend("harness", async () => {
             resolvedHarness,
             input,
             explicitJudgeHarness,
-            metadata,
             signal
           );
         }
@@ -1356,7 +1346,6 @@ var evalTest = test.extend("harness", async () => {
         resolvedHarness,
         input,
         explicitJudgeHarness,
-        metadata,
         signal
       );
       if (automaticJudges.length > 0) {
@@ -1367,7 +1356,6 @@ var evalTest = test.extend("harness", async () => {
           resolvedHarness,
           input,
           judgeHarness,
-          metadata,
           run,
           signal
         );
@@ -1434,10 +1422,7 @@ function describeEval(name, options, define) {
     define(it);
   });
 }
-function createMetadata(metadata) {
-  return { ...metadata ?? {} };
-}
-async function applyAutomaticJudges(task, judges, threshold, harness, input, judgeHarness, metadata, run, signal) {
+async function applyAutomaticJudges(task, judges, threshold, harness, input, judgeHarness, run, signal) {
   const runToolCalls = toolCalls2(run.session);
   const scores = await Promise.all(
     judges.map((judge) => {
@@ -1449,7 +1434,6 @@ async function applyAutomaticJudges(task, judges, threshold, harness, input, jud
         input,
         output: run.output,
         toolCalls: runToolCalls,
-        metadata,
         run,
         session: run.session,
         signal,
@@ -1494,12 +1478,11 @@ function setHarnessMeta(task, name, run) {
     run
   };
 }
-function recordJudgeRunContext(run, harness, input, judgeHarness, metadata, signal) {
+function recordJudgeRunContext(run, harness, input, judgeHarness, signal) {
   const context = {
     harness,
     input,
     judgeHarness,
-    metadata,
     run,
     signal
   };
@@ -1569,7 +1552,6 @@ function buildJudgeAssertionOptions(received, judge, options, task) {
   const judgeHarness = options.judgeHarness ?? resolveJudgeHarnessForJudge(judge, registeredContext?.judgeHarness);
   const runJudge = createRunJudge(judgeHarness, registeredContext?.signal);
   const signal = registeredContext?.signal;
-  const metadata = options.metadata ?? registeredContext?.metadata ?? {};
   const input = options.input ?? registeredContext?.input ?? void 0;
   const contextualOptions = {
     ...options,
@@ -1592,7 +1574,6 @@ function buildJudgeAssertionOptions(received, judge, options, task) {
     ...judgeParams,
     input: resolvedInput,
     output,
-    metadata,
     run,
     session: options.session ?? run.session,
     signal,