vitest-evals 0.9.0 → 0.10.0

package/dist/index.js

@@ -20,17 +20,21 @@ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: tru
 // src/index.ts
 var index_exports = {};
 __export(index_exports, {
+  FactualityJudge: () => FactualityJudge,
   StructuredOutputJudge: () => StructuredOutputJudge,
   ToolCallJudge: () => ToolCallJudge,
   assistantMessages: () => assistantMessages,
   attachHarnessRunToError: () => attachHarnessRunToError,
   createHarness: () => createHarness,
   createJudge: () => createJudge,
+  createJudgeHarness: () => createJudgeHarness,
   describeEval: () => describeEval,
   formatScores: () => formatScores,
   getHarnessRunFromError: () => getHarnessRunFromError,
+  latestAssistantMessageContent: () => latestAssistantMessageContent,
   messagesByRole: () => messagesByRole,
   normalizeHarnessRun: () => normalizeHarnessRun,
+  runJudgeHarness: () => runJudgeHarness,
   systemMessages: () => systemMessages,
   toolCalls: () => toolCalls,
   toolMessages: () => toolMessages,
@@ -223,6 +227,9 @@ function toolCalls(session) {
 function messagesByRole(session, role) {
   return session.messages.filter((message) => message.role === role);
 }
+function hasNonEmptyMessageContent(message) {
+  return message.content !== void 0 && (typeof message.content !== "string" || message.content.trim().length > 0);
+}
 function systemMessages(session) {
   return messagesByRole(session, "system");
 }
@@ -232,6 +239,9 @@ function userMessages(session) {
 function assistantMessages(session) {
   return messagesByRole(session, "assistant");
 }
+function latestAssistantMessageContent(session) {
+  return [...assistantMessages(session)].reverse().find(hasNonEmptyMessageContent)?.content;
+}
 function toolMessages(session) {
   return messagesByRole(session, "tool");
 }
@@ -270,6 +280,64 @@ function serializeError(error) {
   };
 }
 
+// src/judges/judgeHarness.ts
+function createJudgeHarness(options) {
+  return createHarness({
+    name: options.name ?? "judge-harness",
+    run: async ({ input, signal, metadata }) => {
+      return normalizeJudgeHarnessResult(
+        await options.run(input, { signal, metadata })
+      );
+    }
+  });
+}
+async function runJudgeHarness(judgeHarness, input, options = {}) {
+  const artifacts = {};
+  const run = await judgeHarness.run(input, {
+    metadata: options.metadata ?? {},
+    signal: options.signal,
+    artifacts,
+    setArtifact: (name, value) => {
+      artifacts[name] = value;
+    }
+  });
+  return run.output !== void 0 ? run.output : resolveJudgeHarnessAssistantOutput(run);
+}
+function createRunJudge(judgeHarness, signal) {
+  if (!judgeHarness) {
+    return void 0;
+  }
+  return (input, options) => runJudgeHarness(judgeHarness, input, {
+    metadata: options?.metadata,
+    signal
+  });
+}
+function normalizeJudgeHarnessResult(result) {
+  if (isHarnessRun(result)) {
+    return result;
+  }
+  if (hasOutputField(result)) {
+    return {
+      output: normalizeJudgeHarnessOutput(result.output)
+    };
+  }
+  return {
+    output: normalizeJudgeHarnessOutput(result)
+  };
+}
+function hasOutputField(value) {
+  return value !== null && typeof value === "object" && !Array.isArray(value) && Object.keys(value).length === 1 && "output" in value;
+}
+function normalizeJudgeHarnessOutput(value) {
+  if (value === void 0) {
+    return void 0;
+  }
+  return normalizeContent(value);
+}
+function resolveJudgeHarnessAssistantOutput(run) {
+  return latestAssistantMessageContent(run.session) ?? "";
+}
+
 // src/wrapText.ts
 function wrapText(text, width = 80) {
   if (!text || text.length <= width) {
@@ -292,6 +360,171 @@ function wrapText(text, width = 80) {
   return lines.join("\n");
 }
 
+// src/judges/factualityJudge.ts
+var FACTUALITY_CHOICE_SCORES = {
+  A: 0.4,
+  B: 0.6,
+  C: 1,
+  D: 0,
+  E: 1
+};
+var FACTUALITY_SYSTEM = "You are comparing factual content. Ignore differences in style, grammar, punctuation, and formatting.";
+var FACTUALITY_RESPONSE_SCHEMA = {
+  type: "object",
+  additionalProperties: false,
+  required: ["choice", "rationale"],
+  properties: {
+    choice: {
+      enum: ["A", "B", "C", "D", "E"]
+    },
+    rationale: {
+      type: "string"
+    }
+  }
+};
+function FactualityJudge(config = {}) {
+  const judgeHarness = config.judgeHarness;
+  return {
+    name: config.name ?? "FactualityJudge",
+    judgeHarness,
+    assess: (opts) => assessFactuality(opts, judgeHarness)
+  };
+}
+async function assessFactuality(opts, configuredJudgeHarness) {
+  const metadata = opts.metadata;
+  const expected = opts.expected === void 0 ? metadata.expected : opts.expected;
+  if (isMissingExpectedAnswer(expected)) {
+    return {
+      score: 0,
+      metadata: {
+        rationale: "FactualityJudge requires a non-empty expert answer in `expected` or `metadata.expected`."
+      }
+    };
+  }
+  const runJudge = opts.runJudge ?? createRunJudge(
+    configuredJudgeHarness,
+    opts.signal
+  );
+  if (!runJudge) {
+    throw new Error(
+      "FactualityJudge requires a judgeHarness in FactualityJudge(...) config, describeEval(...) options, toSatisfyJudge(...) options, or JudgeContext.runJudge."
+    );
+  }
+  const verdict = await runJudge({
+    system: FACTUALITY_SYSTEM,
+    prompt: formatFactualityPrompt({
+      input: opts.input,
+      expected,
+      output: resolveJudgeOutput(opts)
+    }),
+    responseFormat: {
+      type: "json",
+      schema: FACTUALITY_RESPONSE_SCHEMA
+    }
+  });
+  return formatJudgeResult(parseFactualityJudgeVerdict(verdict));
+}
+function isMissingExpectedAnswer(value) {
+  return value == null || typeof value === "string" && value.trim().length === 0;
+}
+function resolveJudgeOutput(opts) {
+  if (opts.output !== void 0) {
+    return opts.output;
+  }
+  return latestAssistantMessageContent(opts.session) ?? "";
+}
+function parseFactualityJudgeVerdict(value) {
+  const parsed = typeof value === "string" ? parseJsonObject(value) : value;
+  if (!parsed || typeof parsed !== "object") {
+    throw new Error(
+      "FactualityJudge judgeHarness must return an object with `choice` and `rationale`."
+    );
+  }
+  const verdict = parsed;
+  if (!isFactualityChoice(verdict.choice)) {
+    throw new Error(
+      "FactualityJudge judgeHarness must return choice A, B, C, D, or E."
+    );
+  }
+  if (typeof verdict.rationale !== "string") {
+    throw new Error(
+      "FactualityJudge judgeHarness must return a string `rationale`."
+    );
+  }
+  return {
+    choice: verdict.choice,
+    rationale: verdict.rationale
+  };
+}
+function parseJsonObject(value) {
+  try {
+    return JSON.parse(value);
+  } catch {
+    const fencedJson = value.match(/```(?:json)?\s*([\s\S]*?)\s*```/i);
+    if (!fencedJson) {
+      throw new Error(
+        "FactualityJudge judgeHarness must return JSON with `choice` and `rationale`."
+      );
+    }
+    return JSON.parse(fencedJson[1]);
+  }
+}
+function isFactualityChoice(value) {
+  return value === "A" || value === "B" || value === "C" || value === "D" || value === "E";
+}
+function formatFactualityPrompt({
+  input,
+  expected,
+  output
+}) {
+  const comparison = formatJudgeValue({
+    question: input ?? "",
+    expert_answer: expected,
+    submitted_answer: output ?? ""
+  });
+  return `Compare the submitted answer with the expert answer.
+
+Comparison payload:
+${comparison}
+
+Select exactly one option:
+A: The submission is a fully consistent subset of the expert answer.
+B: The submission is a fully consistent superset of the expert answer.
+C: The submission contains the same factual details as the expert answer.
+D: The submission disagrees with the expert answer.
+E: The answers differ only in ways that do not affect factuality.
+
+Return JSON with exactly these fields:
+{
+  "choice": "C",
+  "rationale": "Brief explanation for the selected choice"
+}
+
+The choice value must be one of A, B, C, D, or E.`;
+}
+function formatJudgeValue(value) {
+  if (typeof value === "string") {
+    return value;
+  }
+  if (value === void 0) {
+    return "";
+  }
+  try {
+    return JSON.stringify(value, null, 2) ?? String(value);
+  } catch {
+    return String(value);
+  }
+}
+function formatJudgeResult(object) {
+  return {
+    score: FACTUALITY_CHOICE_SCORES[object.choice],
+    metadata: {
+      rationale: object.rationale,
+      choice: object.choice
+    }
+  };
+}
+
 // src/internal/matchers.ts
 function isRecord(value) {
   return typeof value === "object" && value !== null && !Array.isArray(value);
@@ -669,8 +902,8 @@ function evaluateOrderedTools(expected, actual, options) {
           score: expectedIndex / expected.length,
           metadata: {
             rationale: `Tool '${expectedTool.name}' called with incorrect arguments at position ${expectedIndex + 1} (${expectedIndex}/${expected.length} tools matched correctly)`,
-            expected: expectedTool.arguments,
-            actual: actualTool.arguments,
+            expected: normalizeContent(expectedTool.arguments),
+            actual: actualTool.arguments === void 0 ? void 0 : normalizeContent(actualTool.arguments),
             matched: expectedIndex,
             total: expected.length
           }
@@ -802,8 +1035,8 @@ function ToolCallJudge(config = {}) {
       const metadata = opts.metadata;
       return scorer({
         ...opts,
-        input: formatJudgeValue(opts.input),
-        output: formatJudgeValue(opts.output),
+        input: formatJudgeValue2(opts.input),
+        output: formatJudgeValue2(opts.output),
         expectedTools: normalizeExpectedTools(
           opts.expectedTools ?? metadata.expectedTools
         )
@@ -816,7 +1049,7 @@ function normalizeExpectedTools(expectedTools) {
     (tool) => typeof tool === "string" ? { name: tool } : tool
   );
 }
-function formatJudgeValue(value) {
+function formatJudgeValue2(value) {
   if (typeof value === "string") {
     return value;
   }
@@ -839,9 +1072,17 @@ var evalTest = import_vitest.test.extend("harness", async () => {
 }).extend(
   "automaticJudges",
   []
-).extend("judgeThreshold", void 0).extend(
+).extend("judgeThreshold", void 0).extend("judgeHarness", void 0).extend("explicitJudgeHarness", void 0).extend(
   "run",
-  async ({ automaticJudges, harness, judgeThreshold, signal, task }) => {
+  async ({
+    automaticJudges,
+    explicitJudgeHarness,
+    harness,
+    judgeHarness,
+    judgeThreshold,
+    signal,
+    task
+  }) => {
     return async (input, options) => {
       const resolvedHarness = harness;
       const metadata = createMetadata(options?.metadata);
@@ -869,6 +1110,7 @@ var evalTest = import_vitest.test.extend("harness", async () => {
             partialRun,
             resolvedHarness,
             input,
+            explicitJudgeHarness,
             metadata,
             signal
           );
@@ -879,7 +1121,14 @@ var evalTest = import_vitest.test.extend("harness", async () => {
         run.artifacts = artifacts;
       }
       setHarnessMeta(task, resolvedHarness.name, run);
-      recordJudgeRunContext(run, resolvedHarness, input, metadata, signal);
+      recordJudgeRunContext(
+        run,
+        resolvedHarness,
+        input,
+        explicitJudgeHarness,
+        metadata,
+        signal
+      );
       if (automaticJudges.length > 0) {
         await applyAutomaticJudges(
           task,
@@ -887,6 +1136,7 @@ var evalTest = import_vitest.test.extend("harness", async () => {
           judgeThreshold,
           resolvedHarness,
           input,
+          judgeHarness,
           metadata,
           run,
           signal
@@ -901,6 +1151,7 @@ import_vitest.expect.extend({
     const { threshold = 1, ...context } = options ?? {};
     const judgeOptions = buildJudgeAssertionOptions(
       received,
+      judge,
       context,
       isEvalTaskLike(this.task) ? this.task : void 0
     );
@@ -941,9 +1192,13 @@ function formatJudgeOutputForMessage(output) {
 function describeEval(name, options, define) {
   const suite = options.skipIf ? import_vitest.describe.skipIf(options.skipIf()) : import_vitest.describe;
   return suite(name, () => {
+    const automaticJudges = options.judges ?? [];
+    const explicitJudgeHarness = options.judgeHarness ?? resolveDefaultJudgeHarness(automaticJudges);
     const it = evalTest.override({
       harness: options.harness,
-      automaticJudges: options.judges ?? [],
+      automaticJudges,
+      judgeHarness: options.judgeHarness,
+      explicitJudgeHarness,
       judgeThreshold: options.judgeThreshold
     });
     define(it);
@@ -952,10 +1207,14 @@ function describeEval(name, options, define) {
 function createMetadata(metadata) {
   return { ...metadata ?? {} };
 }
-async function applyAutomaticJudges(task, judges, threshold, harness, input, metadata, run, signal) {
+async function applyAutomaticJudges(task, judges, threshold, harness, input, judgeHarness, metadata, run, signal) {
   const runToolCalls = toolCalls(run.session);
   const scores = await Promise.all(
     judges.map((judge) => {
+      const runJudge = createRunJudge(
+        resolveJudgeHarnessForJudge(judge, judgeHarness),
+        signal
+      );
       const judgeOptions = {
         input,
         output: run.output,
@@ -964,7 +1223,8 @@ async function applyAutomaticJudges(task, judges, threshold, harness, input, met
         run,
         session: run.session,
         signal,
-        harness
+        harness,
+        runJudge
       };
       return Promise.resolve(judge.assess(judgeOptions));
     })
@@ -1004,10 +1264,11 @@ function setHarnessMeta(task, name, run) {
     run
   };
 }
-function recordJudgeRunContext(run, harness, input, metadata, signal) {
+function recordJudgeRunContext(run, harness, input, judgeHarness, metadata, signal) {
   const context = {
     harness,
     input,
+    judgeHarness,
     metadata,
     run,
     signal
@@ -1068,13 +1329,15 @@ function formatJudgeTextOutput(run) {
   }
   return typeof assistantOutput === "string" ? assistantOutput : JSON.stringify(assistantOutput);
 }
-function buildJudgeAssertionOptions(received, options, task) {
+function buildJudgeAssertionOptions(received, judge, options, task) {
   const registeredContext = resolveRegisteredJudgeRunContext(
     received,
     options,
     task
   );
   const harness = options.harness ?? registeredContext?.harness;
+  const judgeHarness = options.judgeHarness ?? resolveJudgeHarnessForJudge(judge, registeredContext?.judgeHarness);
+  const runJudge = createRunJudge(judgeHarness, registeredContext?.signal);
   const signal = registeredContext?.signal;
   const metadata = options.metadata ?? registeredContext?.metadata ?? {};
   const input = options.input ?? registeredContext?.input ?? void 0;
@@ -1094,8 +1357,9 @@ function buildJudgeAssertionOptions(received, options, task) {
     options.output
   );
   const resolvedToolCalls = options.toolCalls ?? toolCalls(run.session);
+  const { judgeHarness: _judgeHarness, ...judgeParams } = options;
   return {
-    ...options,
+    ...judgeParams,
     input: resolvedInput,
     output,
     metadata,
@@ -1103,9 +1367,23 @@ function buildJudgeAssertionOptions(received, options, task) {
     session: options.session ?? run.session,
     signal,
     toolCalls: resolvedToolCalls,
-    harness
+    harness,
+    runJudge
   };
 }
+function resolveJudgeHarnessForJudge(judge, fallback) {
+  return judge.judgeHarness ?? fallback;
+}
+function resolveDefaultJudgeHarness(judges) {
+  const configured = judges.map((judge) => judge.judgeHarness).filter(
+    (judgeHarness) => Boolean(judgeHarness)
+  );
+  const [first] = configured;
+  if (!first || configured.some((judgeHarness) => judgeHarness !== first)) {
+    return void 0;
+  }
+  return first;
+}
 function resolveRegisteredJudgeRunContext(received, options, task) {
   if (options.run) {
     return getRegisteredJudgeRunContext(options.run);
@@ -1195,11 +1473,8 @@ function inferJudgeOutputValue(received, session) {
   return normalizeJudgeJsonValue(received);
 }
 function resolveAssistantOutput(session) {
-  const assistantContent = [...assistantMessages(session)].reverse().find(hasAssistantOutputContent);
-  return assistantContent?.content !== void 0 ? normalizeContent(assistantContent.content) : void 0;
-}
-function hasAssistantOutputContent(message) {
-  return message.content !== void 0 && (typeof message.content !== "string" || message.content.trim().length > 0);
+  const assistantContent = latestAssistantMessageContent(session);
+  return assistantContent !== void 0 ? normalizeContent(assistantContent) : void 0;
 }
 function normalizeJudgeJsonValue(value) {
   if (value === void 0) {
@@ -1228,7 +1503,15 @@ reason  ${wrapText(s.metadata.rationale)}` : ""}${formattedOutput}`;
     return scoreLine;
   }).join("\n\n");
 }
-function createJudge(name, assessOrAssessor, assess) {
+function createJudge(nameOrConfig, assessOrAssessor, assess) {
+  if (typeof nameOrConfig !== "string") {
+    return {
+      name: nameOrConfig.name,
+      judgeHarness: nameOrConfig.judgeHarness,
+      assess: nameOrConfig.assess
+    };
+  }
+  const name = nameOrConfig;
   if (!assess) {
     return {
       name,
@@ -1249,17 +1532,21 @@ function createJudge(name, assessOrAssessor, assess) {
 }
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
+  FactualityJudge,
   StructuredOutputJudge,
   ToolCallJudge,
   assistantMessages,
   attachHarnessRunToError,
   createHarness,
   createJudge,
+  createJudgeHarness,
   describeEval,
   formatScores,
   getHarnessRunFromError,
+  latestAssistantMessageContent,
   messagesByRole,
   normalizeHarnessRun,
+  runJudgeHarness,
   systemMessages,
   toolCalls,
   toolMessages,