vitest-evals 0.9.0 → 0.10.0

package/dist/judges/index.js

@@ -20,11 +20,448 @@ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: tru
 // src/judges/index.ts
 var judges_exports = {};
 __export(judges_exports, {
+  FactualityJudge: () => FactualityJudge,
   StructuredOutputJudge: () => StructuredOutputJudge,
-  ToolCallJudge: () => ToolCallJudge
+  ToolCallJudge: () => ToolCallJudge,
+  createJudgeHarness: () => createJudgeHarness,
+  runJudgeHarness: () => runJudgeHarness
 });
 module.exports = __toCommonJS(judges_exports);
 
+// src/harness.ts
+function isJsonPrimitive(value) {
+  return value === null || typeof value === "string" || typeof value === "number" || typeof value === "boolean";
+}
+function isJsonRecord(value) {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+function normalizeJsonArray(value) {
+  return value.map((item) => {
+    const normalized = toJsonValue(item);
+    return normalized === void 0 ? null : normalized;
+  });
+}
+function normalizeJsonObject(value) {
+  const normalized = {};
+  for (const [key, entryValue] of Object.entries(value)) {
+    const entry = toJsonValue(entryValue);
+    if (entry !== void 0) {
+      normalized[key] = entry;
+    }
+  }
+  return normalized;
+}
+function toJsonValue(value) {
+  if (isJsonPrimitive(value)) {
+    return value;
+  }
+  if (Array.isArray(value)) {
+    return normalizeJsonArray(value);
+  }
+  if (isJsonRecord(value)) {
+    return normalizeJsonObject(value);
+  }
+  return void 0;
+}
+function normalizeRecord(value) {
+  return normalizeJsonObject(value);
+}
+function normalizeMetadata(value) {
+  const normalized = normalizeRecord(value);
+  return Object.keys(normalized).length > 0 ? normalized : void 0;
+}
+function normalizeContent(value) {
+  const normalized = toJsonValue(value);
+  return normalized !== void 0 ? normalized : String(value);
+}
+function createHarness(options) {
+  const harness = {
+    name: options.name,
+    run: async (input, context) => {
+      const result = await options.run({
+        input,
+        metadata: context.metadata,
+        signal: context.signal,
+        artifacts: context.artifacts,
+        setArtifact: context.setArtifact
+      });
+      return normalizeHarnessRun(input, result, context);
+    }
+  };
+  return harness;
+}
+function normalizeHarnessRun(input, result, context) {
+  if (isHarnessRun(result)) {
+    if (context && Object.keys(context.artifacts).length > 0 && !result.artifacts) {
+      return {
+        ...result,
+        artifacts: context.artifacts
+      };
+    }
+    return result;
+  }
+  const output = result.output;
+  const toolCalls = normalizeSimpleToolCalls(result.toolCalls);
+  const usage = result.usage ?? {};
+  const messages = result.messages ?? createDefaultSessionMessages({
+    input,
+    output,
+    toolCalls
+  });
+  const metadata = result.metadata ? normalizeMetadata(result.metadata) : void 0;
+  const artifacts = normalizeMergedArtifacts(
+    context?.artifacts,
+    result.artifacts
+  );
+  return {
+    session: {
+      messages,
+      ...usage.provider ? { provider: usage.provider } : {},
+      ...usage.model ? { model: usage.model } : {},
+      ...metadata ? { metadata } : {}
+    },
+    ...output !== void 0 ? { output } : {},
+    usage,
+    ...result.timings ? { timings: result.timings } : {},
+    ...artifacts ? { artifacts } : {},
+    errors: normalizeSimpleErrors(result.errors)
+  };
+}
+function createDefaultSessionMessages({
+  input,
+  output,
+  toolCalls: normalizedToolCalls
+}) {
+  const messages = [
+    {
+      role: "user",
+      content: normalizeContent(input)
+    }
+  ];
+  if (output !== void 0 || normalizedToolCalls.length > 0) {
+    messages.push({
+      role: "assistant",
+      ...output !== void 0 ? { content: normalizeContent(output) } : {},
+      ...normalizedToolCalls.length > 0 ? { toolCalls: normalizedToolCalls } : {}
+    });
+  }
+  return messages;
+}
+function normalizeSimpleToolCalls(calls) {
+  return (calls ?? []).map((call) => {
+    const {
+      arguments: rawArguments,
+      result: rawResult,
+      error: rawError,
+      metadata: rawMetadata,
+      ...toolCall
+    } = call;
+    const args = normalizeToolCallArguments(rawArguments);
+    const result = toJsonValue(rawResult);
+    const error = normalizeToolCallError(rawError);
+    const metadata = rawMetadata ? normalizeMetadata(rawMetadata) : void 0;
+    return {
+      ...toolCall,
+      ...args ? { arguments: args } : {},
+      ...result !== void 0 ? { result } : {},
+      ...error ? { error } : {},
+      ...metadata ? { metadata } : {}
+    };
+  });
+}
+function normalizeToolCallArguments(value) {
+  if (value === void 0) {
+    return void 0;
+  }
+  const normalized = toJsonValue(value);
+  return normalized && typeof normalized === "object" && !Array.isArray(normalized) ? normalized : void 0;
+}
+function normalizeToolCallError(value) {
+  if (value === void 0) {
+    return void 0;
+  }
+  const serialized = serializeError(value);
+  const { message, type, ...details } = serialized;
+  return {
+    ...details,
+    message: typeof message === "string" ? message : String(message),
+    ...typeof type === "string" ? { type } : {}
+  };
+}
+function normalizeMergedArtifacts(contextArtifacts, resultArtifacts) {
+  const artifacts = {
+    ...contextArtifacts ?? {},
+    ...resultArtifacts ? normalizeRecord(resultArtifacts) : {}
+  };
+  return Object.keys(artifacts).length > 0 ? artifacts : void 0;
+}
+function normalizeSimpleErrors(errors) {
+  return (errors ?? []).map((error) => {
+    const normalized = toJsonValue(error);
+    if (normalized && typeof normalized === "object" && !Array.isArray(normalized) && Object.keys(normalized).length > 0) {
+      return normalized;
+    }
+    return serializeError(error);
+  });
+}
+function messagesByRole(session, role) {
+  return session.messages.filter((message) => message.role === role);
+}
+function hasNonEmptyMessageContent(message) {
+  return message.content !== void 0 && (typeof message.content !== "string" || message.content.trim().length > 0);
+}
+function assistantMessages(session) {
+  return messagesByRole(session, "assistant");
+}
+function latestAssistantMessageContent(session) {
+  return [...assistantMessages(session)].reverse().find(hasNonEmptyMessageContent)?.content;
+}
+function isHarnessRun(value) {
+  if (!value || typeof value !== "object") {
+    return false;
+  }
+  const candidate = value;
+  return isNormalizedSession(candidate.session) && Boolean(candidate.usage) && typeof candidate.usage === "object" && !Array.isArray(candidate.usage) && Array.isArray(candidate.errors);
+}
+function isNormalizedSession(value) {
+  return Boolean(value) && typeof value === "object" && value !== null && "messages" in value && Array.isArray(value.messages);
+}
+function serializeError(error) {
+  if (error instanceof Error) {
+    return {
+      type: error.name,
+      message: error.message
+    };
+  }
+  return {
+    type: "Error",
+    message: String(error)
+  };
+}
+
+// src/judges/judgeHarness.ts
+function createJudgeHarness(options) {
+  return createHarness({
+    name: options.name ?? "judge-harness",
+    run: async ({ input, signal, metadata }) => {
+      return normalizeJudgeHarnessResult(
+        await options.run(input, { signal, metadata })
+      );
+    }
+  });
+}
+async function runJudgeHarness(judgeHarness, input, options = {}) {
+  const artifacts = {};
+  const run = await judgeHarness.run(input, {
+    metadata: options.metadata ?? {},
+    signal: options.signal,
+    artifacts,
+    setArtifact: (name, value) => {
+      artifacts[name] = value;
+    }
+  });
+  return run.output !== void 0 ? run.output : resolveJudgeHarnessAssistantOutput(run);
+}
+function createRunJudge(judgeHarness, signal) {
+  if (!judgeHarness) {
+    return void 0;
+  }
+  return (input, options) => runJudgeHarness(judgeHarness, input, {
+    metadata: options?.metadata,
+    signal
+  });
+}
+function normalizeJudgeHarnessResult(result) {
+  if (isHarnessRun(result)) {
+    return result;
+  }
+  if (hasOutputField(result)) {
+    return {
+      output: normalizeJudgeHarnessOutput(result.output)
+    };
+  }
+  return {
+    output: normalizeJudgeHarnessOutput(result)
+  };
+}
+function hasOutputField(value) {
+  return value !== null && typeof value === "object" && !Array.isArray(value) && Object.keys(value).length === 1 && "output" in value;
+}
+function normalizeJudgeHarnessOutput(value) {
+  if (value === void 0) {
+    return void 0;
+  }
+  return normalizeContent(value);
+}
+function resolveJudgeHarnessAssistantOutput(run) {
+  return latestAssistantMessageContent(run.session) ?? "";
+}
+
+// src/judges/factualityJudge.ts
+var FACTUALITY_CHOICE_SCORES = {
+  A: 0.4,
+  B: 0.6,
+  C: 1,
+  D: 0,
+  E: 1
+};
+var FACTUALITY_SYSTEM = "You are comparing factual content. Ignore differences in style, grammar, punctuation, and formatting.";
+var FACTUALITY_RESPONSE_SCHEMA = {
+  type: "object",
+  additionalProperties: false,
+  required: ["choice", "rationale"],
+  properties: {
+    choice: {
+      enum: ["A", "B", "C", "D", "E"]
+    },
+    rationale: {
+      type: "string"
+    }
+  }
+};
+function FactualityJudge(config = {}) {
+  const judgeHarness = config.judgeHarness;
+  return {
+    name: config.name ?? "FactualityJudge",
+    judgeHarness,
+    assess: (opts) => assessFactuality(opts, judgeHarness)
+  };
+}
+async function assessFactuality(opts, configuredJudgeHarness) {
+  const metadata = opts.metadata;
+  const expected = opts.expected === void 0 ? metadata.expected : opts.expected;
+  if (isMissingExpectedAnswer(expected)) {
+    return {
+      score: 0,
+      metadata: {
+        rationale: "FactualityJudge requires a non-empty expert answer in `expected` or `metadata.expected`."
+      }
+    };
+  }
+  const runJudge = opts.runJudge ?? createRunJudge(
+    configuredJudgeHarness,
+    opts.signal
+  );
+  if (!runJudge) {
+    throw new Error(
+      "FactualityJudge requires a judgeHarness in FactualityJudge(...) config, describeEval(...) options, toSatisfyJudge(...) options, or JudgeContext.runJudge."
+    );
+  }
+  const verdict = await runJudge({
+    system: FACTUALITY_SYSTEM,
+    prompt: formatFactualityPrompt({
+      input: opts.input,
+      expected,
+      output: resolveJudgeOutput(opts)
+    }),
+    responseFormat: {
+      type: "json",
+      schema: FACTUALITY_RESPONSE_SCHEMA
+    }
+  });
+  return formatJudgeResult(parseFactualityJudgeVerdict(verdict));
+}
+function isMissingExpectedAnswer(value) {
+  return value == null || typeof value === "string" && value.trim().length === 0;
+}
+function resolveJudgeOutput(opts) {
+  if (opts.output !== void 0) {
+    return opts.output;
+  }
+  return latestAssistantMessageContent(opts.session) ?? "";
+}
+function parseFactualityJudgeVerdict(value) {
+  const parsed = typeof value === "string" ? parseJsonObject(value) : value;
+  if (!parsed || typeof parsed !== "object") {
+    throw new Error(
+      "FactualityJudge judgeHarness must return an object with `choice` and `rationale`."
+    );
+  }
+  const verdict = parsed;
+  if (!isFactualityChoice(verdict.choice)) {
+    throw new Error(
+      "FactualityJudge judgeHarness must return choice A, B, C, D, or E."
+    );
+  }
+  if (typeof verdict.rationale !== "string") {
+    throw new Error(
+      "FactualityJudge judgeHarness must return a string `rationale`."
+    );
+  }
+  return {
+    choice: verdict.choice,
+    rationale: verdict.rationale
+  };
+}
+function parseJsonObject(value) {
+  try {
+    return JSON.parse(value);
+  } catch {
+    const fencedJson = value.match(/```(?:json)?\s*([\s\S]*?)\s*```/i);
+    if (!fencedJson) {
+      throw new Error(
+        "FactualityJudge judgeHarness must return JSON with `choice` and `rationale`."
+      );
+    }
+    return JSON.parse(fencedJson[1]);
+  }
+}
+function isFactualityChoice(value) {
+  return value === "A" || value === "B" || value === "C" || value === "D" || value === "E";
+}
+function formatFactualityPrompt({
+  input,
+  expected,
+  output
+}) {
+  const comparison = formatJudgeValue({
+    question: input ?? "",
+    expert_answer: expected,
+    submitted_answer: output ?? ""
+  });
+  return `Compare the submitted answer with the expert answer.
+
+Comparison payload:
+${comparison}
+
+Select exactly one option:
+A: The submission is a fully consistent subset of the expert answer.
+B: The submission is a fully consistent superset of the expert answer.
+C: The submission contains the same factual details as the expert answer.
+D: The submission disagrees with the expert answer.
+E: The answers differ only in ways that do not affect factuality.
+
+Return JSON with exactly these fields:
+{
+  "choice": "C",
+  "rationale": "Brief explanation for the selected choice"
+}
+
+The choice value must be one of A, B, C, D, or E.`;
+}
+function formatJudgeValue(value) {
+  if (typeof value === "string") {
+    return value;
+  }
+  if (value === void 0) {
+    return "";
+  }
+  try {
+    return JSON.stringify(value, null, 2) ?? String(value);
+  } catch {
+    return String(value);
+  }
+}
+function formatJudgeResult(object) {
+  return {
+    score: FACTUALITY_CHOICE_SCORES[object.choice],
+    metadata: {
+      rationale: object.rationale,
+      choice: object.choice
+    }
+  };
+}
+
 // src/internal/matchers.ts
 function isRecord(value) {
   return typeof value === "object" && value !== null && !Array.isArray(value);
@@ -402,8 +839,8 @@ function evaluateOrderedTools(expected, actual, options) {
           score: expectedIndex / expected.length,
           metadata: {
             rationale: `Tool '${expectedTool.name}' called with incorrect arguments at position ${expectedIndex + 1} (${expectedIndex}/${expected.length} tools matched correctly)`,
-            expected: expectedTool.arguments,
-            actual: actualTool.arguments,
+            expected: normalizeContent(expectedTool.arguments),
+            actual: actualTool.arguments === void 0 ? void 0 : normalizeContent(actualTool.arguments),
             matched: expectedIndex,
             total: expected.length
           }
@@ -535,8 +972,8 @@ function ToolCallJudge(config = {}) {
       const metadata = opts.metadata;
       return scorer({
         ...opts,
-        input: formatJudgeValue(opts.input),
-        output: formatJudgeValue(opts.output),
+        input: formatJudgeValue2(opts.input),
+        output: formatJudgeValue2(opts.output),
         expectedTools: normalizeExpectedTools(
           opts.expectedTools ?? metadata.expectedTools
         )
@@ -549,7 +986,7 @@ function normalizeExpectedTools(expectedTools) {
     (tool) => typeof tool === "string" ? { name: tool } : tool
   );
 }
-function formatJudgeValue(value) {
+function formatJudgeValue2(value) {
   if (typeof value === "string") {
     return value;
   }
@@ -564,7 +1001,10 @@ function formatJudgeValue(value) {
 }
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
+  FactualityJudge,
   StructuredOutputJudge,
-  ToolCallJudge
+  ToolCallJudge,
+  createJudgeHarness,
+  runJudgeHarness
 });
 //# sourceMappingURL=index.js.map