vitest-evals 0.5.0 → 0.7.0

package/dist/index.mjs.map

@@ -1 +1 @@
-{"version":3,"sources":["../src/index.ts","../src/scorers/utils.ts","../src/scorers/toolCallScorer.ts","../src/scorers/structuredOutputScorer.ts"],"sourcesContent":["import { assert, describe, expect, test } from \"vitest\";\nimport \"vitest\";\n\n/**\n * Represents a tool/function call made during task execution.\n * Supports various LLM provider formats and use cases.\n */\nexport type ToolCall = {\n  // Core fields (required for basic usage)\n  name: string;\n  arguments?: Record<string, any>;\n\n  // Additional metadata\n  [key: string]: any; // Allow provider-specific fields\n};\n\nexport type TaskResult = {\n  result: string;\n  toolCalls?: ToolCall[];\n};\n\n/**\n * Task function that processes an input and returns either a string result\n * or a TaskResult object containing the result and any tool calls made.\n *\n * @param input - The input string to process\n * @returns Promise resolving to either a string or TaskResult object\n *\n * @example\n * // Simple tasks can just return a string\n * const simpleTask: TaskFn = async (input) => \"The answer is 42\";\n *\n * // Tasks that use tools should return TaskResult\n * const taskWithTools: TaskFn = async (input) => ({\n *   result: \"The answer is 42\",\n *   toolCalls: [{ name: \"calculate\", arguments: { expr: \"6*7\" }, result: 42 }]\n * });\n */\nexport type TaskFn = (input: string) => Promise<string | TaskResult>;\n\nexport type Score = {\n  score: number | null;\n  metadata?: {\n    rationale?: string;\n    output?: any;\n  } & Record<string, any>;\n};\n\nexport interface BaseScorerOptions {\n  input: string;\n  output: string;\n  toolCalls?: ToolCall[];\n}\n\nexport type ScoreFn<TOptions extends BaseScorerOptions = BaseScorerOptions> = (\n  opts: TOptions,\n) => Promise<Score> | Score;\n\n/**\n * @deprecated Use describeEval() instead for better test organization and multiple scorers support\n */\nexport type ToEval<R = unknown> = (\n  expected: any,\n  taskFn: TaskFn,\n  scoreFn: ScoreFn<any>,\n  threshold?: number,\n) => Promise<R>;\n\nexport interface EvalMatchers<R = unknown> {\n  toEval: ToEval<R>;\n}\n\ndeclare module \"vitest\" {\n  interface Assertion<T = any> extends EvalMatchers<T> {}\n  interface AsymmetricMatchersContaining extends EvalMatchers {}\n\n  interface TaskMeta {\n    eval?: {\n      scores: (Score & { name: string })[];\n      avgScore: number;\n      toolCalls?: ToolCall[];\n    };\n  }\n}\n\nexpect.extend({\n  /**\n   * Evaluates a language model output against an expected answer using a scoring function.\n   *\n   * @deprecated Use describeEval() instead for better test organization and multiple scorers support\n   * @param expected - The expected (ground truth) answer, can be any type depending on the scorer\n   * @param taskFn - Async function that processes the input and returns the model output\n   *                 Can return either a string or TaskResult object with result and optional toolCalls\n   * @param scoreFn - Function that evaluates the model output against the expected answer\n   * @param threshold - Minimum acceptable score (0-1), defaults to 1.0\n   *\n   * @example\n   * ```javascript\n   * test(\"checks capital of France\", async () => {\n   *   expect(\"What is the capital of France?\").toEval(\n   *     \"Paris\",\n   *     async (input) => {\n   *       const response = await queryLLM(input);\n   *       // Recommended: return TaskResult\n   *       return {\n   *         result: response.text,\n   *         toolCalls: response.toolCalls || []\n   *       };\n   *     },\n   *     checkFactuality,\n   *     0.8\n   *   );\n   * });\n   * ```\n   */\n  // TODO: this needs to be support true extensibility with Eval scorers\n  toEval: async function toEval(\n    input: string,\n    expected: any,\n    taskFn: TaskFn,\n    scoreFn: ScoreFn<any>,\n    threshold = 1.0,\n  ) {\n    const { isNot } = this;\n\n    const taskOutput = await taskFn(input);\n    const output =\n      typeof taskOutput === \"string\" ? taskOutput : taskOutput.result;\n    const toolCalls =\n      typeof taskOutput === \"object\" ? taskOutput.toolCalls : undefined;\n\n    let result = scoreFn({ input, expected, output, toolCalls });\n    if (result instanceof Promise) {\n      result = await result;\n    }\n\n    return {\n      pass: (result.score ?? 0) >= threshold,\n      message: () => formatScores([{ ...result, name: scoreFn.name }]),\n    };\n  },\n});\n\n/**\n * Creates a test suite for evaluating language model outputs.\n *\n * @param name - The name of the test suite\n * @param options - Configuration options\n * @param options.data - Async function that returns an array of test cases with input and any additional fields\n * @param options.task - Function that processes the input and returns the model output\n *                       Can return either a string or TaskResult object with result and optional toolCalls\n * @param options.skipIf - Optional function that determines if tests should be skipped\n * @param options.scorers - Array of scoring functions that evaluate model outputs\n * @param options.threshold - Minimum acceptable average score (0-1), defaults to 1.0\n * @param options.timeout - Test timeout in milliseconds, defaults to 60000 (60s)\n *\n * @example\n * ```javascript\n * // Recommended: TaskResult format with tool tracking\n * describeEval(\"capital cities test\", {\n *   data: async () => [{\n *     input: \"What is the capital of France?\",\n *     expected: \"Paris\"\n *   }],\n *   task: async (input) => {\n *     const response = await queryLLM(input);\n *     return {\n *       result: response.text,\n *       toolCalls: response.toolCalls || []\n *     };\n *   },\n *   scorers: [checkFactuality],\n *   threshold: 0.8\n * });\n *\n * // Example with tool usage evaluation\n * describeEval(\"tool usage test\", {\n *   data: async () => [{\n *     input: \"Search for weather in Seattle\",\n *     expectedTools: [{ name: \"weather_api\", arguments: { location: \"Seattle\" } }]\n *   }],\n *   task: async (input) => {\n *     return {\n *       result: \"The weather in Seattle is 65°F\",\n *       toolCalls: [{\n *         name: \"weather_api\",\n *         arguments: { location: \"Seattle\" },\n *         result: { temp: 65, condition: \"partly cloudy\" }\n *       }]\n *     };\n *   },\n *   scorers: [ToolCallScorer()],\n *   threshold: 1.0\n * });\n * ```\n */\nexport function describeEval(\n  name: string,\n  {\n    data,\n    task,\n    skipIf,\n    scorers,\n    threshold = 1.0,\n    // increase default test timeout as 5s is usually not enough for\n    // a single factuality check\n    timeout = 60000,\n  }: {\n    data: () => Promise<Array<{ input: string } & Record<string, any>>>;\n    task: TaskFn;\n    skipIf?: () => boolean;\n    scorers: ScoreFn<any>[];\n    threshold?: number | null;\n    timeout?: number;\n  },\n) {\n  return describe(name, async () => {\n    const testFn = skipIf ? test.skipIf(skipIf()) : test;\n    // TODO: should data just be a generator?\n    for (const { input, ...params } of await data()) {\n      testFn(\n        input,\n        {\n          timeout,\n        },\n        async ({ task: testTask }) => {\n          const taskOutput = await task(input);\n          const output =\n            typeof taskOutput === \"string\" ? taskOutput : taskOutput.result;\n          const toolCalls =\n            typeof taskOutput === \"object\" ? taskOutput.toolCalls : undefined;\n\n          const scores = await Promise.all(\n            scorers.map((scorer) => {\n              const result = scorer({ input, ...params, output, toolCalls });\n              if (result instanceof Promise) {\n                return result;\n              }\n              return new Promise<Score>((resolve) => resolve(result));\n            }),\n          );\n          const scoresWithName = scores.map((s, i) => ({\n            ...s,\n            name: scorers[i].name,\n          }));\n\n          const avgScore =\n            scores.reduce((acc, s) => acc + (s.score ?? 0), 0) / scores.length;\n\n          testTask.meta.eval = {\n            scores: scoresWithName,\n            avgScore,\n            ...(toolCalls && { toolCalls }),\n          };\n\n          if (threshold) {\n            assert(\n              avgScore >= threshold,\n              `Score: ${avgScore} below threshold: ${threshold}\\n\\n## Output:\\n${wrapText(output)}\\n\\n${formatScores(\n                scoresWithName,\n              )}`,\n            );\n          }\n        },\n      );\n    }\n  });\n}\n\nexport function formatScores(scores: (Score & { name: string })[]) {\n  return scores\n    .sort((a, b) => (a.score ?? 0) - (b.score ?? 0))\n    .map((s) => {\n      const scoreLine = `# ${s.name || \"Unknown\"} [${(s.score ?? 0).toFixed(1)}]`;\n      if (\n        ((s.score ?? 0) < 1.0 && s.metadata?.rationale) ||\n        s.metadata?.output\n      ) {\n        // Format output - handle both strings and objects\n        let formattedOutput = \"\";\n        if (s.metadata?.output !== undefined) {\n          const output = s.metadata.output;\n          if (typeof output === \"string\") {\n            formattedOutput = `\\n\\n## Response\\n\\n${wrapText(output)}`;\n          } else {\n            // For objects, stringify with proper formatting\n            formattedOutput = `\\n\\n## Response\\n\\n${wrapText(JSON.stringify(output, null, 2))}`;\n          }\n        }\n\n        return `${scoreLine}${\n          s.metadata?.rationale\n            ? `\\n\\n## Rationale\\n\\n${wrapText(s.metadata.rationale)}`\n            : \"\"\n        }${formattedOutput}`;\n      }\n      return scoreLine;\n    })\n    .join(\"\\n\\n\");\n}\n\n/**\n * Wraps text to fit within a specified width, breaking at word boundaries.\n *\n * @param text - The text to wrap\n * @param width - The maximum width in characters (default: 80)\n * @returns The wrapped text with line breaks\n *\n * @example\n * ```javascript\n * const wrapped = wrapText(\"This is a very long text that needs to be wrapped to fit within an 80 character width.\", 20);\n * console.log(wrapped);\n * // Output:\n * // This is a very\n * // long text that\n * // needs to be\n * // wrapped to fit\n * // within an 80\n * // character width.\n * ```\n */\nexport function wrapText(text: string, width = 80): string {\n  if (!text || text.length <= width) {\n    return text;\n  }\n\n  const words = text.split(/\\s+/);\n  const lines: string[] = [];\n  let currentLine = \"\";\n\n  for (const word of words) {\n    // If adding this word would exceed the width, start a new line\n    if (currentLine.length + word.length + 1 > width) {\n      lines.push(currentLine.trim());\n      currentLine = word;\n    } else {\n      // Add the word to the current line\n      currentLine += (currentLine ? \" \" : \"\") + word;\n    }\n  }\n\n  // Add the last line if it's not empty\n  if (currentLine) {\n    lines.push(currentLine);\n  }\n\n  return lines.join(\"\\n\");\n}\n\n// Export built-in scorers\nexport {\n  ToolCallScorer,\n  type ToolCallScorerOptions,\n  StructuredOutputScorer,\n  type StructuredOutputScorerOptions,\n} from \"./scorers\";\n","/**\n * Shared utilities for scorer implementations\n */\n\n/**\n * Common configuration options for scorers\n */\nexport interface BaseMatcherConfig {\n  /**\n   * Whether all expected items must match for a passing score\n   * When false: gives partial credit based on items matched\n   * @default true\n   */\n  requireAll?: boolean;\n\n  /**\n   * Whether to allow additional items beyond those expected\n   * @default true\n   */\n  allowExtras?: boolean;\n\n  /**\n   * Enable debug logging\n   * @default false\n   */\n  debug?: boolean;\n}\n\n/**\n * Matching strategy type\n */\nexport type MatchStrategy<T = any> =\n  | \"strict\"\n  | \"fuzzy\"\n  | ((expected: T, actual: T, context?: string) => boolean);\n\n/**\n * Options for fuzzy matching behavior\n */\nexport interface FuzzyMatchOptions {\n  /**\n   * Allow case-insensitive string matching\n   * @default true\n   */\n  caseInsensitive?: boolean;\n\n  /**\n   * For strings: use substring matching instead of exact match\n   * When enabled, either string can contain the other as a substring:\n   * - \"weather\" matches \"weath\" (expected contains actual)\n   * - \"weather forecast\" matches \"weather\" (actual contains expected)\n   * @default false\n   */\n  substring?: boolean;\n\n  /**\n   * For numbers: tolerance for comparison (0.001 = 0.1%)\n   * @default 0.001\n   */\n  numericTolerance?: number;\n\n  /**\n   * For arrays: ignore order when comparing\n   * @default true\n   */\n  ignoreArrayOrder?: boolean;\n\n  /**\n   * Allow type coercion (e.g., \"42\" matches 42)\n   * @default false\n   */\n  coerceTypes?: boolean;\n}\n\n/**\n * Strict equality comparison (deep equals)\n */\nexport function strictEquals(\n  expected: any,\n  actual: any,\n  context?: string,\n): boolean {\n  // Handle primitive types and null/undefined\n  if (expected === actual) return true;\n  if (\n    expected === null ||\n    expected === undefined ||\n    actual === null ||\n    actual === undefined\n  ) {\n    return false;\n  }\n\n  // Must be same type\n  if (typeof expected !== typeof actual) return false;\n\n  // Handle arrays\n  if (Array.isArray(expected)) {\n    if (!Array.isArray(actual)) return false;\n    if (expected.length !== actual.length) return false;\n    return expected.every((item, i) => strictEquals(item, actual[i], context));\n  }\n\n  // Handle objects\n  if (typeof expected === \"object\") {\n    const expectedKeys = Object.keys(expected);\n    const actualKeys = Object.keys(actual);\n\n    // Must have same number of keys\n    if (expectedKeys.length !== actualKeys.length) return false;\n\n    // All expected keys must exist in actual and have matching values\n    return expectedKeys.every(\n      (key) =>\n        key in actual && strictEquals(expected[key], actual[key], context),\n    );\n  }\n\n  // All other primitive types already handled by line 84\n  return false;\n}\n\n/**\n * Fuzzy string matching with configurable options\n */\nfunction fuzzyMatchString(\n  expected: string,\n  actual: string,\n  options: FuzzyMatchOptions,\n): boolean {\n  const { caseInsensitive = true, substring = false } = options;\n\n  const expectedStr = caseInsensitive ? expected.toLowerCase() : expected;\n  const actualStr = caseInsensitive ? actual.toLowerCase() : actual;\n\n  return substring\n    ? actualStr.includes(expectedStr) || expectedStr.includes(actualStr)\n    : expectedStr === actualStr;\n}\n\n/**\n * Fuzzy number matching with tolerance\n */\nfunction fuzzyMatchNumber(\n  expected: number,\n  actual: number,\n  options: FuzzyMatchOptions,\n): boolean {\n  const { numericTolerance = 0.001 } = options;\n\n  const tolerance = Math.max(\n    Math.abs(expected) * numericTolerance,\n    numericTolerance,\n  );\n  return Math.abs(expected - actual) <= tolerance;\n}\n\n/**\n * Fuzzy array matching with optional order independence\n */\nfunction fuzzyMatchArray(\n  expected: any[],\n  actual: any[],\n  options: FuzzyMatchOptions,\n  context?: string,\n): boolean {\n  const { ignoreArrayOrder = true } = options;\n\n  if (ignoreArrayOrder) {\n    // Track which actual items have been consumed\n    const actualUsed = actual.map(() => false);\n\n    // Try to find a unique match for each expected item\n    return expected.every((expItem) => {\n      // Find first unused actual item that matches\n      for (let i = 0; i < actual.length; i++) {\n        if (actualUsed[i]) continue; // Already used\n\n        if (fuzzyMatch(expItem, actual[i], options, context)) {\n          actualUsed[i] = true; // Mark as used\n          return true;\n        }\n      }\n      return false; // No match found\n    });\n  }\n\n  return (\n    expected.length === actual.length &&\n    expected.every((item, i) => fuzzyMatch(item, actual[i], options, context))\n  );\n}\n\n/**\n * Fuzzy object matching (subset matching)\n */\nfunction fuzzyMatchObject(\n  expected: object,\n  actual: object,\n  options: FuzzyMatchOptions,\n  context?: string,\n): boolean {\n  return Object.entries(expected).every(\n    ([k, value]) =>\n      k in actual && fuzzyMatch(value, (actual as any)[k], options, context),\n  );\n}\n\n/**\n * Type coercion matching\n */\nfunction fuzzyMatchWithCoercion(\n  expected: any,\n  actual: any,\n  options: FuzzyMatchOptions,\n): boolean {\n  // Boolean coercion\n  if (typeof expected === \"boolean\" && typeof actual === \"string\") {\n    return expected === (actual.toLowerCase() === \"true\" || actual === \"1\");\n  }\n\n  // Number-string coercion\n  if (typeof expected === \"string\" && typeof actual === \"number\") {\n    return Number.parseFloat(expected) === actual;\n  }\n  if (typeof expected === \"number\" && typeof actual === \"string\") {\n    return expected === Number.parseFloat(actual);\n  }\n\n  return false;\n}\n\n/**\n * Fuzzy matching with flexible comparison logic\n */\nexport function fuzzyMatch(\n  expected: any,\n  actual: any,\n  options: FuzzyMatchOptions = {},\n  context?: string,\n): boolean {\n  const { coerceTypes = false } = options;\n\n  // Handle regex patterns\n  if (expected instanceof RegExp) {\n    return typeof actual === \"string\" && expected.test(actual);\n  }\n\n  // Handle functions (custom validators)\n  if (typeof expected === \"function\") {\n    return expected(actual);\n  }\n\n  // Null/undefined handling\n  if (\n    expected === null ||\n    expected === undefined ||\n    actual === null ||\n    actual === undefined\n  ) {\n    return expected === actual;\n  }\n\n  // Type-specific matching\n  if (typeof expected === \"string\" && typeof actual === \"string\") {\n    return fuzzyMatchString(expected, actual, options);\n  }\n\n  if (typeof expected === \"number\" && typeof actual === \"number\") {\n    return fuzzyMatchNumber(expected, actual, options);\n  }\n\n  if (Array.isArray(expected) && Array.isArray(actual)) {\n    return fuzzyMatchArray(expected, actual, options, context);\n  }\n\n  if (\n    typeof expected === \"object\" &&\n    typeof actual === \"object\" &&\n    !Array.isArray(expected) &&\n    !Array.isArray(actual)\n  ) {\n    return fuzzyMatchObject(expected, actual, options, context);\n  }\n\n  // Handle type coercion if enabled\n  if (coerceTypes) {\n    const coercionResult = fuzzyMatchWithCoercion(expected, actual, options);\n    if (coercionResult) return true;\n  }\n\n  // For all other cases, strict equality\n  return expected === actual;\n}\n\n/**\n * Create a matcher function based on strategy\n */\nexport function createMatcher<T = any>(\n  strategy: MatchStrategy<T>,\n  options?: FuzzyMatchOptions,\n): (expected: T, actual: T, context?: string) => boolean {\n  if (typeof strategy === \"function\") {\n    return (expected, actual, context) => strategy(expected, actual, context);\n  }\n\n  if (strategy === \"strict\") {\n    return (expected, actual, context) =>\n      strictEquals(expected, actual, context);\n  }\n\n  return (expected, actual, context) =>\n    fuzzyMatch(expected, actual, options || {}, context);\n}\n\n/**\n * Format a value for display in error messages\n */\nexport function formatValue(value: any): string {\n  if (value === undefined) return \"undefined\";\n  if (value === null) return \"null\";\n  if (value instanceof RegExp) return value.toString();\n  if (typeof value === \"string\") return `\"${value}\"`;\n  if (typeof value === \"object\") return JSON.stringify(value);\n  return String(value);\n}\n\n/**\n * Calculate partial score based on matches\n */\nexport function calculatePartialScore(\n  matched: number,\n  total: number,\n  requireAll: boolean,\n): number {\n  if (requireAll && matched < total) {\n    return 0.0;\n  }\n  return total > 0 ? matched / total : 1.0;\n}\n\n/**\n * Logger interface for debug output\n */\nexport interface Logger {\n  log: (message: string, ...args: any[]) => void;\n}\n\n/**\n * Default logger that respects NODE_ENV and can be disabled\n */\nconst defaultLogger: Logger = {\n  log: (message: string, ...args: any[]) => {\n    // Only log in development or test environments, or when explicitly enabled\n    if (\n      process.env.NODE_ENV === \"development\" ||\n      process.env.NODE_ENV === \"test\" ||\n      process.env.VITEST_EVALS_DEBUG === \"true\"\n    ) {\n      console.log(message, ...args);\n    }\n  },\n};\n\n/**\n * Helper for debugging matcher results with configurable logging\n */\nexport function debugLog(\n  context: string,\n  data: {\n    expected: any;\n    actual: any;\n    matches?: string[];\n    mismatches?: Array<{ key: string; expected: any; actual: any }>;\n    extras?: string[];\n  },\n  logger: Logger = defaultLogger,\n): void {\n  logger.log(`${context} debug:`);\n  logger.log(\"Expected:\", data.expected);\n  logger.log(\"Actual:\", data.actual);\n  if (data.matches) logger.log(\"Matches:\", data.matches);\n  if (data.mismatches) logger.log(\"Mismatches:\", data.mismatches);\n  if (data.extras) logger.log(\"Extras:\", data.extras);\n}\n","import type { ScoreFn, BaseScorerOptions, ToolCall } from \"../index\";\nimport {\n  type BaseMatcherConfig,\n  type MatchStrategy,\n  type FuzzyMatchOptions,\n  createMatcher,\n} from \"./utils\";\n\nexport interface ToolCallScorerOptions extends BaseScorerOptions {\n  // Expected tools are now defined in the test data\n  expectedTools?: Array<{\n    name: string;\n    arguments?: any;\n  }>;\n}\n\nexport interface ToolCallScorerConfig extends BaseMatcherConfig {\n  /**\n   * Whether tools must be called in the exact order specified\n   * @default false\n   */\n  ordered?: boolean;\n\n  /**\n   * How to match tool arguments/parameters\n   * - \"strict\": Exact equality required (default)\n   * - \"fuzzy\": Flexible matching with tolerance for differences\n   *   - Case-insensitive string matching\n   *   - Numeric tolerance for small differences\n   *   - Unordered array comparison\n   *   - Subset matching for objects (actual can have extra properties)\n   * - Custom function: Your own comparison logic\n   *\n   * NOTE: Each expected tool call requires a unique actual tool call to match.\n   * Multiple identical expected tools need separate actual tool calls.\n   *\n   * @default \"strict\"\n   */\n  params?: MatchStrategy;\n\n  /**\n   * Options for fuzzy matching when params=\"fuzzy\"\n   * These options are MERGED with defaults, not replaced.\n   *\n   * Default fuzzy options for tool calls:\n   * - substring: true (allow substring matching for strings)\n   * - caseInsensitive: true (ignore case differences)\n   * - ignoreArrayOrder: true (arrays can be in different orders)\n   * - numericTolerance: 0.001 (0.1% tolerance for numbers)\n   * - coerceTypes: false (no automatic type conversion)\n   *\n   * @default { substring: true, caseInsensitive: true, ignoreArrayOrder: true }\n   */\n  fuzzyOptions?: FuzzyMatchOptions;\n}\n\n/**\n * A configurable scorer for evaluating tool usage in LLM responses.\n *\n * The test data defines WHAT tools/arguments are expected,\n * while this scorer defines HOW to evaluate them.\n *\n * @param config - Configuration options for the scorer\n * @param config.ordered - Require exact order of tool calls\n * @param config.requireAll - Require all expected tools (vs partial credit)\n * @param config.allowExtras - Allow additional tool calls\n * @param config.params - How to match parameters: \"strict\", \"fuzzy\", or custom function\n *\n * @example\n * // Default: strict params, any order\n * describeEval(\"search test\", {\n *   data: async () => [{\n *     input: \"Find restaurants\",\n *     expectedTools: [\n *       { name: \"search\", arguments: { type: \"restaurant\" } },\n *       { name: \"filter\" }\n *     ]\n *   }],\n *   task: myTask,\n *   scorers: [ToolCallScorer()]\n * });\n *\n * @example\n * // Strict order and parameters\n * describeEval(\"payment flow\", {\n *   data: async () => [{\n *     input: \"Process payment\",\n *     expectedTools: [\n *       { name: \"validate\", arguments: { amount: 100 } },\n *       { name: \"charge\", arguments: { amount: 100, method: \"card\" } }\n *     ]\n *   }],\n *   task: myTask,\n *   scorers: [ToolCallScorer({ ordered: true, params: \"strict\" })]\n * });\n */\nexport function ToolCallScorer(\n  config: ToolCallScorerConfig = {},\n): ScoreFn<ToolCallScorerOptions> {\n  const {\n    ordered = false,\n    requireAll = true,\n    allowExtras = true,\n    params = \"strict\",\n    fuzzyOptions: userFuzzyOptions,\n  } = config;\n\n  // Merge user fuzzyOptions with defaults for tool calls\n  const defaultFuzzyOptions: FuzzyMatchOptions = {\n    substring: true,\n    caseInsensitive: true,\n    ignoreArrayOrder: true,\n    numericTolerance: 0.001,\n    coerceTypes: false,\n  };\n  const fuzzyOptions: FuzzyMatchOptions = {\n    ...defaultFuzzyOptions,\n    ...userFuzzyOptions,\n  };\n\n  // Determine the argument matcher\n  const argMatcher = createMatcher(params, fuzzyOptions);\n\n  return async (opts) => {\n    const expectedTools = opts.expectedTools || [];\n    const actualCalls = opts.toolCalls || [];\n\n    // No expectations means pass\n    if (expectedTools.length === 0) {\n      return {\n        score: 1.0,\n        metadata: {\n          rationale: \"No tool calls expected\",\n        },\n      };\n    }\n\n    // No actual calls when we expected some\n    if (actualCalls.length === 0) {\n      return {\n        score: 0.0,\n        metadata: {\n          rationale: `Expected ${expectedTools.length} tool(s) but none were called`,\n        },\n      };\n    }\n\n    if (ordered) {\n      return evaluateOrderedTools(expectedTools, actualCalls, {\n        argMatcher,\n        allowExtras,\n        requireAllTools: requireAll,\n      });\n    }\n\n    return evaluateUnorderedTools(expectedTools, actualCalls, {\n      argMatcher,\n      requireAllTools: requireAll,\n      allowExtras,\n    });\n  };\n}\n\n/**\n * Evaluate tools that must be called in a specific order\n */\nfunction evaluateOrderedTools(\n  expected: Array<{ name: string; arguments?: any }>,\n  actual: ToolCall[],\n  options: {\n    argMatcher: (expected: any, actual: any) => boolean;\n    allowExtras: boolean;\n    requireAllTools: boolean;\n  },\n) {\n  let expectedIndex = 0;\n  let actualIndex = 0;\n\n  // Match expected tools in order\n  while (expectedIndex < expected.length && actualIndex < actual.length) {\n    const exp = expected[expectedIndex];\n    const act = actual[actualIndex];\n\n    if (exp.name === act.name) {\n      // Check arguments if specified\n      if (exp.arguments !== undefined) {\n        const argsMatch = options.argMatcher(\n          exp.arguments,\n          act.arguments || {},\n        );\n        if (!argsMatch) {\n          // Give partial credit for tools matched up to this point\n          const partialScore = expectedIndex / expected.length;\n          return {\n            score: partialScore,\n            metadata: {\n              rationale: `Tool '${exp.name}' called with incorrect arguments at position ${expectedIndex + 1} (${expectedIndex}/${expected.length} tools matched correctly)`,\n              expected: exp.arguments,\n              actual: act.arguments,\n              matched: expectedIndex,\n              total: expected.length,\n            },\n          };\n        }\n      }\n      expectedIndex++;\n      actualIndex++;\n    } else if (options.allowExtras) {\n      // Skip extra tool\n      actualIndex++;\n    } else {\n      // Wrong tool in sequence when extra tools not allowed\n      return {\n        score: 0.0,\n        metadata: {\n          rationale: `Expected '${exp.name}' at position ${expectedIndex + 1} but found '${act.name}'`,\n        },\n      };\n    }\n  }\n\n  // Check if all expected tools were matched\n  if (expectedIndex < expected.length) {\n    const missing = expected.slice(expectedIndex).map((t) => t.name);\n\n    if (options.requireAllTools) {\n      return {\n        score: 0.0,\n        metadata: {\n          rationale: `Missing required tools in sequence: ${missing.join(\", \")}`,\n        },\n      };\n    }\n\n    // Partial credit when requireAllTools is false\n    const matchedCount = expectedIndex;\n    const totalCount = expected.length;\n    const score = totalCount > 0 ? matchedCount / totalCount : 1.0;\n\n    return {\n      score,\n      metadata: {\n        rationale: `Partial match: ${matchedCount}/${totalCount} tools called in order (missing: ${missing.join(\", \")})`,\n        matched: matchedCount,\n        total: totalCount,\n      },\n    };\n  }\n\n  // Check for extra tools at the end if not allowed\n  if (!options.allowExtras && actualIndex < actual.length) {\n    const extra = actual.slice(actualIndex).map((t) => t.name);\n    return {\n      score: 0.0,\n      metadata: {\n        rationale: `Unexpected extra tools: ${extra.join(\", \")}`,\n      },\n    };\n  }\n\n  return {\n    score: 1.0,\n    metadata: {\n      rationale: \"All tools called in expected order with correct arguments\",\n    },\n  };\n}\n\n/**\n * Evaluate tools that can be called in any order\n *\n * Simple logic:\n * 1. Start with copies of expected and actual tool arrays\n * 2. For each expected tool, find and remove a matching actual tool\n * 3. Remaining expected = missing, remaining actual = extras\n */\nfunction evaluateUnorderedTools(\n  expected: Array<{ name: string; arguments?: any }>,\n  actual: ToolCall[],\n  options: {\n    argMatcher: (expected: any, actual: any) => boolean;\n    requireAllTools: boolean;\n    allowExtras: boolean;\n  },\n) {\n  // Work with copies so we can remove items\n  const remainingExpected = [...expected];\n  const remainingActual = [...actual];\n  const issues: string[] = [];\n\n  // For each expected tool, find and remove a matching actual tool\n  for (let i = remainingExpected.length - 1; i >= 0; i--) {\n    const expectedTool = remainingExpected[i];\n\n    // Find a matching actual tool\n    const matchIndex = remainingActual.findIndex((actualTool) => {\n      // Check if this actual tool matches the expected tool\n      if (expectedTool.name !== actualTool.name) {\n        return false;\n      }\n\n      // Check arguments if specified\n      if (expectedTool.arguments !== undefined) {\n        return options.argMatcher(\n          expectedTool.arguments,\n          actualTool.arguments || {},\n        );\n      }\n\n      return true;\n    });\n\n    if (matchIndex !== -1) {\n      // Found a match - remove both\n      remainingExpected.splice(i, 1);\n      remainingActual.splice(matchIndex, 1);\n    }\n  }\n\n  // Generate issues for missing tools\n  for (const missingTool of remainingExpected) {\n    if (missingTool.arguments !== undefined) {\n      // Check if tool was called but with wrong args\n      const wrongArgsCalls = actual.filter((a) => a.name === missingTool.name);\n      if (wrongArgsCalls.length > 0) {\n        issues.push(\n          `Tool '${missingTool.name}' called but with incorrect arguments`,\n        );\n      } else {\n        issues.push(`Missing required tool: ${missingTool.name}`);\n      }\n    } else {\n      issues.push(`Missing required tool: ${missingTool.name}`);\n    }\n  }\n\n  // Extra tools = remaining actual tools\n  const extraTools = remainingActual.map((tool) => tool.name);\n\n  if (!options.allowExtras && extraTools.length > 0) {\n    issues.push(`Unexpected extra tools: ${extraTools.join(\", \")}`);\n  }\n\n  // Calculate score\n  const expectedMatched = expected.length - remainingExpected.length;\n  const score = expected.length > 0 ? expectedMatched / expected.length : 1.0;\n\n  // If we have any critical issues (wrong tools, missing tools when required, or extra tools when not allowed)\n  if (issues.length > 0 && (options.requireAllTools || !options.allowExtras)) {\n    return {\n      score: 0.0,\n      metadata: {\n        rationale: issues.join(\"; \"),\n      },\n    };\n  }\n\n  if (score === 1.0) {\n    const extraInfo =\n      extraTools.length > 0 ? ` (plus extra: ${extraTools.join(\", \")})` : \"\";\n    return {\n      score: 1.0,\n      metadata: {\n        rationale: `All expected tools were called${extraInfo}`,\n      },\n    };\n  }\n\n  return {\n    score,\n    metadata: {\n      rationale: issues.join(\"; \"),\n      matched: expectedMatched,\n      total: expected.length,\n    },\n  };\n}\n","import type { ScoreFn, BaseScorerOptions } from \"../index\";\nimport {\n  type BaseMatcherConfig,\n  type MatchStrategy,\n  type FuzzyMatchOptions,\n  createMatcher,\n  formatValue,\n  debugLog,\n} from \"./utils\";\n\nexport interface StructuredOutputScorerOptions extends BaseScorerOptions {\n  // Expected structured output defined in test data\n  expected?: Record<string, any>;\n}\n\nexport interface StructuredOutputScorerConfig extends BaseMatcherConfig {\n  /**\n   * How to match field values\n   * - \"strict\": Exact equality required (default)\n   * - \"fuzzy\": More flexible matching (case-insensitive strings, numeric tolerance, regex patterns, subset matching)\n   * - Custom function: Your own comparison logic\n   * @default \"strict\"\n   */\n  match?: MatchStrategy;\n\n  /**\n   * Field name to check for errors in the output\n   * Set to null to disable error checking\n   * @default \"error\"\n   */\n  errorField?: string | null;\n\n  /**\n   * Options for fuzzy matching when match=\"fuzzy\"\n   * @default {} for structured output (no substring matching by default)\n   */\n  fuzzyOptions?: FuzzyMatchOptions;\n}\n\n/**\n * Format mismatch details for error messages\n */\nfunction formatMismatchDetails(\n  mismatches: Array<{ key: string; expected: any; actual: any }>,\n): string {\n  return mismatches\n    .map(\n      (m) =>\n        `${m.key}: expected ${formatValue(m.expected)}, got ${formatValue(m.actual)}`,\n    )\n    .join(\"; \");\n}\n\n/**\n * A configurable scorer for evaluating structured outputs (e.g., JSON) from LLM responses.\n *\n * Similar to ToolCallScorer but for validating structured data outputs like API queries,\n * configuration objects, or any JSON-serializable data structure.\n *\n * @param config - Configuration options for the scorer\n * @param config.match - How to match field values: \"strict\", \"fuzzy\", or custom function\n * @param config.requireAll - Require all expected fields (vs partial credit)\n * @param config.allowExtras - Allow additional fields in output\n * @param config.debug - Enable debug logging\n *\n * @example\n * // Default: strict matching\n * describeEval(\"query generation\", {\n *   data: async () => [{\n *     input: \"Show me errors from today\",\n *     expected: {\n *       dataset: \"errors\",\n *       query: \"\",\n *       sort: \"-timestamp\",\n *       timeRange: { statsPeriod: \"24h\" }\n *     }\n *   }],\n *   task: myTask,\n *   scorers: [StructuredOutputScorer()]\n * });\n *\n * @example\n * // Fuzzy matching with regex patterns\n * describeEval(\"flexible query matching\", {\n *   data: async () => [{\n *     input: \"Find slow API calls\",\n *     expected: {\n *       dataset: \"spans\",\n *       query: /span\\.duration:>1000|span\\.duration:>1s/,\n *       sort: \"-span.duration\"\n *     }\n *   }],\n *   task: myTask,\n *   scorers: [StructuredOutputScorer({ match: \"fuzzy\" })]\n * });\n *\n * @example\n * // Custom field matching\n * describeEval(\"custom validation\", {\n *   data: async () => [{\n *     input: \"Create user config\",\n *     expected: {\n *       name: \"test\",\n *       age: 25,\n *       tags: [\"user\", \"active\"]\n *     }\n *   }],\n *   task: myTask,\n *   scorers: [StructuredOutputScorer({\n *     match: (expected, actual, key) => {\n *       if (key === \"age\") return actual >= 18 && actual <= 100;\n *       return expected === actual; // Use strict equality for other fields\n *     }\n *   })]\n * });\n */\nexport function StructuredOutputScorer(\n  config: StructuredOutputScorerConfig = {},\n): ScoreFn<StructuredOutputScorerOptions> {\n  const {\n    match = \"strict\",\n    requireAll = true,\n    allowExtras = true,\n    debug = false,\n    errorField = \"error\",\n    fuzzyOptions = {}, // Default: no special fuzzy options for structured output\n  } = config;\n\n  // Determine the field matcher. If a custom function is provided, it will be used directly\n  // with its original signature. Note that createMatcher only supports two parameters: match and fuzzyOptions.\n  const fieldMatcher =\n    typeof match === \"function\"\n      ? match // Use custom function directly with its original signature\n      : createMatcher(match, fuzzyOptions);\n\n  return async (opts) => {\n    const expected = opts.expected || {};\n    const output = opts.output;\n\n    // Parse the output as JSON\n    let parsed: Record<string, any>;\n    try {\n      parsed = JSON.parse(output);\n    } catch (error) {\n      return {\n        score: 0.0,\n        metadata: {\n          rationale: `Failed to parse output as JSON: ${error}`,\n          output,\n        },\n      };\n    }\n\n    // No expectations means we just check for valid JSON\n    if (Object.keys(expected).length === 0) {\n      return {\n        score: 1.0,\n        metadata: {\n          rationale: \"Valid JSON output (no expected fields specified)\",\n        },\n      };\n    }\n\n    // Check for error field in output (common pattern for API responses)\n    if (\n      errorField !== null &&\n      parsed[errorField] &&\n      parsed[errorField] !== \"\" &&\n      parsed[errorField] !== null\n    ) {\n      return {\n        score: 0.0,\n        metadata: {\n          rationale: `Output contains error: ${parsed[errorField]}`,\n          output,\n        },\n      };\n    }\n\n    // Compare expected vs actual fields\n    const matches: string[] = [];\n    const mismatches: Array<{ key: string; expected: any; actual: any }> = [];\n    const extras: string[] = [];\n\n    // Check each expected field\n    for (const [key, expectedValue] of Object.entries(expected)) {\n      const actualValue = parsed[key];\n\n      const isMatch = fieldMatcher(expectedValue, actualValue, key);\n      if (isMatch) {\n        matches.push(key);\n      } else {\n        mismatches.push({ key, expected: expectedValue, actual: actualValue });\n      }\n    }\n\n    // Find extra fields\n    const expectedKeys = new Set(Object.keys(expected));\n    for (const key of Object.keys(parsed)) {\n      if (!expectedKeys.has(key)) {\n        extras.push(key);\n      }\n    }\n\n    if (debug) {\n      debugLog(\"StructuredOutputScorer\", {\n        expected,\n        actual: parsed,\n        matches,\n        mismatches,\n        extras,\n      });\n    }\n\n    // Calculate score and rationale\n    const totalExpected = Object.keys(expected).length;\n    const totalMatched = matches.length;\n\n    // Handle various failure conditions\n    if (requireAll && mismatches.length > 0) {\n      const mismatchDetails = formatMismatchDetails(mismatches);\n      return {\n        score: 0.0,\n        metadata: {\n          rationale: `Missing required fields: ${mismatches.map((m) => m.key).join(\", \")} - ${mismatchDetails}`,\n        },\n      };\n    }\n\n    if (!allowExtras && extras.length > 0) {\n      return {\n        score: 0.0,\n        metadata: {\n          rationale: `Unexpected extra fields: ${extras.join(\", \")}`,\n        },\n      };\n    }\n\n    // Calculate partial credit score\n    const score = totalExpected > 0 ? totalMatched / totalExpected : 1.0;\n\n    if (score === 1.0) {\n      const extraInfo =\n        extras.length > 0 ? ` (plus extra fields: ${extras.join(\", \")})` : \"\";\n      return {\n        score: 1.0,\n        metadata: {\n          rationale: `All expected fields match${extraInfo}`,\n        },\n      };\n    }\n\n    // Partial match\n    const mismatchDetails = formatMismatchDetails(mismatches);\n\n    return {\n      score,\n      metadata: {\n        rationale: `Matched ${totalMatched}/${totalExpected} fields - ${mismatchDetails}`,\n        matched: totalMatched,\n        total: totalExpected,\n      },\n    };\n  };\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,SAAS,QAAQ,UAAU,QAAQ,YAAY;AAC/C,OAAO;;;AC4EA,SAAS,aACd,UACA,QACA,SACS;AAET,MAAI,aAAa,OAAQ,QAAO;AAChC,MACE,aAAa,QACb,aAAa,UACb,WAAW,QACX,WAAW,QACX;AACA,WAAO;AAAA,EACT;AAGA,MAAI,OAAO,aAAa,OAAO,OAAQ,QAAO;AAG9C,MAAI,MAAM,QAAQ,QAAQ,GAAG;AAC3B,QAAI,CAAC,MAAM,QAAQ,MAAM,EAAG,QAAO;AACnC,QAAI,SAAS,WAAW,OAAO,OAAQ,QAAO;AAC9C,WAAO,SAAS,MAAM,CAAC,MAAM,MAAM,aAAa,MAAM,OAAO,CAAC,GAAG,OAAO,CAAC;AAAA,EAC3E;AAGA,MAAI,OAAO,aAAa,UAAU;AAChC,UAAM,eAAe,OAAO,KAAK,QAAQ;AACzC,UAAM,aAAa,OAAO,KAAK,MAAM;AAGrC,QAAI,aAAa,WAAW,WAAW,OAAQ,QAAO;AAGtD,WAAO,aAAa;AAAA,MAClB,CAAC,QACC,OAAO,UAAU,aAAa,SAAS,GAAG,GAAG,OAAO,GAAG,GAAG,OAAO;AAAA,IACrE;AAAA,EACF;AAGA,SAAO;AACT;AAKA,SAAS,iBACP,UACA,QACA,SACS;AACT,QAAM,EAAE,kBAAkB,MAAM,YAAY,MAAM,IAAI;AAEtD,QAAM,cAAc,kBAAkB,SAAS,YAAY,IAAI;AAC/D,QAAM,YAAY,kBAAkB,OAAO,YAAY,IAAI;AAE3D,SAAO,YACH,UAAU,SAAS,WAAW,KAAK,YAAY,SAAS,SAAS,IACjE,gBAAgB;AACtB;AAKA,SAAS,iBACP,UACA,QACA,SACS;AACT,QAAM,EAAE,mBAAmB,KAAM,IAAI;AAErC,QAAM,YAAY,KAAK;AAAA,IACrB,KAAK,IAAI,QAAQ,IAAI;AAAA,IACrB;AAAA,EACF;AACA,SAAO,KAAK,IAAI,WAAW,MAAM,KAAK;AACxC;AAKA,SAAS,gBACP,UACA,QACA,SACA,SACS;AACT,QAAM,EAAE,mBAAmB,KAAK,IAAI;AAEpC,MAAI,kBAAkB;AAEpB,UAAM,aAAa,OAAO,IAAI,MAAM,KAAK;AAGzC,WAAO,SAAS,MAAM,CAAC,YAAY;AAEjC,eAAS,IAAI,GAAG,IAAI,OAAO,QAAQ,KAAK;AACtC,YAAI,WAAW,CAAC,EAAG;AAEnB,YAAI,WAAW,SAAS,OAAO,CAAC,GAAG,SAAS,OAAO,GAAG;AACpD,qBAAW,CAAC,IAAI;AAChB,iBAAO;AAAA,QACT;AAAA,MACF;AACA,aAAO;AAAA,IACT,CAAC;AAAA,EACH;AAEA,SACE,SAAS,WAAW,OAAO,UAC3B,SAAS,MAAM,CAAC,MAAM,MAAM,WAAW,MAAM,OAAO,CAAC,GAAG,SAAS,OAAO,CAAC;AAE7E;AAKA,SAAS,iBACP,UACA,QACA,SACA,SACS;AACT,SAAO,OAAO,QAAQ,QAAQ,EAAE;AAAA,IAC9B,CAAC,CAAC,GAAG,KAAK,MACR,KAAK,UAAU,WAAW,OAAQ,OAAe,CAAC,GAAG,SAAS,OAAO;AAAA,EACzE;AACF;AAKA,SAAS,uBACP,UACA,QACA,SACS;AAET,MAAI,OAAO,aAAa,aAAa,OAAO,WAAW,UAAU;AAC/D,WAAO,cAAc,OAAO,YAAY,MAAM,UAAU,WAAW;AAAA,EACrE;AAGA,MAAI,OAAO,aAAa,YAAY,OAAO,WAAW,UAAU;AAC9D,WAAO,OAAO,WAAW,QAAQ,MAAM;AAAA,EACzC;AACA,MAAI,OAAO,aAAa,YAAY,OAAO,WAAW,UAAU;AAC9D,WAAO,aAAa,OAAO,WAAW,MAAM;AAAA,EAC9C;AAEA,SAAO;AACT;AAKO,SAAS,WACd,UACA,QACA,UAA6B,CAAC,GAC9B,SACS;AACT,QAAM,EAAE,cAAc,MAAM,IAAI;AAGhC,MAAI,oBAAoB,QAAQ;AAC9B,WAAO,OAAO,WAAW,YAAY,SAAS,KAAK,MAAM;AAAA,EAC3D;AAGA,MAAI,OAAO,aAAa,YAAY;AAClC,WAAO,SAAS,MAAM;AAAA,EACxB;AAGA,MACE,aAAa,QACb,aAAa,UACb,WAAW,QACX,WAAW,QACX;AACA,WAAO,aAAa;AAAA,EACtB;AAGA,MAAI,OAAO,aAAa,YAAY,OAAO,WAAW,UAAU;AAC9D,WAAO,iBAAiB,UAAU,QAAQ,OAAO;AAAA,EACnD;AAEA,MAAI,OAAO,aAAa,YAAY,OAAO,WAAW,UAAU;AAC9D,WAAO,iBAAiB,UAAU,QAAQ,OAAO;AAAA,EACnD;AAEA,MAAI,MAAM,QAAQ,QAAQ,KAAK,MAAM,QAAQ,MAAM,GAAG;AACpD,WAAO,gBAAgB,UAAU,QAAQ,SAAS,OAAO;AAAA,EAC3D;AAEA,MACE,OAAO,aAAa,YACpB,OAAO,WAAW,YAClB,CAAC,MAAM,QAAQ,QAAQ,KACvB,CAAC,MAAM,QAAQ,MAAM,GACrB;AACA,WAAO,iBAAiB,UAAU,QAAQ,SAAS,OAAO;AAAA,EAC5D;AAGA,MAAI,aAAa;AACf,UAAM,iBAAiB,uBAAuB,UAAU,QAAQ,OAAO;AACvE,QAAI,eAAgB,QAAO;AAAA,EAC7B;AAGA,SAAO,aAAa;AACtB;AAKO,SAAS,cACd,UACA,SACuD;AACvD,MAAI,OAAO,aAAa,YAAY;AAClC,WAAO,CAAC,UAAU,QAAQ,YAAY,SAAS,UAAU,QAAQ,OAAO;AAAA,EAC1E;AAEA,MAAI,aAAa,UAAU;AACzB,WAAO,CAAC,UAAU,QAAQ,YACxB,aAAa,UAAU,QAAQ,OAAO;AAAA,EAC1C;AAEA,SAAO,CAAC,UAAU,QAAQ,YACxB,WAAW,UAAU,QAAQ,WAAW,CAAC,GAAG,OAAO;AACvD;AAKO,SAAS,YAAY,OAAoB;AAC9C,MAAI,UAAU,OAAW,QAAO;AAChC,MAAI,UAAU,KAAM,QAAO;AAC3B,MAAI,iBAAiB,OAAQ,QAAO,MAAM,SAAS;AACnD,MAAI,OAAO,UAAU,SAAU,QAAO,IAAI,KAAK;AAC/C,MAAI,OAAO,UAAU,SAAU,QAAO,KAAK,UAAU,KAAK;AAC1D,SAAO,OAAO,KAAK;AACrB;AA0BA,IAAM,gBAAwB;AAAA,EAC5B,KAAK,CAAC,YAAoB,SAAgB;AAExC,QACE,QAAQ,IAAI,aAAa,iBACzB,QAAQ,IAAI,aAAa,UACzB,QAAQ,IAAI,uBAAuB,QACnC;AACA,cAAQ,IAAI,SAAS,GAAG,IAAI;AAAA,IAC9B;AAAA,EACF;AACF;AAKO,SAAS,SACd,SACA,MAOA,SAAiB,eACX;AACN,SAAO,IAAI,GAAG,OAAO,SAAS;AAC9B,SAAO,IAAI,aAAa,KAAK,QAAQ;AACrC,SAAO,IAAI,WAAW,KAAK,MAAM;AACjC,MAAI,KAAK,QAAS,QAAO,IAAI,YAAY,KAAK,OAAO;AACrD,MAAI,KAAK,WAAY,QAAO,IAAI,eAAe,KAAK,UAAU;AAC9D,MAAI,KAAK,OAAQ,QAAO,IAAI,WAAW,KAAK,MAAM;AACpD;;;AChSO,SAAS,eACd,SAA+B,CAAC,GACA;AAChC,QAAM;AAAA,IACJ,UAAU;AAAA,IACV,aAAa;AAAA,IACb,cAAc;AAAA,IACd,SAAS;AAAA,IACT,cAAc;AAAA,EAChB,IAAI;AAGJ,QAAM,sBAAyC;AAAA,IAC7C,WAAW;AAAA,IACX,iBAAiB;AAAA,IACjB,kBAAkB;AAAA,IAClB,kBAAkB;AAAA,IAClB,aAAa;AAAA,EACf;AACA,QAAM,eAAkC,kCACnC,sBACA;AAIL,QAAM,aAAa,cAAc,QAAQ,YAAY;AAErD,SAAO,CAAO,SAAS;AACrB,UAAM,gBAAgB,KAAK,iBAAiB,CAAC;AAC7C,UAAM,cAAc,KAAK,aAAa,CAAC;AAGvC,QAAI,cAAc,WAAW,GAAG;AAC9B,aAAO;AAAA,QACL,OAAO;AAAA,QACP,UAAU;AAAA,UACR,WAAW;AAAA,QACb;AAAA,MACF;AAAA,IACF;AAGA,QAAI,YAAY,WAAW,GAAG;AAC5B,aAAO;AAAA,QACL,OAAO;AAAA,QACP,UAAU;AAAA,UACR,WAAW,YAAY,cAAc,MAAM;AAAA,QAC7C;AAAA,MACF;AAAA,IACF;AAEA,QAAI,SAAS;AACX,aAAO,qBAAqB,eAAe,aAAa;AAAA,QACtD;AAAA,QACA;AAAA,QACA,iBAAiB;AAAA,MACnB,CAAC;AAAA,IACH;AAEA,WAAO,uBAAuB,eAAe,aAAa;AAAA,MACxD;AAAA,MACA,iBAAiB;AAAA,MACjB;AAAA,IACF,CAAC;AAAA,EACH;AACF;AAKA,SAAS,qBACP,UACA,QACA,SAKA;AACA,MAAI,gBAAgB;AACpB,MAAI,cAAc;AAGlB,SAAO,gBAAgB,SAAS,UAAU,cAAc,OAAO,QAAQ;AACrE,UAAM,MAAM,SAAS,aAAa;AAClC,UAAM,MAAM,OAAO,WAAW;AAE9B,QAAI,IAAI,SAAS,IAAI,MAAM;AAEzB,UAAI,IAAI,cAAc,QAAW;AAC/B,cAAM,YAAY,QAAQ;AAAA,UACxB,IAAI;AAAA,UACJ,IAAI,aAAa,CAAC;AAAA,QACpB;AACA,YAAI,CAAC,WAAW;AAEd,gBAAM,eAAe,gBAAgB,SAAS;AAC9C,iBAAO;AAAA,YACL,OAAO;AAAA,YACP,UAAU;AAAA,cACR,WAAW,SAAS,IAAI,IAAI,iDAAiD,gBAAgB,CAAC,KAAK,aAAa,IAAI,SAAS,MAAM;AAAA,cACnI,UAAU,IAAI;AAAA,cACd,QAAQ,IAAI;AAAA,cACZ,SAAS;AAAA,cACT,OAAO,SAAS;AAAA,YAClB;AAAA,UACF;AAAA,QACF;AAAA,MACF;AACA;AACA;AAAA,IACF,WAAW,QAAQ,aAAa;AAE9B;AAAA,IACF,OAAO;AAEL,aAAO;AAAA,QACL,OAAO;AAAA,QACP,UAAU;AAAA,UACR,WAAW,aAAa,IAAI,IAAI,iBAAiB,gBAAgB,CAAC,eAAe,IAAI,IAAI;AAAA,QAC3F;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAGA,MAAI,gBAAgB,SAAS,QAAQ;AACnC,UAAM,UAAU,SAAS,MAAM,aAAa,EAAE,IAAI,CAAC,MAAM,EAAE,IAAI;AAE/D,QAAI,QAAQ,iBAAiB;AAC3B,aAAO;AAAA,QACL,OAAO;AAAA,QACP,UAAU;AAAA,UACR,WAAW,uCAAuC,QAAQ,KAAK,IAAI,CAAC;AAAA,QACtE;AAAA,MACF;AAAA,IACF;AAGA,UAAM,eAAe;AACrB,UAAM,aAAa,SAAS;AAC5B,UAAM,QAAQ,aAAa,IAAI,eAAe,aAAa;AAE3D,WAAO;AAAA,MACL;AAAA,MACA,UAAU;AAAA,QACR,WAAW,kBAAkB,YAAY,IAAI,UAAU,oCAAoC,QAAQ,KAAK,IAAI,CAAC;AAAA,QAC7G,SAAS;AAAA,QACT,OAAO;AAAA,MACT;AAAA,IACF;AAAA,EACF;AAGA,MAAI,CAAC,QAAQ,eAAe,cAAc,OAAO,QAAQ;AACvD,UAAM,QAAQ,OAAO,MAAM,WAAW,EAAE,IAAI,CAAC,MAAM,EAAE,IAAI;AACzD,WAAO;AAAA,MACL,OAAO;AAAA,MACP,UAAU;AAAA,QACR,WAAW,2BAA2B,MAAM,KAAK,IAAI,CAAC;AAAA,MACxD;AAAA,IACF;AAAA,EACF;AAEA,SAAO;AAAA,IACL,OAAO;AAAA,IACP,UAAU;AAAA,MACR,WAAW;AAAA,IACb;AAAA,EACF;AACF;AAUA,SAAS,uBACP,UACA,QACA,SAKA;AAEA,QAAM,oBAAoB,CAAC,GAAG,QAAQ;AACtC,QAAM,kBAAkB,CAAC,GAAG,MAAM;AAClC,QAAM,SAAmB,CAAC;AAG1B,WAAS,IAAI,kBAAkB,SAAS,GAAG,KAAK,GAAG,KAAK;AACtD,UAAM,eAAe,kBAAkB,CAAC;AAGxC,UAAM,aAAa,gBAAgB,UAAU,CAAC,eAAe;AAE3D,UAAI,aAAa,SAAS,WAAW,MAAM;AACzC,eAAO;AAAA,MACT;AAGA,UAAI,aAAa,cAAc,QAAW;AACxC,eAAO,QAAQ;AAAA,UACb,aAAa;AAAA,UACb,WAAW,aAAa,CAAC;AAAA,QAC3B;AAAA,MACF;AAEA,aAAO;AAAA,IACT,CAAC;AAED,QAAI,eAAe,IAAI;AAErB,wBAAkB,OAAO,GAAG,CAAC;AAC7B,sBAAgB,OAAO,YAAY,CAAC;AAAA,IACtC;AAAA,EACF;AAGA,aAAW,eAAe,mBAAmB;AAC3C,QAAI,YAAY,cAAc,QAAW;AAEvC,YAAM,iBAAiB,OAAO,OAAO,CAAC,MAAM,EAAE,SAAS,YAAY,IAAI;AACvE,UAAI,eAAe,SAAS,GAAG;AAC7B,eAAO;AAAA,UACL,SAAS,YAAY,IAAI;AAAA,QAC3B;AAAA,MACF,OAAO;AACL,eAAO,KAAK,0BAA0B,YAAY,IAAI,EAAE;AAAA,MAC1D;AAAA,IACF,OAAO;AACL,aAAO,KAAK,0BAA0B,YAAY,IAAI,EAAE;AAAA,IAC1D;AAAA,EACF;AAGA,QAAM,aAAa,gBAAgB,IAAI,CAAC,SAAS,KAAK,IAAI;AAE1D,MAAI,CAAC,QAAQ,eAAe,WAAW,SAAS,GAAG;AACjD,WAAO,KAAK,2BAA2B,WAAW,KAAK,IAAI,CAAC,EAAE;AAAA,EAChE;AAGA,QAAM,kBAAkB,SAAS,SAAS,kBAAkB;AAC5D,QAAM,QAAQ,SAAS,SAAS,IAAI,kBAAkB,SAAS,SAAS;AAGxE,MAAI,OAAO,SAAS,MAAM,QAAQ,mBAAmB,CAAC,QAAQ,cAAc;AAC1E,WAAO;AAAA,MACL,OAAO;AAAA,MACP,UAAU;AAAA,QACR,WAAW,OAAO,KAAK,IAAI;AAAA,MAC7B;AAAA,IACF;AAAA,EACF;AAEA,MAAI,UAAU,GAAK;AACjB,UAAM,YACJ,WAAW,SAAS,IAAI,iBAAiB,WAAW,KAAK,IAAI,CAAC,MAAM;AACtE,WAAO;AAAA,MACL,OAAO;AAAA,MACP,UAAU;AAAA,QACR,WAAW,iCAAiC,SAAS;AAAA,MACvD;AAAA,IACF;AAAA,EACF;AAEA,SAAO;AAAA,IACL;AAAA,IACA,UAAU;AAAA,MACR,WAAW,OAAO,KAAK,IAAI;AAAA,MAC3B,SAAS;AAAA,MACT,OAAO,SAAS;AAAA,IAClB;AAAA,EACF;AACF;;;AC9UA,SAAS,sBACP,YACQ;AACR,SAAO,WACJ;AAAA,IACC,CAAC,MACC,GAAG,EAAE,GAAG,cAAc,YAAY,EAAE,QAAQ,CAAC,SAAS,YAAY,EAAE,MAAM,CAAC;AAAA,EAC/E,EACC,KAAK,IAAI;AACd;AAiEO,SAAS,uBACd,SAAuC,CAAC,GACA;AACxC,QAAM;AAAA,IACJ,QAAQ;AAAA,IACR,aAAa;AAAA,IACb,cAAc;AAAA,IACd,QAAQ;AAAA,IACR,aAAa;AAAA,IACb,eAAe,CAAC;AAAA;AAAA,EAClB,IAAI;AAIJ,QAAM,eACJ,OAAO,UAAU,aACb,QACA,cAAc,OAAO,YAAY;AAEvC,SAAO,CAAO,SAAS;AACrB,UAAM,WAAW,KAAK,YAAY,CAAC;AACnC,UAAM,SAAS,KAAK;AAGpB,QAAI;AACJ,QAAI;AACF,eAAS,KAAK,MAAM,MAAM;AAAA,IAC5B,SAAS,OAAO;AACd,aAAO;AAAA,QACL,OAAO;AAAA,QACP,UAAU;AAAA,UACR,WAAW,mCAAmC,KAAK;AAAA,UACnD;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAGA,QAAI,OAAO,KAAK,QAAQ,EAAE,WAAW,GAAG;AACtC,aAAO;AAAA,QACL,OAAO;AAAA,QACP,UAAU;AAAA,UACR,WAAW;AAAA,QACb;AAAA,MACF;AAAA,IACF;AAGA,QACE,eAAe,QACf,OAAO,UAAU,KACjB,OAAO,UAAU,MAAM,MACvB,OAAO,UAAU,MAAM,MACvB;AACA,aAAO;AAAA,QACL,OAAO;AAAA,QACP,UAAU;AAAA,UACR,WAAW,0BAA0B,OAAO,UAAU,CAAC;AAAA,UACvD;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAGA,UAAM,UAAoB,CAAC;AAC3B,UAAM,aAAiE,CAAC;AACxE,UAAM,SAAmB,CAAC;AAG1B,eAAW,CAAC,KAAK,aAAa,KAAK,OAAO,QAAQ,QAAQ,GAAG;AAC3D,YAAM,cAAc,OAAO,GAAG;AAE9B,YAAM,UAAU,aAAa,eAAe,aAAa,GAAG;AAC5D,UAAI,SAAS;AACX,gBAAQ,KAAK,GAAG;AAAA,MAClB,OAAO;AACL,mBAAW,KAAK,EAAE,KAAK,UAAU,eAAe,QAAQ,YAAY,CAAC;AAAA,MACvE;AAAA,IACF;AAGA,UAAM,eAAe,IAAI,IAAI,OAAO,KAAK,QAAQ,CAAC;AAClD,eAAW,OAAO,OAAO,KAAK,MAAM,GAAG;AACrC,UAAI,CAAC,aAAa,IAAI,GAAG,GAAG;AAC1B,eAAO,KAAK,GAAG;AAAA,MACjB;AAAA,IACF;AAEA,QAAI,OAAO;AACT,eAAS,0BAA0B;AAAA,QACjC;AAAA,QACA,QAAQ;AAAA,QACR;AAAA,QACA;AAAA,QACA;AAAA,MACF,CAAC;AAAA,IACH;AAGA,UAAM,gBAAgB,OAAO,KAAK,QAAQ,EAAE;AAC5C,UAAM,eAAe,QAAQ;AAG7B,QAAI,cAAc,WAAW,SAAS,GAAG;AACvC,YAAMA,mBAAkB,sBAAsB,UAAU;AACxD,aAAO;AAAA,QACL,OAAO;AAAA,QACP,UAAU;AAAA,UACR,WAAW,4BAA4B,WAAW,IAAI,CAAC,MAAM,EAAE,GAAG,EAAE,KAAK,IAAI,CAAC,MAAMA,gBAAe;AAAA,QACrG;AAAA,MACF;AAAA,IACF;AAEA,QAAI,CAAC,eAAe,OAAO,SAAS,GAAG;AACrC,aAAO;AAAA,QACL,OAAO;AAAA,QACP,UAAU;AAAA,UACR,WAAW,4BAA4B,OAAO,KAAK,IAAI,CAAC;AAAA,QAC1D;AAAA,MACF;AAAA,IACF;AAGA,UAAM,QAAQ,gBAAgB,IAAI,eAAe,gBAAgB;AAEjE,QAAI,UAAU,GAAK;AACjB,YAAM,YACJ,OAAO,SAAS,IAAI,wBAAwB,OAAO,KAAK,IAAI,CAAC,MAAM;AACrE,aAAO;AAAA,QACL,OAAO;AAAA,QACP,UAAU;AAAA,UACR,WAAW,4BAA4B,SAAS;AAAA,QAClD;AAAA,MACF;AAAA,IACF;AAGA,UAAM,kBAAkB,sBAAsB,UAAU;AAExD,WAAO;AAAA,MACL;AAAA,MACA,UAAU;AAAA,QACR,WAAW,WAAW,YAAY,IAAI,aAAa,aAAa,eAAe;AAAA,QAC/E,SAAS;AAAA,QACT,OAAO;AAAA,MACT;AAAA,IACF;AAAA,EACF;AACF;;;AHnLA,OAAO,OAAO;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EA+BZ,QAAQ,SAAe,OACrB,OACA,UACA,QACA,SACA,YAAY,GACZ;AAAA;AA1HJ;AA2HI,YAAM,EAAE,MAAM,IAAI;AAElB,YAAM,aAAa,MAAM,OAAO,KAAK;AACrC,YAAM,SACJ,OAAO,eAAe,WAAW,aAAa,WAAW;AAC3D,YAAM,YACJ,OAAO,eAAe,WAAW,WAAW,YAAY;AAE1D,UAAI,SAAS,QAAQ,EAAE,OAAO,UAAU,QAAQ,UAAU,CAAC;AAC3D,UAAI,kBAAkB,SAAS;AAC7B,iBAAS,MAAM;AAAA,MACjB;AAEA,aAAO;AAAA,QACL,QAAO,YAAO,UAAP,YAAgB,MAAM;AAAA,QAC7B,SAAS,MAAM,aAAa,CAAC,iCAAK,SAAL,EAAa,MAAM,QAAQ,KAAK,EAAC,CAAC;AAAA,MACjE;AAAA,IACF;AAAA;AACF,CAAC;AAuDM,SAAS,aACd,MACA;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA,YAAY;AAAA;AAAA;AAAA,EAGZ,UAAU;AACZ,GAQA;AACA,SAAO,SAAS,MAAM,MAAY;AAChC,UAAM,SAAS,SAAS,KAAK,OAAO,OAAO,CAAC,IAAI;AAEhD,eAAW,MAAwB,MAAM,KAAK,GAAG;AAA5C,qBAAQ,QA3NjB,IA2NS,IAAkB,mBAAlB,IAAkB,CAAV;AACX;AAAA,QACE;AAAA,QACA;AAAA,UACE;AAAA,QACF;AAAA,QACA,CAAO,OAAuB,eAAvB,KAAuB,WAAvB,EAAE,MAAM,SAAS,GAAM;AAC5B,gBAAM,aAAa,MAAM,KAAK,KAAK;AACnC,gBAAM,SACJ,OAAO,eAAe,WAAW,aAAa,WAAW;AAC3D,gBAAM,YACJ,OAAO,eAAe,WAAW,WAAW,YAAY;AAE1D,gBAAM,SAAS,MAAM,QAAQ;AAAA,YAC3B,QAAQ,IAAI,CAAC,WAAW;AACtB,oBAAM,SAAS,OAAO,+BAAE,SAAU,SAAZ,EAAoB,QAAQ,UAAU,EAAC;AAC7D,kBAAI,kBAAkB,SAAS;AAC7B,uBAAO;AAAA,cACT;AACA,qBAAO,IAAI,QAAe,CAAC,YAAY,QAAQ,MAAM,CAAC;AAAA,YACxD,CAAC;AAAA,UACH;AACA,gBAAM,iBAAiB,OAAO,IAAI,CAAC,GAAG,MAAO,iCACxC,IADwC;AAAA,YAE3C,MAAM,QAAQ,CAAC,EAAE;AAAA,UACnB,EAAE;AAEF,gBAAM,WACJ,OAAO,OAAO,CAAC,KAAK,MAAG;AAvPnC,gBAAAC;AAuPsC,2BAAOA,MAAA,EAAE,UAAF,OAAAA,MAAW;AAAA,aAAI,CAAC,IAAI,OAAO;AAE9D,mBAAS,KAAK,OAAO;AAAA,YACnB,QAAQ;AAAA,YACR;AAAA,aACI,aAAa,EAAE,UAAU;AAG/B,cAAI,WAAW;AACb;AAAA,cACE,YAAY;AAAA,cACZ,UAAU,QAAQ,qBAAqB,SAAS;AAAA;AAAA;AAAA,EAAmB,SAAS,MAAM,CAAC;AAAA;AAAA,EAAO;AAAA,gBACxF;AAAA,cACF,CAAC;AAAA,YACH;AAAA,UACF;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAAA,EACF,EAAC;AACH;AAEO,SAAS,aAAa,QAAsC;AACjE,SAAO,OACJ,KAAK,CAAC,GAAG,MAAG;AA/QjB;AA+QqB,oBAAE,UAAF,YAAW,OAAM,OAAE,UAAF,YAAW;AAAA,GAAE,EAC9C,IAAI,CAAC,MAAM;AAhRhB;AAiRM,UAAM,YAAY,KAAK,EAAE,QAAQ,SAAS,OAAM,OAAE,UAAF,YAAW,GAAG,QAAQ,CAAC,CAAC;AACxE,UACI,OAAE,UAAF,YAAW,KAAK,OAAO,OAAE,aAAF,mBAAY,gBACrC,OAAE,aAAF,mBAAY,SACZ;AAEA,UAAI,kBAAkB;AACtB,YAAI,OAAE,aAAF,mBAAY,YAAW,QAAW;AACpC,cAAM,SAAS,EAAE,SAAS;AAC1B,YAAI,OAAO,WAAW,UAAU;AAC9B,4BAAkB;AAAA;AAAA;AAAA;AAAA,EAAsB,SAAS,MAAM,CAAC;AAAA,QAC1D,OAAO;AAEL,4BAAkB;AAAA;AAAA;AAAA;AAAA,EAAsB,SAAS,KAAK,UAAU,QAAQ,MAAM,CAAC,CAAC,CAAC;AAAA,QACnF;AAAA,MACF;AAEA,aAAO,GAAG,SAAS,KACjB,OAAE,aAAF,mBAAY,aACR;AAAA;AAAA;AAAA;AAAA,EAAuB,SAAS,EAAE,SAAS,SAAS,CAAC,KACrD,EACN,GAAG,eAAe;AAAA,IACpB;AACA,WAAO;AAAA,EACT,CAAC,EACA,KAAK,MAAM;AAChB;AAsBO,SAAS,SAAS,MAAc,QAAQ,IAAY;AACzD,MAAI,CAAC,QAAQ,KAAK,UAAU,OAAO;AACjC,WAAO;AAAA,EACT;AAEA,QAAM,QAAQ,KAAK,MAAM,KAAK;AAC9B,QAAM,QAAkB,CAAC;AACzB,MAAI,cAAc;AAElB,aAAW,QAAQ,OAAO;AAExB,QAAI,YAAY,SAAS,KAAK,SAAS,IAAI,OAAO;AAChD,YAAM,KAAK,YAAY,KAAK,CAAC;AAC7B,oBAAc;AAAA,IAChB,OAAO;AAEL,sBAAgB,cAAc,MAAM,MAAM;AAAA,IAC5C;AAAA,EACF;AAGA,MAAI,aAAa;AACf,UAAM,KAAK,WAAW;AAAA,EACxB;AAEA,SAAO,MAAM,KAAK,IAAI;AACxB;","names":["mismatchDetails","_a"]}
+{"version":3,"sources":["../src/index.ts","../src/wrapText.ts","../src/scorers/utils.ts","../src/scorers/toolCallScorer.ts","../src/scorers/structuredOutputScorer.ts"],"sourcesContent":["import {\n  assert,\n  beforeEach as vitestBeforeEach,\n  afterEach as vitestAfterEach,\n  describe,\n  expect,\n  test,\n} from \"vitest\";\nimport \"vitest\";\nimport { wrapText } from \"./wrapText\";\n\n/**\n * Represents a tool/function call made during task execution.\n * Supports various LLM provider formats and use cases.\n */\nexport type ToolCall = {\n  // Core fields (required for basic usage)\n  name: string;\n  arguments?: Record<string, any>;\n\n  // Additional metadata\n  [key: string]: any; // Allow provider-specific fields\n};\n\nexport type TaskResult = {\n  result: string;\n  toolCalls?: ToolCall[];\n};\n\n/**\n * Task function that processes an input and returns either a string result\n * or a TaskResult object containing the result and any tool calls made.\n *\n * @param input - The input string to process\n * @returns Promise resolving to either a string or TaskResult object\n *\n * @example\n * // Simple tasks can just return a string\n * const simpleTask: TaskFn = async (input) => \"The answer is 42\";\n *\n * // Tasks that use tools should return TaskResult\n * const taskWithTools: TaskFn = async (input) => ({\n *   result: \"The answer is 42\",\n *   toolCalls: [{ name: \"calculate\", arguments: { expr: \"6*7\" }, result: 42 }]\n * });\n */\nexport type TaskFn = (input: string) => Promise<string | TaskResult>;\n\nexport type Score = {\n  score: number | null;\n  metadata?: {\n    rationale?: string;\n    output?: any;\n  } & Record<string, any>;\n};\n\nexport interface BaseScorerOptions {\n  input: string;\n  output: string;\n  toolCalls?: ToolCall[];\n}\n\nexport type ScoreFn<TOptions extends BaseScorerOptions = BaseScorerOptions> = (\n  opts: TOptions,\n) => Promise<Score> | Score;\n\n/**\n * @deprecated Use describeEval() instead for better test organization and multiple scorers support\n */\nexport type ToEval<R = unknown> = (\n  expected: any,\n  taskFn: TaskFn,\n  scoreFn: ScoreFn<any>,\n  threshold?: number,\n) => Promise<R>;\n\nexport interface EvalMatchers<R = unknown> {\n  toEval: ToEval<R>;\n}\n\ndeclare module \"vitest\" {\n  interface Assertion<T = any> extends EvalMatchers<T> {}\n  interface AsymmetricMatchersContaining extends EvalMatchers {}\n\n  interface TaskMeta {\n    eval?: {\n      scores: (Score & { name: string })[];\n      avgScore: number;\n      toolCalls?: ToolCall[];\n    };\n  }\n}\n\nexpect.extend({\n  /**\n   * Evaluates a language model output against an expected answer using a scoring function.\n   *\n   * @deprecated Use describeEval() instead for better test organization and multiple scorers support\n   * @param expected - The expected (ground truth) answer, can be any type depending on the scorer\n   * @param taskFn - Async function that processes the input and returns the model output\n   *                 Can return either a string or TaskResult object with result and optional toolCalls\n   * @param scoreFn - Function that evaluates the model output against the expected answer\n   * @param threshold - Minimum acceptable score (0-1), defaults to 1.0\n   *\n   * @example\n   * ```javascript\n   * test(\"checks capital of France\", async () => {\n   *   expect(\"What is the capital of France?\").toEval(\n   *     \"Paris\",\n   *     async (input) => {\n   *       const response = await queryLLM(input);\n   *       // Recommended: return TaskResult\n   *       return {\n   *         result: response.text,\n   *         toolCalls: response.toolCalls || []\n   *       };\n   *     },\n   *     checkFactuality,\n   *     0.8\n   *   );\n   * });\n   * ```\n   */\n  // TODO: this needs to be support true extensibility with Eval scorers\n  toEval: async function toEval(\n    input: string,\n    expected: any,\n    taskFn: TaskFn,\n    scoreFn: ScoreFn<any>,\n    threshold = 1.0,\n  ) {\n    const { isNot } = this;\n\n    const taskOutput = await taskFn(input);\n    const output =\n      typeof taskOutput === \"string\" ? taskOutput : taskOutput.result;\n    const toolCalls =\n      typeof taskOutput === \"object\" ? taskOutput.toolCalls : undefined;\n\n    let result = scoreFn({ input, expected, output, toolCalls });\n    if (result instanceof Promise) {\n      result = await result;\n    }\n\n    return {\n      pass: (result.score ?? 0) >= threshold,\n      message: () => formatScores([{ ...result, name: scoreFn.name }]),\n    };\n  },\n});\n\n/**\n * Creates a test suite for evaluating language model outputs.\n *\n * @param name - The name of the test suite\n * @param options - Configuration options\n * @param options.data - Async function that returns an array of test cases with input and any additional fields\n * @param options.task - Function that processes the input and returns the model output\n *                       Can return either a string or TaskResult object with result and optional toolCalls\n * @param options.skipIf - Optional function that determines if tests should be skipped\n * @param options.scorers - Array of scoring functions that evaluate model outputs\n * @param options.threshold - Minimum acceptable average score (0-1), defaults to 1.0\n * @param options.timeout - Test timeout in milliseconds, defaults to 60000 (60s)\n *\n * @example\n * ```javascript\n * // Recommended: TaskResult format with tool tracking\n * describeEval(\"capital cities test\", {\n *   data: async () => [{\n *     input: \"What is the capital of France?\",\n *     expected: \"Paris\"\n *   }],\n *   task: async (input) => {\n *     const response = await queryLLM(input);\n *     return {\n *       result: response.text,\n *       toolCalls: response.toolCalls || []\n *     };\n *   },\n *   scorers: [checkFactuality],\n *   threshold: 0.8\n * });\n *\n * // Example with tool usage evaluation\n * describeEval(\"tool usage test\", {\n *   data: async () => [{\n *     input: \"Search for weather in Seattle\",\n *     expectedTools: [{ name: \"weather_api\", arguments: { location: \"Seattle\" } }]\n *   }],\n *   task: async (input) => {\n *     return {\n *       result: \"The weather in Seattle is 65°F\",\n *       toolCalls: [{\n *         name: \"weather_api\",\n *         arguments: { location: \"Seattle\" },\n *         result: { temp: 65, condition: \"partly cloudy\" }\n *       }]\n *     };\n *   },\n *   scorers: [ToolCallScorer()],\n *   threshold: 1.0\n * });\n * ```\n */\nexport function describeEval(\n  name: string,\n  {\n    data,\n    task,\n    skipIf,\n    scorers,\n    threshold = 1.0,\n    // increase default test timeout as 5s is usually not enough for\n    // a single factuality check\n    timeout = 60000,\n    beforeEach: beforeEachHook,\n    afterEach: afterEachHook,\n  }: {\n    data: () => Promise<\n      Array<{ input: string; name?: string } & Record<string, any>>\n    >;\n    task: TaskFn;\n    skipIf?: () => boolean;\n    scorers: ScoreFn<any>[];\n    threshold?: number | null;\n    timeout?: number;\n    beforeEach?: () => void | Promise<void>;\n    afterEach?: () => void | Promise<void>;\n  },\n) {\n  return describe(name, async () => {\n    if (beforeEachHook) {\n      vitestBeforeEach(beforeEachHook);\n    }\n    if (afterEachHook) {\n      vitestAfterEach(afterEachHook);\n    }\n\n    const testFn = skipIf ? test.skipIf(skipIf()) : test;\n    // TODO: should data just be a generator?\n    for (const { input, name: testName, ...params } of await data()) {\n      testFn(\n        testName ?? input,\n        {\n          timeout,\n        },\n        async ({ task: testTask }) => {\n          const taskOutput = await task(input);\n          const output =\n            typeof taskOutput === \"string\" ? taskOutput : taskOutput.result;\n          const toolCalls =\n            typeof taskOutput === \"object\" ? taskOutput.toolCalls : undefined;\n\n          const scores = await Promise.all(\n            scorers.map((scorer) => {\n              const result = scorer({ input, ...params, output, toolCalls });\n              if (result instanceof Promise) {\n                return result;\n              }\n              return new Promise<Score>((resolve) => resolve(result));\n            }),\n          );\n          const scoresWithName = scores.map((s, i) => ({\n            ...s,\n            name: scorers[i].name,\n          }));\n\n          const avgScore =\n            scores.reduce((acc, s) => acc + (s.score ?? 0), 0) / scores.length;\n\n          testTask.meta.eval = {\n            scores: scoresWithName,\n            avgScore,\n            ...(toolCalls && { toolCalls }),\n          };\n\n          if (threshold) {\n            assert(\n              avgScore >= threshold,\n              `Score: ${avgScore} below threshold: ${threshold}\\n\\n## Output:\\n${wrapText(output)}\\n\\n${formatScores(\n                scoresWithName,\n              )}`,\n            );\n          }\n        },\n      );\n    }\n  });\n}\n\nexport function formatScores(scores: (Score & { name: string })[]) {\n  return scores\n    .sort((a, b) => (a.score ?? 0) - (b.score ?? 0))\n    .map((s) => {\n      const scoreLine = `# ${s.name || \"Unknown\"} [${(s.score ?? 0).toFixed(1)}]`;\n      if (\n        ((s.score ?? 0) < 1.0 && s.metadata?.rationale) ||\n        s.metadata?.output\n      ) {\n        // Format output - handle both strings and objects\n        let formattedOutput = \"\";\n        if (s.metadata?.output !== undefined) {\n          const output = s.metadata.output;\n          if (typeof output === \"string\") {\n            formattedOutput = `\\n\\n## Response\\n\\n${wrapText(output)}`;\n          } else {\n            // For objects, stringify with proper formatting\n            formattedOutput = `\\n\\n## Response\\n\\n${wrapText(JSON.stringify(output, null, 2))}`;\n          }\n        }\n\n        return `${scoreLine}${\n          s.metadata?.rationale\n            ? `\\n\\n## Rationale\\n\\n${wrapText(s.metadata.rationale)}`\n            : \"\"\n        }${formattedOutput}`;\n      }\n      return scoreLine;\n    })\n    .join(\"\\n\\n\");\n}\n\nexport { wrapText } from \"./wrapText\";\n\n// Export built-in scorers\nexport {\n  ToolCallScorer,\n  type ToolCallScorerOptions,\n  StructuredOutputScorer,\n  type StructuredOutputScorerOptions,\n} from \"./scorers\";\n","/**\n * Wraps text to fit within a specified width, breaking at word boundaries.\n *\n * @param text - The text to wrap\n * @param width - The maximum width in characters (default: 80)\n * @returns The wrapped text with line breaks\n *\n * @example\n * ```javascript\n * const wrapped = wrapText(\"This is a very long text that needs to be wrapped to fit within an 80 character width.\", 20);\n * console.log(wrapped);\n * // Output:\n * // This is a very\n * // long text that\n * // needs to be\n * // wrapped to fit\n * // within an 80\n * // character width.\n * ```\n */\nexport function wrapText(text: string, width = 80): string {\n  if (!text || text.length <= width) {\n    return text;\n  }\n\n  const words = text.split(/\\s+/);\n  const lines: string[] = [];\n  let currentLine = \"\";\n\n  for (const word of words) {\n    // If adding this word would exceed the width, start a new line\n    if (currentLine.length + word.length + 1 > width) {\n      lines.push(currentLine.trim());\n      currentLine = word;\n    } else {\n      // Add the word to the current line\n      currentLine += (currentLine ? \" \" : \"\") + word;\n    }\n  }\n\n  // Add the last line if it's not empty\n  if (currentLine) {\n    lines.push(currentLine);\n  }\n\n  return lines.join(\"\\n\");\n}\n","/**\n * Shared utilities for scorer implementations\n */\n\n/**\n * Common configuration options for scorers\n */\nexport interface BaseMatcherConfig {\n  /**\n   * Whether all expected items must match for a passing score\n   * When false: gives partial credit based on items matched\n   * @default true\n   */\n  requireAll?: boolean;\n\n  /**\n   * Whether to allow additional items beyond those expected\n   * @default true\n   */\n  allowExtras?: boolean;\n\n  /**\n   * Enable debug logging\n   * @default false\n   */\n  debug?: boolean;\n}\n\n/**\n * Matching strategy type\n */\nexport type MatchStrategy<T = any> =\n  | \"strict\"\n  | \"fuzzy\"\n  | ((expected: T, actual: T, context?: string) => boolean);\n\n/**\n * Options for fuzzy matching behavior\n */\nexport interface FuzzyMatchOptions {\n  /**\n   * Allow case-insensitive string matching\n   * @default true\n   */\n  caseInsensitive?: boolean;\n\n  /**\n   * For strings: use substring matching instead of exact match\n   * When enabled, either string can contain the other as a substring:\n   * - \"weather\" matches \"weath\" (expected contains actual)\n   * - \"weather forecast\" matches \"weather\" (actual contains expected)\n   * @default false\n   */\n  substring?: boolean;\n\n  /**\n   * For numbers: tolerance for comparison (0.001 = 0.1%)\n   * @default 0.001\n   */\n  numericTolerance?: number;\n\n  /**\n   * For arrays: ignore order when comparing\n   * @default true\n   */\n  ignoreArrayOrder?: boolean;\n\n  /**\n   * Allow type coercion (e.g., \"42\" matches 42)\n   * @default false\n   */\n  coerceTypes?: boolean;\n}\n\n/**\n * Strict equality comparison (deep equals)\n */\nexport function strictEquals(\n  expected: any,\n  actual: any,\n  context?: string,\n): boolean {\n  // Handle primitive types and null/undefined\n  if (expected === actual) return true;\n  if (\n    expected === null ||\n    expected === undefined ||\n    actual === null ||\n    actual === undefined\n  ) {\n    return false;\n  }\n\n  // Must be same type\n  if (typeof expected !== typeof actual) return false;\n\n  // Handle arrays\n  if (Array.isArray(expected)) {\n    if (!Array.isArray(actual)) return false;\n    if (expected.length !== actual.length) return false;\n    return expected.every((item, i) => strictEquals(item, actual[i], context));\n  }\n\n  // Handle objects\n  if (typeof expected === \"object\") {\n    const expectedKeys = Object.keys(expected);\n    const actualKeys = Object.keys(actual);\n\n    // Must have same number of keys\n    if (expectedKeys.length !== actualKeys.length) return false;\n\n    // All expected keys must exist in actual and have matching values\n    return expectedKeys.every(\n      (key) =>\n        key in actual && strictEquals(expected[key], actual[key], context),\n    );\n  }\n\n  // All other primitive types already handled by line 84\n  return false;\n}\n\n/**\n * Fuzzy string matching with configurable options\n */\nfunction fuzzyMatchString(\n  expected: string,\n  actual: string,\n  options: FuzzyMatchOptions,\n): boolean {\n  const { caseInsensitive = true, substring = false } = options;\n\n  const expectedStr = caseInsensitive ? expected.toLowerCase() : expected;\n  const actualStr = caseInsensitive ? actual.toLowerCase() : actual;\n\n  return substring\n    ? actualStr.includes(expectedStr) || expectedStr.includes(actualStr)\n    : expectedStr === actualStr;\n}\n\n/**\n * Fuzzy number matching with tolerance\n */\nfunction fuzzyMatchNumber(\n  expected: number,\n  actual: number,\n  options: FuzzyMatchOptions,\n): boolean {\n  const { numericTolerance = 0.001 } = options;\n\n  const tolerance = Math.max(\n    Math.abs(expected) * numericTolerance,\n    numericTolerance,\n  );\n  return Math.abs(expected - actual) <= tolerance;\n}\n\n/**\n * Fuzzy array matching with optional order independence\n */\nfunction fuzzyMatchArray(\n  expected: any[],\n  actual: any[],\n  options: FuzzyMatchOptions,\n  context?: string,\n): boolean {\n  const { ignoreArrayOrder = true } = options;\n\n  if (ignoreArrayOrder) {\n    // Track which actual items have been consumed\n    const actualUsed = actual.map(() => false);\n\n    // Try to find a unique match for each expected item\n    return expected.every((expItem) => {\n      // Find first unused actual item that matches\n      for (let i = 0; i < actual.length; i++) {\n        if (actualUsed[i]) continue; // Already used\n\n        if (fuzzyMatch(expItem, actual[i], options, context)) {\n          actualUsed[i] = true; // Mark as used\n          return true;\n        }\n      }\n      return false; // No match found\n    });\n  }\n\n  return (\n    expected.length === actual.length &&\n    expected.every((item, i) => fuzzyMatch(item, actual[i], options, context))\n  );\n}\n\n/**\n * Fuzzy object matching (subset matching)\n */\nfunction fuzzyMatchObject(\n  expected: object,\n  actual: object,\n  options: FuzzyMatchOptions,\n  context?: string,\n): boolean {\n  return Object.entries(expected).every(\n    ([k, value]) =>\n      k in actual && fuzzyMatch(value, (actual as any)[k], options, context),\n  );\n}\n\n/**\n * Type coercion matching\n */\nfunction fuzzyMatchWithCoercion(\n  expected: any,\n  actual: any,\n  options: FuzzyMatchOptions,\n): boolean {\n  // Boolean coercion\n  if (typeof expected === \"boolean\" && typeof actual === \"string\") {\n    return expected === (actual.toLowerCase() === \"true\" || actual === \"1\");\n  }\n\n  // Number-string coercion\n  if (typeof expected === \"string\" && typeof actual === \"number\") {\n    return Number.parseFloat(expected) === actual;\n  }\n  if (typeof expected === \"number\" && typeof actual === \"string\") {\n    return expected === Number.parseFloat(actual);\n  }\n\n  return false;\n}\n\n/**\n * Fuzzy matching with flexible comparison logic\n */\nexport function fuzzyMatch(\n  expected: any,\n  actual: any,\n  options: FuzzyMatchOptions = {},\n  context?: string,\n): boolean {\n  const { coerceTypes = false } = options;\n\n  // Handle regex patterns\n  if (expected instanceof RegExp) {\n    return typeof actual === \"string\" && expected.test(actual);\n  }\n\n  // Handle functions (custom validators)\n  if (typeof expected === \"function\") {\n    return expected(actual);\n  }\n\n  // Null/undefined handling\n  if (\n    expected === null ||\n    expected === undefined ||\n    actual === null ||\n    actual === undefined\n  ) {\n    return expected === actual;\n  }\n\n  // Type-specific matching\n  if (typeof expected === \"string\" && typeof actual === \"string\") {\n    return fuzzyMatchString(expected, actual, options);\n  }\n\n  if (typeof expected === \"number\" && typeof actual === \"number\") {\n    return fuzzyMatchNumber(expected, actual, options);\n  }\n\n  if (Array.isArray(expected) && Array.isArray(actual)) {\n    return fuzzyMatchArray(expected, actual, options, context);\n  }\n\n  if (\n    typeof expected === \"object\" &&\n    typeof actual === \"object\" &&\n    !Array.isArray(expected) &&\n    !Array.isArray(actual)\n  ) {\n    return fuzzyMatchObject(expected, actual, options, context);\n  }\n\n  // Handle type coercion if enabled\n  if (coerceTypes) {\n    const coercionResult = fuzzyMatchWithCoercion(expected, actual, options);\n    if (coercionResult) return true;\n  }\n\n  // For all other cases, strict equality\n  return expected === actual;\n}\n\n/**\n * Create a matcher function based on strategy\n */\nexport function createMatcher<T = any>(\n  strategy: MatchStrategy<T>,\n  options?: FuzzyMatchOptions,\n): (expected: T, actual: T, context?: string) => boolean {\n  if (typeof strategy === \"function\") {\n    return (expected, actual, context) => strategy(expected, actual, context);\n  }\n\n  if (strategy === \"strict\") {\n    return (expected, actual, context) =>\n      strictEquals(expected, actual, context);\n  }\n\n  return (expected, actual, context) =>\n    fuzzyMatch(expected, actual, options || {}, context);\n}\n\n/**\n * Format a value for display in error messages\n */\nexport function formatValue(value: any): string {\n  if (value === undefined) return \"undefined\";\n  if (value === null) return \"null\";\n  if (value instanceof RegExp) return value.toString();\n  if (typeof value === \"string\") return `\"${value}\"`;\n  if (typeof value === \"object\") return JSON.stringify(value);\n  return String(value);\n}\n\n/**\n * Calculate partial score based on matches\n */\nexport function calculatePartialScore(\n  matched: number,\n  total: number,\n  requireAll: boolean,\n): number {\n  if (requireAll && matched < total) {\n    return 0.0;\n  }\n  return total > 0 ? matched / total : 1.0;\n}\n\n/**\n * Logger interface for debug output\n */\nexport interface Logger {\n  log: (message: string, ...args: any[]) => void;\n}\n\n/**\n * Default logger that respects NODE_ENV and can be disabled\n */\nconst defaultLogger: Logger = {\n  log: (message: string, ...args: any[]) => {\n    // Only log in development or test environments, or when explicitly enabled\n    if (\n      process.env.NODE_ENV === \"development\" ||\n      process.env.NODE_ENV === \"test\" ||\n      process.env.VITEST_EVALS_DEBUG === \"true\"\n    ) {\n      console.log(message, ...args);\n    }\n  },\n};\n\n/**\n * Helper for debugging matcher results with configurable logging\n */\nexport function debugLog(\n  context: string,\n  data: {\n    expected: any;\n    actual: any;\n    matches?: string[];\n    mismatches?: Array<{ key: string; expected: any; actual: any }>;\n    extras?: string[];\n  },\n  logger: Logger = defaultLogger,\n): void {\n  logger.log(`${context} debug:`);\n  logger.log(\"Expected:\", data.expected);\n  logger.log(\"Actual:\", data.actual);\n  if (data.matches) logger.log(\"Matches:\", data.matches);\n  if (data.mismatches) logger.log(\"Mismatches:\", data.mismatches);\n  if (data.extras) logger.log(\"Extras:\", data.extras);\n}\n","import type { ScoreFn, BaseScorerOptions, ToolCall } from \"../index\";\nimport {\n  type BaseMatcherConfig,\n  type MatchStrategy,\n  type FuzzyMatchOptions,\n  createMatcher,\n} from \"./utils\";\n\nexport interface ToolCallScorerOptions extends BaseScorerOptions {\n  // Expected tools are now defined in the test data\n  expectedTools?: Array<{\n    name: string;\n    arguments?: any;\n  }>;\n}\n\nexport interface ToolCallScorerConfig extends BaseMatcherConfig {\n  /**\n   * Whether tools must be called in the exact order specified\n   * @default false\n   */\n  ordered?: boolean;\n\n  /**\n   * How to match tool arguments/parameters\n   * - \"strict\": Exact equality required (default)\n   * - \"fuzzy\": Flexible matching with tolerance for differences\n   *   - Case-insensitive string matching\n   *   - Numeric tolerance for small differences\n   *   - Unordered array comparison\n   *   - Subset matching for objects (actual can have extra properties)\n   * - Custom function: Your own comparison logic\n   *\n   * NOTE: Each expected tool call requires a unique actual tool call to match.\n   * Multiple identical expected tools need separate actual tool calls.\n   *\n   * @default \"strict\"\n   */\n  params?: MatchStrategy;\n\n  /**\n   * Options for fuzzy matching when params=\"fuzzy\"\n   * These options are MERGED with defaults, not replaced.\n   *\n   * Default fuzzy options for tool calls:\n   * - substring: true (allow substring matching for strings)\n   * - caseInsensitive: true (ignore case differences)\n   * - ignoreArrayOrder: true (arrays can be in different orders)\n   * - numericTolerance: 0.001 (0.1% tolerance for numbers)\n   * - coerceTypes: false (no automatic type conversion)\n   *\n   * @default { substring: true, caseInsensitive: true, ignoreArrayOrder: true }\n   */\n  fuzzyOptions?: FuzzyMatchOptions;\n}\n\n/**\n * A configurable scorer for evaluating tool usage in LLM responses.\n *\n * The test data defines WHAT tools/arguments are expected,\n * while this scorer defines HOW to evaluate them.\n *\n * @param config - Configuration options for the scorer\n * @param config.ordered - Require exact order of tool calls\n * @param config.requireAll - Require all expected tools (vs partial credit)\n * @param config.allowExtras - Allow additional tool calls\n * @param config.params - How to match parameters: \"strict\", \"fuzzy\", or custom function\n *\n * @example\n * // Default: strict params, any order\n * describeEval(\"search test\", {\n *   data: async () => [{\n *     input: \"Find restaurants\",\n *     expectedTools: [\n *       { name: \"search\", arguments: { type: \"restaurant\" } },\n *       { name: \"filter\" }\n *     ]\n *   }],\n *   task: myTask,\n *   scorers: [ToolCallScorer()]\n * });\n *\n * @example\n * // Strict order and parameters\n * describeEval(\"payment flow\", {\n *   data: async () => [{\n *     input: \"Process payment\",\n *     expectedTools: [\n *       { name: \"validate\", arguments: { amount: 100 } },\n *       { name: \"charge\", arguments: { amount: 100, method: \"card\" } }\n *     ]\n *   }],\n *   task: myTask,\n *   scorers: [ToolCallScorer({ ordered: true, params: \"strict\" })]\n * });\n */\nexport function ToolCallScorer(\n  config: ToolCallScorerConfig = {},\n): ScoreFn<ToolCallScorerOptions> {\n  const {\n    ordered = false,\n    requireAll = true,\n    allowExtras = true,\n    params = \"strict\",\n    fuzzyOptions: userFuzzyOptions,\n  } = config;\n\n  // Merge user fuzzyOptions with defaults for tool calls\n  const defaultFuzzyOptions: FuzzyMatchOptions = {\n    substring: true,\n    caseInsensitive: true,\n    ignoreArrayOrder: true,\n    numericTolerance: 0.001,\n    coerceTypes: false,\n  };\n  const fuzzyOptions: FuzzyMatchOptions = {\n    ...defaultFuzzyOptions,\n    ...userFuzzyOptions,\n  };\n\n  // Determine the argument matcher\n  const argMatcher = createMatcher(params, fuzzyOptions);\n\n  return async (opts) => {\n    const expectedTools = opts.expectedTools || [];\n    const actualCalls = opts.toolCalls || [];\n\n    // No expectations means pass\n    if (expectedTools.length === 0) {\n      return {\n        score: 1.0,\n        metadata: {\n          rationale: \"No tool calls expected\",\n        },\n      };\n    }\n\n    // No actual calls when we expected some\n    if (actualCalls.length === 0) {\n      return {\n        score: 0.0,\n        metadata: {\n          rationale: `Expected ${expectedTools.length} tool(s) but none were called`,\n        },\n      };\n    }\n\n    if (ordered) {\n      return evaluateOrderedTools(expectedTools, actualCalls, {\n        argMatcher,\n        allowExtras,\n        requireAllTools: requireAll,\n      });\n    }\n\n    return evaluateUnorderedTools(expectedTools, actualCalls, {\n      argMatcher,\n      requireAllTools: requireAll,\n      allowExtras,\n    });\n  };\n}\n\n/**\n * Evaluate tools that must be called in a specific order\n */\nfunction evaluateOrderedTools(\n  expected: Array<{ name: string; arguments?: any }>,\n  actual: ToolCall[],\n  options: {\n    argMatcher: (expected: any, actual: any) => boolean;\n    allowExtras: boolean;\n    requireAllTools: boolean;\n  },\n) {\n  let expectedIndex = 0;\n  let actualIndex = 0;\n\n  // Match expected tools in order\n  while (expectedIndex < expected.length && actualIndex < actual.length) {\n    const exp = expected[expectedIndex];\n    const act = actual[actualIndex];\n\n    if (exp.name === act.name) {\n      // Check arguments if specified\n      if (exp.arguments !== undefined) {\n        const argsMatch = options.argMatcher(\n          exp.arguments,\n          act.arguments || {},\n        );\n        if (!argsMatch) {\n          // Give partial credit for tools matched up to this point\n          const partialScore = expectedIndex / expected.length;\n          return {\n            score: partialScore,\n            metadata: {\n              rationale: `Tool '${exp.name}' called with incorrect arguments at position ${expectedIndex + 1} (${expectedIndex}/${expected.length} tools matched correctly)`,\n              expected: exp.arguments,\n              actual: act.arguments,\n              matched: expectedIndex,\n              total: expected.length,\n            },\n          };\n        }\n      }\n      expectedIndex++;\n      actualIndex++;\n    } else if (options.allowExtras) {\n      // Skip extra tool\n      actualIndex++;\n    } else {\n      // Wrong tool in sequence when extra tools not allowed\n      return {\n        score: 0.0,\n        metadata: {\n          rationale: `Expected '${exp.name}' at position ${expectedIndex + 1} but found '${act.name}'`,\n        },\n      };\n    }\n  }\n\n  // Check if all expected tools were matched\n  if (expectedIndex < expected.length) {\n    const missing = expected.slice(expectedIndex).map((t) => t.name);\n\n    if (options.requireAllTools) {\n      return {\n        score: 0.0,\n        metadata: {\n          rationale: `Missing required tools in sequence: ${missing.join(\", \")}`,\n        },\n      };\n    }\n\n    // Partial credit when requireAllTools is false\n    const matchedCount = expectedIndex;\n    const totalCount = expected.length;\n    const score = totalCount > 0 ? matchedCount / totalCount : 1.0;\n\n    return {\n      score,\n      metadata: {\n        rationale: `Partial match: ${matchedCount}/${totalCount} tools called in order (missing: ${missing.join(\", \")})`,\n        matched: matchedCount,\n        total: totalCount,\n      },\n    };\n  }\n\n  // Check for extra tools at the end if not allowed\n  if (!options.allowExtras && actualIndex < actual.length) {\n    const extra = actual.slice(actualIndex).map((t) => t.name);\n    return {\n      score: 0.0,\n      metadata: {\n        rationale: `Unexpected extra tools: ${extra.join(\", \")}`,\n      },\n    };\n  }\n\n  return {\n    score: 1.0,\n    metadata: {\n      rationale: \"All tools called in expected order with correct arguments\",\n    },\n  };\n}\n\n/**\n * Evaluate tools that can be called in any order\n *\n * Simple logic:\n * 1. Start with copies of expected and actual tool arrays\n * 2. For each expected tool, find and remove a matching actual tool\n * 3. Remaining expected = missing, remaining actual = extras\n */\nfunction evaluateUnorderedTools(\n  expected: Array<{ name: string; arguments?: any }>,\n  actual: ToolCall[],\n  options: {\n    argMatcher: (expected: any, actual: any) => boolean;\n    requireAllTools: boolean;\n    allowExtras: boolean;\n  },\n) {\n  // Work with copies so we can remove items\n  const remainingExpected = [...expected];\n  const remainingActual = [...actual];\n  const issues: string[] = [];\n\n  // For each expected tool, find and remove a matching actual tool\n  for (let i = remainingExpected.length - 1; i >= 0; i--) {\n    const expectedTool = remainingExpected[i];\n\n    // Find a matching actual tool\n    const matchIndex = remainingActual.findIndex((actualTool) => {\n      // Check if this actual tool matches the expected tool\n      if (expectedTool.name !== actualTool.name) {\n        return false;\n      }\n\n      // Check arguments if specified\n      if (expectedTool.arguments !== undefined) {\n        return options.argMatcher(\n          expectedTool.arguments,\n          actualTool.arguments || {},\n        );\n      }\n\n      return true;\n    });\n\n    if (matchIndex !== -1) {\n      // Found a match - remove both\n      remainingExpected.splice(i, 1);\n      remainingActual.splice(matchIndex, 1);\n    }\n  }\n\n  // Generate issues for missing tools\n  for (const missingTool of remainingExpected) {\n    if (missingTool.arguments !== undefined) {\n      // Check if tool was called but with wrong args\n      const wrongArgsCalls = actual.filter((a) => a.name === missingTool.name);\n      if (wrongArgsCalls.length > 0) {\n        issues.push(\n          `Tool '${missingTool.name}' called but with incorrect arguments`,\n        );\n      } else {\n        issues.push(`Missing required tool: ${missingTool.name}`);\n      }\n    } else {\n      issues.push(`Missing required tool: ${missingTool.name}`);\n    }\n  }\n\n  // Extra tools = remaining actual tools\n  const extraTools = remainingActual.map((tool) => tool.name);\n\n  if (!options.allowExtras && extraTools.length > 0) {\n    issues.push(`Unexpected extra tools: ${extraTools.join(\", \")}`);\n  }\n\n  // Calculate score\n  const expectedMatched = expected.length - remainingExpected.length;\n  const score = expected.length > 0 ? expectedMatched / expected.length : 1.0;\n\n  // If we have any critical issues (wrong tools, missing tools when required, or extra tools when not allowed)\n  if (issues.length > 0 && (options.requireAllTools || !options.allowExtras)) {\n    return {\n      score: 0.0,\n      metadata: {\n        rationale: issues.join(\"; \"),\n      },\n    };\n  }\n\n  if (score === 1.0) {\n    const extraInfo =\n      extraTools.length > 0 ? ` (plus extra: ${extraTools.join(\", \")})` : \"\";\n    return {\n      score: 1.0,\n      metadata: {\n        rationale: `All expected tools were called${extraInfo}`,\n      },\n    };\n  }\n\n  return {\n    score,\n    metadata: {\n      rationale: issues.join(\"; \"),\n      matched: expectedMatched,\n      total: expected.length,\n    },\n  };\n}\n","import type { ScoreFn, BaseScorerOptions } from \"../index\";\nimport {\n  type BaseMatcherConfig,\n  type MatchStrategy,\n  type FuzzyMatchOptions,\n  createMatcher,\n  formatValue,\n  debugLog,\n} from \"./utils\";\n\nexport interface StructuredOutputScorerOptions extends BaseScorerOptions {\n  // Expected structured output defined in test data\n  expected?: Record<string, any>;\n}\n\nexport interface StructuredOutputScorerConfig extends BaseMatcherConfig {\n  /**\n   * How to match field values\n   * - \"strict\": Exact equality required (default)\n   * - \"fuzzy\": More flexible matching (case-insensitive strings, numeric tolerance, regex patterns, subset matching)\n   * - Custom function: Your own comparison logic\n   * @default \"strict\"\n   */\n  match?: MatchStrategy;\n\n  /**\n   * Field name to check for errors in the output\n   * Set to null to disable error checking\n   * @default \"error\"\n   */\n  errorField?: string | null;\n\n  /**\n   * Options for fuzzy matching when match=\"fuzzy\"\n   * @default {} for structured output (no substring matching by default)\n   */\n  fuzzyOptions?: FuzzyMatchOptions;\n}\n\n/**\n * Format mismatch details for error messages\n */\nfunction formatMismatchDetails(\n  mismatches: Array<{ key: string; expected: any; actual: any }>,\n): string {\n  return mismatches\n    .map(\n      (m) =>\n        `${m.key}: expected ${formatValue(m.expected)}, got ${formatValue(m.actual)}`,\n    )\n    .join(\"; \");\n}\n\n/**\n * A configurable scorer for evaluating structured outputs (e.g., JSON) from LLM responses.\n *\n * Similar to ToolCallScorer but for validating structured data outputs like API queries,\n * configuration objects, or any JSON-serializable data structure.\n *\n * @param config - Configuration options for the scorer\n * @param config.match - How to match field values: \"strict\", \"fuzzy\", or custom function\n * @param config.requireAll - Require all expected fields (vs partial credit)\n * @param config.allowExtras - Allow additional fields in output\n * @param config.debug - Enable debug logging\n *\n * @example\n * // Default: strict matching\n * describeEval(\"query generation\", {\n *   data: async () => [{\n *     input: \"Show me errors from today\",\n *     expected: {\n *       dataset: \"errors\",\n *       query: \"\",\n *       sort: \"-timestamp\",\n *       timeRange: { statsPeriod: \"24h\" }\n *     }\n *   }],\n *   task: myTask,\n *   scorers: [StructuredOutputScorer()]\n * });\n *\n * @example\n * // Fuzzy matching with regex patterns\n * describeEval(\"flexible query matching\", {\n *   data: async () => [{\n *     input: \"Find slow API calls\",\n *     expected: {\n *       dataset: \"spans\",\n *       query: /span\\.duration:>1000|span\\.duration:>1s/,\n *       sort: \"-span.duration\"\n *     }\n *   }],\n *   task: myTask,\n *   scorers: [StructuredOutputScorer({ match: \"fuzzy\" })]\n * });\n *\n * @example\n * // Custom field matching\n * describeEval(\"custom validation\", {\n *   data: async () => [{\n *     input: \"Create user config\",\n *     expected: {\n *       name: \"test\",\n *       age: 25,\n *       tags: [\"user\", \"active\"]\n *     }\n *   }],\n *   task: myTask,\n *   scorers: [StructuredOutputScorer({\n *     match: (expected, actual, key) => {\n *       if (key === \"age\") return actual >= 18 && actual <= 100;\n *       return expected === actual; // Use strict equality for other fields\n *     }\n *   })]\n * });\n */\nexport function StructuredOutputScorer(\n  config: StructuredOutputScorerConfig = {},\n): ScoreFn<StructuredOutputScorerOptions> {\n  const {\n    match = \"strict\",\n    requireAll = true,\n    allowExtras = true,\n    debug = false,\n    errorField = \"error\",\n    fuzzyOptions = {}, // Default: no special fuzzy options for structured output\n  } = config;\n\n  // Determine the field matcher. If a custom function is provided, it will be used directly\n  // with its original signature. Note that createMatcher only supports two parameters: match and fuzzyOptions.\n  const fieldMatcher =\n    typeof match === \"function\"\n      ? match // Use custom function directly with its original signature\n      : createMatcher(match, fuzzyOptions);\n\n  return async (opts) => {\n    const expected = opts.expected || {};\n    const output = opts.output;\n\n    // Parse the output as JSON\n    let parsed: Record<string, any>;\n    try {\n      parsed = JSON.parse(output);\n    } catch (error) {\n      return {\n        score: 0.0,\n        metadata: {\n          rationale: `Failed to parse output as JSON: ${error}`,\n          output,\n        },\n      };\n    }\n\n    // No expectations means we just check for valid JSON\n    if (Object.keys(expected).length === 0) {\n      return {\n        score: 1.0,\n        metadata: {\n          rationale: \"Valid JSON output (no expected fields specified)\",\n        },\n      };\n    }\n\n    // Check for error field in output (common pattern for API responses)\n    if (\n      errorField !== null &&\n      parsed[errorField] &&\n      parsed[errorField] !== \"\" &&\n      parsed[errorField] !== null\n    ) {\n      return {\n        score: 0.0,\n        metadata: {\n          rationale: `Output contains error: ${parsed[errorField]}`,\n          output,\n        },\n      };\n    }\n\n    // Compare expected vs actual fields\n    const matches: string[] = [];\n    const mismatches: Array<{ key: string; expected: any; actual: any }> = [];\n    const extras: string[] = [];\n\n    // Check each expected field\n    for (const [key, expectedValue] of Object.entries(expected)) {\n      const actualValue = parsed[key];\n\n      const isMatch = fieldMatcher(expectedValue, actualValue, key);\n      if (isMatch) {\n        matches.push(key);\n      } else {\n        mismatches.push({ key, expected: expectedValue, actual: actualValue });\n      }\n    }\n\n    // Find extra fields\n    const expectedKeys = new Set(Object.keys(expected));\n    for (const key of Object.keys(parsed)) {\n      if (!expectedKeys.has(key)) {\n        extras.push(key);\n      }\n    }\n\n    if (debug) {\n      debugLog(\"StructuredOutputScorer\", {\n        expected,\n        actual: parsed,\n        matches,\n        mismatches,\n        extras,\n      });\n    }\n\n    // Calculate score and rationale\n    const totalExpected = Object.keys(expected).length;\n    const totalMatched = matches.length;\n\n    // Handle various failure conditions\n    if (requireAll && mismatches.length > 0) {\n      const mismatchDetails = formatMismatchDetails(mismatches);\n      return {\n        score: 0.0,\n        metadata: {\n          rationale: `Missing required fields: ${mismatches.map((m) => m.key).join(\", \")} - ${mismatchDetails}`,\n        },\n      };\n    }\n\n    if (!allowExtras && extras.length > 0) {\n      return {\n        score: 0.0,\n        metadata: {\n          rationale: `Unexpected extra fields: ${extras.join(\", \")}`,\n        },\n      };\n    }\n\n    // Calculate partial credit score\n    const score = totalExpected > 0 ? totalMatched / totalExpected : 1.0;\n\n    if (score === 1.0) {\n      const extraInfo =\n        extras.length > 0 ? ` (plus extra fields: ${extras.join(\", \")})` : \"\";\n      return {\n        score: 1.0,\n        metadata: {\n          rationale: `All expected fields match${extraInfo}`,\n        },\n      };\n    }\n\n    // Partial match\n    const mismatchDetails = formatMismatchDetails(mismatches);\n\n    return {\n      score,\n      metadata: {\n        rationale: `Matched ${totalMatched}/${totalExpected} fields - ${mismatchDetails}`,\n        matched: totalMatched,\n        total: totalExpected,\n      },\n    };\n  };\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA,EACE;AAAA,EACA,cAAc;AAAA,EACd,aAAa;AAAA,EACb;AAAA,EACA;AAAA,EACA;AAAA,OACK;AACP,OAAO;;;ACYA,SAAS,SAAS,MAAc,QAAQ,IAAY;AACzD,MAAI,CAAC,QAAQ,KAAK,UAAU,OAAO;AACjC,WAAO;AAAA,EACT;AAEA,QAAM,QAAQ,KAAK,MAAM,KAAK;AAC9B,QAAM,QAAkB,CAAC;AACzB,MAAI,cAAc;AAElB,aAAW,QAAQ,OAAO;AAExB,QAAI,YAAY,SAAS,KAAK,SAAS,IAAI,OAAO;AAChD,YAAM,KAAK,YAAY,KAAK,CAAC;AAC7B,oBAAc;AAAA,IAChB,OAAO;AAEL,sBAAgB,cAAc,MAAM,MAAM;AAAA,IAC5C;AAAA,EACF;AAGA,MAAI,aAAa;AACf,UAAM,KAAK,WAAW;AAAA,EACxB;AAEA,SAAO,MAAM,KAAK,IAAI;AACxB;;;AC+BO,SAAS,aACd,UACA,QACA,SACS;AAET,MAAI,aAAa,OAAQ,QAAO;AAChC,MACE,aAAa,QACb,aAAa,UACb,WAAW,QACX,WAAW,QACX;AACA,WAAO;AAAA,EACT;AAGA,MAAI,OAAO,aAAa,OAAO,OAAQ,QAAO;AAG9C,MAAI,MAAM,QAAQ,QAAQ,GAAG;AAC3B,QAAI,CAAC,MAAM,QAAQ,MAAM,EAAG,QAAO;AACnC,QAAI,SAAS,WAAW,OAAO,OAAQ,QAAO;AAC9C,WAAO,SAAS,MAAM,CAAC,MAAM,MAAM,aAAa,MAAM,OAAO,CAAC,GAAG,OAAO,CAAC;AAAA,EAC3E;AAGA,MAAI,OAAO,aAAa,UAAU;AAChC,UAAM,eAAe,OAAO,KAAK,QAAQ;AACzC,UAAM,aAAa,OAAO,KAAK,MAAM;AAGrC,QAAI,aAAa,WAAW,WAAW,OAAQ,QAAO;AAGtD,WAAO,aAAa;AAAA,MAClB,CAAC,QACC,OAAO,UAAU,aAAa,SAAS,GAAG,GAAG,OAAO,GAAG,GAAG,OAAO;AAAA,IACrE;AAAA,EACF;AAGA,SAAO;AACT;AAKA,SAAS,iBACP,UACA,QACA,SACS;AACT,QAAM,EAAE,kBAAkB,MAAM,YAAY,MAAM,IAAI;AAEtD,QAAM,cAAc,kBAAkB,SAAS,YAAY,IAAI;AAC/D,QAAM,YAAY,kBAAkB,OAAO,YAAY,IAAI;AAE3D,SAAO,YACH,UAAU,SAAS,WAAW,KAAK,YAAY,SAAS,SAAS,IACjE,gBAAgB;AACtB;AAKA,SAAS,iBACP,UACA,QACA,SACS;AACT,QAAM,EAAE,mBAAmB,KAAM,IAAI;AAErC,QAAM,YAAY,KAAK;AAAA,IACrB,KAAK,IAAI,QAAQ,IAAI;AAAA,IACrB;AAAA,EACF;AACA,SAAO,KAAK,IAAI,WAAW,MAAM,KAAK;AACxC;AAKA,SAAS,gBACP,UACA,QACA,SACA,SACS;AACT,QAAM,EAAE,mBAAmB,KAAK,IAAI;AAEpC,MAAI,kBAAkB;AAEpB,UAAM,aAAa,OAAO,IAAI,MAAM,KAAK;AAGzC,WAAO,SAAS,MAAM,CAAC,YAAY;AAEjC,eAAS,IAAI,GAAG,IAAI,OAAO,QAAQ,KAAK;AACtC,YAAI,WAAW,CAAC,EAAG;AAEnB,YAAI,WAAW,SAAS,OAAO,CAAC,GAAG,SAAS,OAAO,GAAG;AACpD,qBAAW,CAAC,IAAI;AAChB,iBAAO;AAAA,QACT;AAAA,MACF;AACA,aAAO;AAAA,IACT,CAAC;AAAA,EACH;AAEA,SACE,SAAS,WAAW,OAAO,UAC3B,SAAS,MAAM,CAAC,MAAM,MAAM,WAAW,MAAM,OAAO,CAAC,GAAG,SAAS,OAAO,CAAC;AAE7E;AAKA,SAAS,iBACP,UACA,QACA,SACA,SACS;AACT,SAAO,OAAO,QAAQ,QAAQ,EAAE;AAAA,IAC9B,CAAC,CAAC,GAAG,KAAK,MACR,KAAK,UAAU,WAAW,OAAQ,OAAe,CAAC,GAAG,SAAS,OAAO;AAAA,EACzE;AACF;AAKA,SAAS,uBACP,UACA,QACA,SACS;AAET,MAAI,OAAO,aAAa,aAAa,OAAO,WAAW,UAAU;AAC/D,WAAO,cAAc,OAAO,YAAY,MAAM,UAAU,WAAW;AAAA,EACrE;AAGA,MAAI,OAAO,aAAa,YAAY,OAAO,WAAW,UAAU;AAC9D,WAAO,OAAO,WAAW,QAAQ,MAAM;AAAA,EACzC;AACA,MAAI,OAAO,aAAa,YAAY,OAAO,WAAW,UAAU;AAC9D,WAAO,aAAa,OAAO,WAAW,MAAM;AAAA,EAC9C;AAEA,SAAO;AACT;AAKO,SAAS,WACd,UACA,QACA,UAA6B,CAAC,GAC9B,SACS;AACT,QAAM,EAAE,cAAc,MAAM,IAAI;AAGhC,MAAI,oBAAoB,QAAQ;AAC9B,WAAO,OAAO,WAAW,YAAY,SAAS,KAAK,MAAM;AAAA,EAC3D;AAGA,MAAI,OAAO,aAAa,YAAY;AAClC,WAAO,SAAS,MAAM;AAAA,EACxB;AAGA,MACE,aAAa,QACb,aAAa,UACb,WAAW,QACX,WAAW,QACX;AACA,WAAO,aAAa;AAAA,EACtB;AAGA,MAAI,OAAO,aAAa,YAAY,OAAO,WAAW,UAAU;AAC9D,WAAO,iBAAiB,UAAU,QAAQ,OAAO;AAAA,EACnD;AAEA,MAAI,OAAO,aAAa,YAAY,OAAO,WAAW,UAAU;AAC9D,WAAO,iBAAiB,UAAU,QAAQ,OAAO;AAAA,EACnD;AAEA,MAAI,MAAM,QAAQ,QAAQ,KAAK,MAAM,QAAQ,MAAM,GAAG;AACpD,WAAO,gBAAgB,UAAU,QAAQ,SAAS,OAAO;AAAA,EAC3D;AAEA,MACE,OAAO,aAAa,YACpB,OAAO,WAAW,YAClB,CAAC,MAAM,QAAQ,QAAQ,KACvB,CAAC,MAAM,QAAQ,MAAM,GACrB;AACA,WAAO,iBAAiB,UAAU,QAAQ,SAAS,OAAO;AAAA,EAC5D;AAGA,MAAI,aAAa;AACf,UAAM,iBAAiB,uBAAuB,UAAU,QAAQ,OAAO;AACvE,QAAI,eAAgB,QAAO;AAAA,EAC7B;AAGA,SAAO,aAAa;AACtB;AAKO,SAAS,cACd,UACA,SACuD;AACvD,MAAI,OAAO,aAAa,YAAY;AAClC,WAAO,CAAC,UAAU,QAAQ,YAAY,SAAS,UAAU,QAAQ,OAAO;AAAA,EAC1E;AAEA,MAAI,aAAa,UAAU;AACzB,WAAO,CAAC,UAAU,QAAQ,YACxB,aAAa,UAAU,QAAQ,OAAO;AAAA,EAC1C;AAEA,SAAO,CAAC,UAAU,QAAQ,YACxB,WAAW,UAAU,QAAQ,WAAW,CAAC,GAAG,OAAO;AACvD;AAKO,SAAS,YAAY,OAAoB;AAC9C,MAAI,UAAU,OAAW,QAAO;AAChC,MAAI,UAAU,KAAM,QAAO;AAC3B,MAAI,iBAAiB,OAAQ,QAAO,MAAM,SAAS;AACnD,MAAI,OAAO,UAAU,SAAU,QAAO,IAAI,KAAK;AAC/C,MAAI,OAAO,UAAU,SAAU,QAAO,KAAK,UAAU,KAAK;AAC1D,SAAO,OAAO,KAAK;AACrB;AA0BA,IAAM,gBAAwB;AAAA,EAC5B,KAAK,CAAC,YAAoB,SAAgB;AAExC,QACE,QAAQ,IAAI,aAAa,iBACzB,QAAQ,IAAI,aAAa,UACzB,QAAQ,IAAI,uBAAuB,QACnC;AACA,cAAQ,IAAI,SAAS,GAAG,IAAI;AAAA,IAC9B;AAAA,EACF;AACF;AAKO,SAAS,SACd,SACA,MAOA,SAAiB,eACX;AACN,SAAO,IAAI,GAAG,OAAO,SAAS;AAC9B,SAAO,IAAI,aAAa,KAAK,QAAQ;AACrC,SAAO,IAAI,WAAW,KAAK,MAAM;AACjC,MAAI,KAAK,QAAS,QAAO,IAAI,YAAY,KAAK,OAAO;AACrD,MAAI,KAAK,WAAY,QAAO,IAAI,eAAe,KAAK,UAAU;AAC9D,MAAI,KAAK,OAAQ,QAAO,IAAI,WAAW,KAAK,MAAM;AACpD;;;AChSO,SAAS,eACd,SAA+B,CAAC,GACA;AAChC,QAAM;AAAA,IACJ,UAAU;AAAA,IACV,aAAa;AAAA,IACb,cAAc;AAAA,IACd,SAAS;AAAA,IACT,cAAc;AAAA,EAChB,IAAI;AAGJ,QAAM,sBAAyC;AAAA,IAC7C,WAAW;AAAA,IACX,iBAAiB;AAAA,IACjB,kBAAkB;AAAA,IAClB,kBAAkB;AAAA,IAClB,aAAa;AAAA,EACf;AACA,QAAM,eAAkC,kCACnC,sBACA;AAIL,QAAM,aAAa,cAAc,QAAQ,YAAY;AAErD,SAAO,CAAO,SAAS;AACrB,UAAM,gBAAgB,KAAK,iBAAiB,CAAC;AAC7C,UAAM,cAAc,KAAK,aAAa,CAAC;AAGvC,QAAI,cAAc,WAAW,GAAG;AAC9B,aAAO;AAAA,QACL,OAAO;AAAA,QACP,UAAU;AAAA,UACR,WAAW;AAAA,QACb;AAAA,MACF;AAAA,IACF;AAGA,QAAI,YAAY,WAAW,GAAG;AAC5B,aAAO;AAAA,QACL,OAAO;AAAA,QACP,UAAU;AAAA,UACR,WAAW,YAAY,cAAc,MAAM;AAAA,QAC7C;AAAA,MACF;AAAA,IACF;AAEA,QAAI,SAAS;AACX,aAAO,qBAAqB,eAAe,aAAa;AAAA,QACtD;AAAA,QACA;AAAA,QACA,iBAAiB;AAAA,MACnB,CAAC;AAAA,IACH;AAEA,WAAO,uBAAuB,eAAe,aAAa;AAAA,MACxD;AAAA,MACA,iBAAiB;AAAA,MACjB;AAAA,IACF,CAAC;AAAA,EACH;AACF;AAKA,SAAS,qBACP,UACA,QACA,SAKA;AACA,MAAI,gBAAgB;AACpB,MAAI,cAAc;AAGlB,SAAO,gBAAgB,SAAS,UAAU,cAAc,OAAO,QAAQ;AACrE,UAAM,MAAM,SAAS,aAAa;AAClC,UAAM,MAAM,OAAO,WAAW;AAE9B,QAAI,IAAI,SAAS,IAAI,MAAM;AAEzB,UAAI,IAAI,cAAc,QAAW;AAC/B,cAAM,YAAY,QAAQ;AAAA,UACxB,IAAI;AAAA,UACJ,IAAI,aAAa,CAAC;AAAA,QACpB;AACA,YAAI,CAAC,WAAW;AAEd,gBAAM,eAAe,gBAAgB,SAAS;AAC9C,iBAAO;AAAA,YACL,OAAO;AAAA,YACP,UAAU;AAAA,cACR,WAAW,SAAS,IAAI,IAAI,iDAAiD,gBAAgB,CAAC,KAAK,aAAa,IAAI,SAAS,MAAM;AAAA,cACnI,UAAU,IAAI;AAAA,cACd,QAAQ,IAAI;AAAA,cACZ,SAAS;AAAA,cACT,OAAO,SAAS;AAAA,YAClB;AAAA,UACF;AAAA,QACF;AAAA,MACF;AACA;AACA;AAAA,IACF,WAAW,QAAQ,aAAa;AAE9B;AAAA,IACF,OAAO;AAEL,aAAO;AAAA,QACL,OAAO;AAAA,QACP,UAAU;AAAA,UACR,WAAW,aAAa,IAAI,IAAI,iBAAiB,gBAAgB,CAAC,eAAe,IAAI,IAAI;AAAA,QAC3F;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAGA,MAAI,gBAAgB,SAAS,QAAQ;AACnC,UAAM,UAAU,SAAS,MAAM,aAAa,EAAE,IAAI,CAAC,MAAM,EAAE,IAAI;AAE/D,QAAI,QAAQ,iBAAiB;AAC3B,aAAO;AAAA,QACL,OAAO;AAAA,QACP,UAAU;AAAA,UACR,WAAW,uCAAuC,QAAQ,KAAK,IAAI,CAAC;AAAA,QACtE;AAAA,MACF;AAAA,IACF;AAGA,UAAM,eAAe;AACrB,UAAM,aAAa,SAAS;AAC5B,UAAM,QAAQ,aAAa,IAAI,eAAe,aAAa;AAE3D,WAAO;AAAA,MACL;AAAA,MACA,UAAU;AAAA,QACR,WAAW,kBAAkB,YAAY,IAAI,UAAU,oCAAoC,QAAQ,KAAK,IAAI,CAAC;AAAA,QAC7G,SAAS;AAAA,QACT,OAAO;AAAA,MACT;AAAA,IACF;AAAA,EACF;AAGA,MAAI,CAAC,QAAQ,eAAe,cAAc,OAAO,QAAQ;AACvD,UAAM,QAAQ,OAAO,MAAM,WAAW,EAAE,IAAI,CAAC,MAAM,EAAE,IAAI;AACzD,WAAO;AAAA,MACL,OAAO;AAAA,MACP,UAAU;AAAA,QACR,WAAW,2BAA2B,MAAM,KAAK,IAAI,CAAC;AAAA,MACxD;AAAA,IACF;AAAA,EACF;AAEA,SAAO;AAAA,IACL,OAAO;AAAA,IACP,UAAU;AAAA,MACR,WAAW;AAAA,IACb;AAAA,EACF;AACF;AAUA,SAAS,uBACP,UACA,QACA,SAKA;AAEA,QAAM,oBAAoB,CAAC,GAAG,QAAQ;AACtC,QAAM,kBAAkB,CAAC,GAAG,MAAM;AAClC,QAAM,SAAmB,CAAC;AAG1B,WAAS,IAAI,kBAAkB,SAAS,GAAG,KAAK,GAAG,KAAK;AACtD,UAAM,eAAe,kBAAkB,CAAC;AAGxC,UAAM,aAAa,gBAAgB,UAAU,CAAC,eAAe;AAE3D,UAAI,aAAa,SAAS,WAAW,MAAM;AACzC,eAAO;AAAA,MACT;AAGA,UAAI,aAAa,cAAc,QAAW;AACxC,eAAO,QAAQ;AAAA,UACb,aAAa;AAAA,UACb,WAAW,aAAa,CAAC;AAAA,QAC3B;AAAA,MACF;AAEA,aAAO;AAAA,IACT,CAAC;AAED,QAAI,eAAe,IAAI;AAErB,wBAAkB,OAAO,GAAG,CAAC;AAC7B,sBAAgB,OAAO,YAAY,CAAC;AAAA,IACtC;AAAA,EACF;AAGA,aAAW,eAAe,mBAAmB;AAC3C,QAAI,YAAY,cAAc,QAAW;AAEvC,YAAM,iBAAiB,OAAO,OAAO,CAAC,MAAM,EAAE,SAAS,YAAY,IAAI;AACvE,UAAI,eAAe,SAAS,GAAG;AAC7B,eAAO;AAAA,UACL,SAAS,YAAY,IAAI;AAAA,QAC3B;AAAA,MACF,OAAO;AACL,eAAO,KAAK,0BAA0B,YAAY,IAAI,EAAE;AAAA,MAC1D;AAAA,IACF,OAAO;AACL,aAAO,KAAK,0BAA0B,YAAY,IAAI,EAAE;AAAA,IAC1D;AAAA,EACF;AAGA,QAAM,aAAa,gBAAgB,IAAI,CAAC,SAAS,KAAK,IAAI;AAE1D,MAAI,CAAC,QAAQ,eAAe,WAAW,SAAS,GAAG;AACjD,WAAO,KAAK,2BAA2B,WAAW,KAAK,IAAI,CAAC,EAAE;AAAA,EAChE;AAGA,QAAM,kBAAkB,SAAS,SAAS,kBAAkB;AAC5D,QAAM,QAAQ,SAAS,SAAS,IAAI,kBAAkB,SAAS,SAAS;AAGxE,MAAI,OAAO,SAAS,MAAM,QAAQ,mBAAmB,CAAC,QAAQ,cAAc;AAC1E,WAAO;AAAA,MACL,OAAO;AAAA,MACP,UAAU;AAAA,QACR,WAAW,OAAO,KAAK,IAAI;AAAA,MAC7B;AAAA,IACF;AAAA,EACF;AAEA,MAAI,UAAU,GAAK;AACjB,UAAM,YACJ,WAAW,SAAS,IAAI,iBAAiB,WAAW,KAAK,IAAI,CAAC,MAAM;AACtE,WAAO;AAAA,MACL,OAAO;AAAA,MACP,UAAU;AAAA,QACR,WAAW,iCAAiC,SAAS;AAAA,MACvD;AAAA,IACF;AAAA,EACF;AAEA,SAAO;AAAA,IACL;AAAA,IACA,UAAU;AAAA,MACR,WAAW,OAAO,KAAK,IAAI;AAAA,MAC3B,SAAS;AAAA,MACT,OAAO,SAAS;AAAA,IAClB;AAAA,EACF;AACF;;;AC9UA,SAAS,sBACP,YACQ;AACR,SAAO,WACJ;AAAA,IACC,CAAC,MACC,GAAG,EAAE,GAAG,cAAc,YAAY,EAAE,QAAQ,CAAC,SAAS,YAAY,EAAE,MAAM,CAAC;AAAA,EAC/E,EACC,KAAK,IAAI;AACd;AAiEO,SAAS,uBACd,SAAuC,CAAC,GACA;AACxC,QAAM;AAAA,IACJ,QAAQ;AAAA,IACR,aAAa;AAAA,IACb,cAAc;AAAA,IACd,QAAQ;AAAA,IACR,aAAa;AAAA,IACb,eAAe,CAAC;AAAA;AAAA,EAClB,IAAI;AAIJ,QAAM,eACJ,OAAO,UAAU,aACb,QACA,cAAc,OAAO,YAAY;AAEvC,SAAO,CAAO,SAAS;AACrB,UAAM,WAAW,KAAK,YAAY,CAAC;AACnC,UAAM,SAAS,KAAK;AAGpB,QAAI;AACJ,QAAI;AACF,eAAS,KAAK,MAAM,MAAM;AAAA,IAC5B,SAAS,OAAO;AACd,aAAO;AAAA,QACL,OAAO;AAAA,QACP,UAAU;AAAA,UACR,WAAW,mCAAmC,KAAK;AAAA,UACnD;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAGA,QAAI,OAAO,KAAK,QAAQ,EAAE,WAAW,GAAG;AACtC,aAAO;AAAA,QACL,OAAO;AAAA,QACP,UAAU;AAAA,UACR,WAAW;AAAA,QACb;AAAA,MACF;AAAA,IACF;AAGA,QACE,eAAe,QACf,OAAO,UAAU,KACjB,OAAO,UAAU,MAAM,MACvB,OAAO,UAAU,MAAM,MACvB;AACA,aAAO;AAAA,QACL,OAAO;AAAA,QACP,UAAU;AAAA,UACR,WAAW,0BAA0B,OAAO,UAAU,CAAC;AAAA,UACvD;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAGA,UAAM,UAAoB,CAAC;AAC3B,UAAM,aAAiE,CAAC;AACxE,UAAM,SAAmB,CAAC;AAG1B,eAAW,CAAC,KAAK,aAAa,KAAK,OAAO,QAAQ,QAAQ,GAAG;AAC3D,YAAM,cAAc,OAAO,GAAG;AAE9B,YAAM,UAAU,aAAa,eAAe,aAAa,GAAG;AAC5D,UAAI,SAAS;AACX,gBAAQ,KAAK,GAAG;AAAA,MAClB,OAAO;AACL,mBAAW,KAAK,EAAE,KAAK,UAAU,eAAe,QAAQ,YAAY,CAAC;AAAA,MACvE;AAAA,IACF;AAGA,UAAM,eAAe,IAAI,IAAI,OAAO,KAAK,QAAQ,CAAC;AAClD,eAAW,OAAO,OAAO,KAAK,MAAM,GAAG;AACrC,UAAI,CAAC,aAAa,IAAI,GAAG,GAAG;AAC1B,eAAO,KAAK,GAAG;AAAA,MACjB;AAAA,IACF;AAEA,QAAI,OAAO;AACT,eAAS,0BAA0B;AAAA,QACjC;AAAA,QACA,QAAQ;AAAA,QACR;AAAA,QACA;AAAA,QACA;AAAA,MACF,CAAC;AAAA,IACH;AAGA,UAAM,gBAAgB,OAAO,KAAK,QAAQ,EAAE;AAC5C,UAAM,eAAe,QAAQ;AAG7B,QAAI,cAAc,WAAW,SAAS,GAAG;AACvC,YAAMA,mBAAkB,sBAAsB,UAAU;AACxD,aAAO;AAAA,QACL,OAAO;AAAA,QACP,UAAU;AAAA,UACR,WAAW,4BAA4B,WAAW,IAAI,CAAC,MAAM,EAAE,GAAG,EAAE,KAAK,IAAI,CAAC,MAAMA,gBAAe;AAAA,QACrG;AAAA,MACF;AAAA,IACF;AAEA,QAAI,CAAC,eAAe,OAAO,SAAS,GAAG;AACrC,aAAO;AAAA,QACL,OAAO;AAAA,QACP,UAAU;AAAA,UACR,WAAW,4BAA4B,OAAO,KAAK,IAAI,CAAC;AAAA,QAC1D;AAAA,MACF;AAAA,IACF;AAGA,UAAM,QAAQ,gBAAgB,IAAI,eAAe,gBAAgB;AAEjE,QAAI,UAAU,GAAK;AACjB,YAAM,YACJ,OAAO,SAAS,IAAI,wBAAwB,OAAO,KAAK,IAAI,CAAC,MAAM;AACrE,aAAO;AAAA,QACL,OAAO;AAAA,QACP,UAAU;AAAA,UACR,WAAW,4BAA4B,SAAS;AAAA,QAClD;AAAA,MACF;AAAA,IACF;AAGA,UAAM,kBAAkB,sBAAsB,UAAU;AAExD,WAAO;AAAA,MACL;AAAA,MACA,UAAU;AAAA,QACR,WAAW,WAAW,YAAY,IAAI,aAAa,aAAa,eAAe;AAAA,QAC/E,SAAS;AAAA,QACT,OAAO;AAAA,MACT;AAAA,IACF;AAAA,EACF;AACF;;;AJ3KA,OAAO,OAAO;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EA+BZ,QAAQ,SAAe,OACrB,OACA,UACA,QACA,SACA,YAAY,GACZ;AAAA;AAlIJ;AAmII,YAAM,EAAE,MAAM,IAAI;AAElB,YAAM,aAAa,MAAM,OAAO,KAAK;AACrC,YAAM,SACJ,OAAO,eAAe,WAAW,aAAa,WAAW;AAC3D,YAAM,YACJ,OAAO,eAAe,WAAW,WAAW,YAAY;AAE1D,UAAI,SAAS,QAAQ,EAAE,OAAO,UAAU,QAAQ,UAAU,CAAC;AAC3D,UAAI,kBAAkB,SAAS;AAC7B,iBAAS,MAAM;AAAA,MACjB;AAEA,aAAO;AAAA,QACL,QAAO,YAAO,UAAP,YAAgB,MAAM;AAAA,QAC7B,SAAS,MAAM,aAAa,CAAC,iCAAK,SAAL,EAAa,MAAM,QAAQ,KAAK,EAAC,CAAC;AAAA,MACjE;AAAA,IACF;AAAA;AACF,CAAC;AAuDM,SAAS,aACd,MACA;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA,YAAY;AAAA;AAAA;AAAA,EAGZ,UAAU;AAAA,EACV,YAAY;AAAA,EACZ,WAAW;AACb,GAYA;AACA,SAAO,SAAS,MAAM,MAAY;AAChC,QAAI,gBAAgB;AAClB,uBAAiB,cAAc;AAAA,IACjC;AACA,QAAI,eAAe;AACjB,sBAAgB,aAAa;AAAA,IAC/B;AAEA,UAAM,SAAS,SAAS,KAAK,OAAO,OAAO,CAAC,IAAI;AAEhD,eAAW,MAAwC,MAAM,KAAK,GAAG;AAA5D,qBAAQ,SAAO,MAAM,SAhP9B,IAgPS,IAAkC,mBAAlC,IAAkC,CAA1B,SAAO;AAClB;AAAA,QACE,8BAAY;AAAA,QACZ;AAAA,UACE;AAAA,QACF;AAAA,QACA,CAAO,OAAuB,eAAvB,KAAuB,WAAvB,EAAE,MAAM,SAAS,GAAM;AAC5B,gBAAM,aAAa,MAAM,KAAK,KAAK;AACnC,gBAAM,SACJ,OAAO,eAAe,WAAW,aAAa,WAAW;AAC3D,gBAAM,YACJ,OAAO,eAAe,WAAW,WAAW,YAAY;AAE1D,gBAAM,SAAS,MAAM,QAAQ;AAAA,YAC3B,QAAQ,IAAI,CAAC,WAAW;AACtB,oBAAM,SAAS,OAAO,+BAAE,SAAU,SAAZ,EAAoB,QAAQ,UAAU,EAAC;AAC7D,kBAAI,kBAAkB,SAAS;AAC7B,uBAAO;AAAA,cACT;AACA,qBAAO,IAAI,QAAe,CAAC,YAAY,QAAQ,MAAM,CAAC;AAAA,YACxD,CAAC;AAAA,UACH;AACA,gBAAM,iBAAiB,OAAO,IAAI,CAAC,GAAG,MAAO,iCACxC,IADwC;AAAA,YAE3C,MAAM,QAAQ,CAAC,EAAE;AAAA,UACnB,EAAE;AAEF,gBAAM,WACJ,OAAO,OAAO,CAAC,KAAK,MAAG;AA5QnC,gBAAAC;AA4QsC,2BAAOA,MAAA,EAAE,UAAF,OAAAA,MAAW;AAAA,aAAI,CAAC,IAAI,OAAO;AAE9D,mBAAS,KAAK,OAAO;AAAA,YACnB,QAAQ;AAAA,YACR;AAAA,aACI,aAAa,EAAE,UAAU;AAG/B,cAAI,WAAW;AACb;AAAA,cACE,YAAY;AAAA,cACZ,UAAU,QAAQ,qBAAqB,SAAS;AAAA;AAAA;AAAA,EAAmB,SAAS,MAAM,CAAC;AAAA;AAAA,EAAO;AAAA,gBACxF;AAAA,cACF,CAAC;AAAA,YACH;AAAA,UACF;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAAA,EACF,EAAC;AACH;AAEO,SAAS,aAAa,QAAsC;AACjE,SAAO,OACJ,KAAK,CAAC,GAAG,MAAG;AApSjB;AAoSqB,oBAAE,UAAF,YAAW,OAAM,OAAE,UAAF,YAAW;AAAA,GAAE,EAC9C,IAAI,CAAC,MAAM;AArShB;AAsSM,UAAM,YAAY,KAAK,EAAE,QAAQ,SAAS,OAAM,OAAE,UAAF,YAAW,GAAG,QAAQ,CAAC,CAAC;AACxE,UACI,OAAE,UAAF,YAAW,KAAK,OAAO,OAAE,aAAF,mBAAY,gBACrC,OAAE,aAAF,mBAAY,SACZ;AAEA,UAAI,kBAAkB;AACtB,YAAI,OAAE,aAAF,mBAAY,YAAW,QAAW;AACpC,cAAM,SAAS,EAAE,SAAS;AAC1B,YAAI,OAAO,WAAW,UAAU;AAC9B,4BAAkB;AAAA;AAAA;AAAA;AAAA,EAAsB,SAAS,MAAM,CAAC;AAAA,QAC1D,OAAO;AAEL,4BAAkB;AAAA;AAAA;AAAA;AAAA,EAAsB,SAAS,KAAK,UAAU,QAAQ,MAAM,CAAC,CAAC,CAAC;AAAA,QACnF;AAAA,MACF;AAEA,aAAO,GAAG,SAAS,KACjB,OAAE,aAAF,mBAAY,aACR;AAAA;AAAA;AAAA;AAAA,EAAuB,SAAS,EAAE,SAAS,SAAS,CAAC,KACrD,EACN,GAAG,eAAe;AAAA,IACpB;AACA,WAAO;AAAA,EACT,CAAC,EACA,KAAK,MAAM;AAChB;","names":["mismatchDetails","_a"]}

package/dist/reporter.js

@@ -39,14 +39,15 @@ var DefaultEvalReporter = class extends import_reporters.DefaultReporter {
   printTestCase(moduleState, test) {
     const meta = test.meta();
     const testResult = test.result();
-    if (!meta.eval || testResult.state === "failed") {
+    if (!meta.eval) {
       super.printTestCase(moduleState, test);
       return;
     }
     const padding = this.getTestIndentation(test.task);
+    const icon = testResult.state === "failed" ? import_tinyrainbow.default.red("\u2717 ") : "  ";
     const colorFn = meta.eval.avgScore < 0.5 ? import_tinyrainbow.default.red : meta.eval.avgScore < 0.75 ? import_tinyrainbow.default.yellow : import_tinyrainbow.default.green;
     this.log(
-      `${padding}  ${this.getTestName(test.task, import_tinyrainbow.default.dim(" > "))} [${colorFn(meta.eval.avgScore.toFixed(2))}]`
+      `${padding}${icon}${this.getTestName(test.task, import_tinyrainbow.default.dim(" > "))} [${colorFn(meta.eval.avgScore.toFixed(2))}]`
     );
   }
 };

package/dist/reporter.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../src/reporter.ts"],"sourcesContent":["// import type { RunnerTask, RunnerTestFile } from \"vitest\";\nimport { DefaultReporter } from \"vitest/reporters\";\nimport c from \"tinyrainbow\";\n\nexport default class DefaultEvalReporter extends DefaultReporter {\n  protected override printTestCase(moduleState: any, test: any): void {\n    const meta = test.meta();\n    const testResult = test.result();\n\n    if (!meta.eval || testResult.state === \"failed\") {\n      super.printTestCase(moduleState, test);\n      return;\n    }\n\n    const padding = this.getTestIndentation(test.task);\n    const colorFn =\n      meta.eval.avgScore < 0.5\n        ? c.red\n        : meta.eval.avgScore < 0.75\n          ? c.yellow\n          : c.green;\n    this.log(\n      `${padding}  ${this.getTestName(test.task, c.dim(\" > \"))} [${colorFn(meta.eval.avgScore.toFixed(2))}]`,\n    );\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AACA,uBAAgC;AAChC,yBAAc;AAEd,IAAqB,sBAArB,cAAiD,iCAAgB;AAAA,EAC5C,cAAc,aAAkB,MAAiB;AAClE,UAAM,OAAO,KAAK,KAAK;AACvB,UAAM,aAAa,KAAK,OAAO;AAE/B,QAAI,CAAC,KAAK,QAAQ,WAAW,UAAU,UAAU;AAC/C,YAAM,cAAc,aAAa,IAAI;AACrC;AAAA,IACF;AAEA,UAAM,UAAU,KAAK,mBAAmB,KAAK,IAAI;AACjD,UAAM,UACJ,KAAK,KAAK,WAAW,MACjB,mBAAAA,QAAE,MACF,KAAK,KAAK,WAAW,OACnB,mBAAAA,QAAE,SACF,mBAAAA,QAAE;AACV,SAAK;AAAA,MACH,GAAG,OAAO,KAAK,KAAK,YAAY,KAAK,MAAM,mBAAAA,QAAE,IAAI,KAAK,CAAC,CAAC,KAAK,QAAQ,KAAK,KAAK,SAAS,QAAQ,CAAC,CAAC,CAAC;AAAA,IACrG;AAAA,EACF;AACF;","names":["c"]}
+{"version":3,"sources":["../src/reporter.ts"],"sourcesContent":["// import type { RunnerTask, RunnerTestFile } from \"vitest\";\n// TODO: Switch to \"vitest/node\" when we drop Vitest 3 support.\nimport { DefaultReporter } from \"vitest/reporters\";\nimport c from \"tinyrainbow\";\n\nexport default class DefaultEvalReporter extends DefaultReporter {\n  protected override printTestCase(moduleState: any, test: any): void {\n    const meta = test.meta();\n    const testResult = test.result();\n\n    if (!meta.eval) {\n      super.printTestCase(moduleState, test);\n      return;\n    }\n\n    const padding = this.getTestIndentation(test.task);\n    const icon = testResult.state === \"failed\" ? c.red(\"✗ \") : \"  \";\n    const colorFn =\n      meta.eval.avgScore < 0.5\n        ? c.red\n        : meta.eval.avgScore < 0.75\n          ? c.yellow\n          : c.green;\n    this.log(\n      `${padding}${icon}${this.getTestName(test.task, c.dim(\" > \"))} [${colorFn(meta.eval.avgScore.toFixed(2))}]`,\n    );\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAEA,uBAAgC;AAChC,yBAAc;AAEd,IAAqB,sBAArB,cAAiD,iCAAgB;AAAA,EAC5C,cAAc,aAAkB,MAAiB;AAClE,UAAM,OAAO,KAAK,KAAK;AACvB,UAAM,aAAa,KAAK,OAAO;AAE/B,QAAI,CAAC,KAAK,MAAM;AACd,YAAM,cAAc,aAAa,IAAI;AACrC;AAAA,IACF;AAEA,UAAM,UAAU,KAAK,mBAAmB,KAAK,IAAI;AACjD,UAAM,OAAO,WAAW,UAAU,WAAW,mBAAAA,QAAE,IAAI,SAAI,IAAI;AAC3D,UAAM,UACJ,KAAK,KAAK,WAAW,MACjB,mBAAAA,QAAE,MACF,KAAK,KAAK,WAAW,OACnB,mBAAAA,QAAE,SACF,mBAAAA,QAAE;AACV,SAAK;AAAA,MACH,GAAG,OAAO,GAAG,IAAI,GAAG,KAAK,YAAY,KAAK,MAAM,mBAAAA,QAAE,IAAI,KAAK,CAAC,CAAC,KAAK,QAAQ,KAAK,KAAK,SAAS,QAAQ,CAAC,CAAC,CAAC;AAAA,IAC1G;AAAA,EACF;AACF;","names":["c"]}

package/dist/reporter.mjs

@@ -5,14 +5,15 @@ var DefaultEvalReporter = class extends DefaultReporter {
   printTestCase(moduleState, test) {
     const meta = test.meta();
     const testResult = test.result();
-    if (!meta.eval || testResult.state === "failed") {
+    if (!meta.eval) {
       super.printTestCase(moduleState, test);
       return;
     }
     const padding = this.getTestIndentation(test.task);
+    const icon = testResult.state === "failed" ? c.red("\u2717 ") : "  ";
     const colorFn = meta.eval.avgScore < 0.5 ? c.red : meta.eval.avgScore < 0.75 ? c.yellow : c.green;
     this.log(
-      `${padding}  ${this.getTestName(test.task, c.dim(" > "))} [${colorFn(meta.eval.avgScore.toFixed(2))}]`
+      `${padding}${icon}${this.getTestName(test.task, c.dim(" > "))} [${colorFn(meta.eval.avgScore.toFixed(2))}]`
     );
   }
 };

package/dist/reporter.mjs.map

@@ -1 +1 @@
-{"version":3,"sources":["../src/reporter.ts"],"sourcesContent":["// import type { RunnerTask, RunnerTestFile } from \"vitest\";\nimport { DefaultReporter } from \"vitest/reporters\";\nimport c from \"tinyrainbow\";\n\nexport default class DefaultEvalReporter extends DefaultReporter {\n  protected override printTestCase(moduleState: any, test: any): void {\n    const meta = test.meta();\n    const testResult = test.result();\n\n    if (!meta.eval || testResult.state === \"failed\") {\n      super.printTestCase(moduleState, test);\n      return;\n    }\n\n    const padding = this.getTestIndentation(test.task);\n    const colorFn =\n      meta.eval.avgScore < 0.5\n        ? c.red\n        : meta.eval.avgScore < 0.75\n          ? c.yellow\n          : c.green;\n    this.log(\n      `${padding}  ${this.getTestName(test.task, c.dim(\" > \"))} [${colorFn(meta.eval.avgScore.toFixed(2))}]`,\n    );\n  }\n}\n"],"mappings":";AACA,SAAS,uBAAuB;AAChC,OAAO,OAAO;AAEd,IAAqB,sBAArB,cAAiD,gBAAgB;AAAA,EAC5C,cAAc,aAAkB,MAAiB;AAClE,UAAM,OAAO,KAAK,KAAK;AACvB,UAAM,aAAa,KAAK,OAAO;AAE/B,QAAI,CAAC,KAAK,QAAQ,WAAW,UAAU,UAAU;AAC/C,YAAM,cAAc,aAAa,IAAI;AACrC;AAAA,IACF;AAEA,UAAM,UAAU,KAAK,mBAAmB,KAAK,IAAI;AACjD,UAAM,UACJ,KAAK,KAAK,WAAW,MACjB,EAAE,MACF,KAAK,KAAK,WAAW,OACnB,EAAE,SACF,EAAE;AACV,SAAK;AAAA,MACH,GAAG,OAAO,KAAK,KAAK,YAAY,KAAK,MAAM,EAAE,IAAI,KAAK,CAAC,CAAC,KAAK,QAAQ,KAAK,KAAK,SAAS,QAAQ,CAAC,CAAC,CAAC;AAAA,IACrG;AAAA,EACF;AACF;","names":[]}
+{"version":3,"sources":["../src/reporter.ts"],"sourcesContent":["// import type { RunnerTask, RunnerTestFile } from \"vitest\";\n// TODO: Switch to \"vitest/node\" when we drop Vitest 3 support.\nimport { DefaultReporter } from \"vitest/reporters\";\nimport c from \"tinyrainbow\";\n\nexport default class DefaultEvalReporter extends DefaultReporter {\n  protected override printTestCase(moduleState: any, test: any): void {\n    const meta = test.meta();\n    const testResult = test.result();\n\n    if (!meta.eval) {\n      super.printTestCase(moduleState, test);\n      return;\n    }\n\n    const padding = this.getTestIndentation(test.task);\n    const icon = testResult.state === \"failed\" ? c.red(\"✗ \") : \"  \";\n    const colorFn =\n      meta.eval.avgScore < 0.5\n        ? c.red\n        : meta.eval.avgScore < 0.75\n          ? c.yellow\n          : c.green;\n    this.log(\n      `${padding}${icon}${this.getTestName(test.task, c.dim(\" > \"))} [${colorFn(meta.eval.avgScore.toFixed(2))}]`,\n    );\n  }\n}\n"],"mappings":";AAEA,SAAS,uBAAuB;AAChC,OAAO,OAAO;AAEd,IAAqB,sBAArB,cAAiD,gBAAgB;AAAA,EAC5C,cAAc,aAAkB,MAAiB;AAClE,UAAM,OAAO,KAAK,KAAK;AACvB,UAAM,aAAa,KAAK,OAAO;AAE/B,QAAI,CAAC,KAAK,MAAM;AACd,YAAM,cAAc,aAAa,IAAI;AACrC;AAAA,IACF;AAEA,UAAM,UAAU,KAAK,mBAAmB,KAAK,IAAI;AACjD,UAAM,OAAO,WAAW,UAAU,WAAW,EAAE,IAAI,SAAI,IAAI;AAC3D,UAAM,UACJ,KAAK,KAAK,WAAW,MACjB,EAAE,MACF,KAAK,KAAK,WAAW,OACnB,EAAE,SACF,EAAE;AACV,SAAK;AAAA,MACH,GAAG,OAAO,GAAG,IAAI,GAAG,KAAK,YAAY,KAAK,MAAM,EAAE,IAAI,KAAK,CAAC,CAAC,KAAK,QAAQ,KAAK,KAAK,SAAS,QAAQ,CAAC,CAAC,CAAC;AAAA,IAC1G;AAAA,EACF;AACF;","names":[]}

package/dist/scorers/index.d.mts

@@ -1,3 +1,4 @@
-export { b as StructuredOutputScorer, a as StructuredOutputScorerConfig, S as StructuredOutputScorerOptions, ToolCallScorer, ToolCallScorerConfig, ToolCallScorerOptions } from './toolCallScorer.mjs';
+export { S as StructuredOutputScorer, a as StructuredOutputScorerConfig, b as StructuredOutputScorerOptions, ToolCallScorer, ToolCallScorerConfig, ToolCallScorerOptions } from './toolCallScorer.mjs';
 export { BaseMatcherConfig, FuzzyMatchOptions, MatchStrategy, calculatePartialScore, createMatcher, debugLog, formatValue, fuzzyMatch, strictEquals } from './utils.mjs';
 import 'vitest';
+import '../wrapText.mjs';

package/dist/scorers/index.d.ts

@@ -1,3 +1,4 @@
-export { b as StructuredOutputScorer, a as StructuredOutputScorerConfig, S as StructuredOutputScorerOptions, ToolCallScorer, ToolCallScorerConfig, ToolCallScorerOptions } from './toolCallScorer.js';
+export { S as StructuredOutputScorer, a as StructuredOutputScorerConfig, b as StructuredOutputScorerOptions, ToolCallScorer, ToolCallScorerConfig, ToolCallScorerOptions } from './toolCallScorer.js';
 export { BaseMatcherConfig, FuzzyMatchOptions, MatchStrategy, calculatePartialScore, createMatcher, debugLog, formatValue, fuzzyMatch, strictEquals } from './utils.js';
 import 'vitest';
+import '../wrapText.js';

package/dist/scorers/structuredOutputScorer.d.mts

@@ -1,3 +1,4 @@
-export { b as StructuredOutputScorer, a as StructuredOutputScorerConfig, S as StructuredOutputScorerOptions } from './toolCallScorer.mjs';
+export { S as StructuredOutputScorer, a as StructuredOutputScorerConfig, b as StructuredOutputScorerOptions } from './toolCallScorer.mjs';
 import './utils.mjs';
 import 'vitest';
+import '../wrapText.mjs';

package/dist/scorers/structuredOutputScorer.d.ts

@@ -1,3 +1,4 @@
-export { b as StructuredOutputScorer, a as StructuredOutputScorerConfig, S as StructuredOutputScorerOptions } from './toolCallScorer.js';
+export { S as StructuredOutputScorer, a as StructuredOutputScorerConfig, b as StructuredOutputScorerOptions } from './toolCallScorer.js';
 import './utils.js';
 import 'vitest';
+import '../wrapText.js';

package/dist/scorers/toolCallScorer.d.mts

@@ -1,5 +1,6 @@
 import { BaseMatcherConfig, MatchStrategy, FuzzyMatchOptions } from './utils.mjs';
 import * as vitest from 'vitest';
+import '../wrapText.mjs';
 
 interface ToolCallScorerOptions extends BaseScorerOptions {
     expectedTools?: Array<{
@@ -294,39 +295,21 @@ declare module "vitest" {
  * });
  * ```
  */
-declare function describeEval(name: string, { data, task, skipIf, scorers, threshold, timeout, }: {
+declare function describeEval(name: string, { data, task, skipIf, scorers, threshold, timeout, beforeEach: beforeEachHook, afterEach: afterEachHook, }: {
     data: () => Promise<Array<{
         input: string;
+        name?: string;
     } & Record<string, any>>>;
     task: TaskFn;
     skipIf?: () => boolean;
     scorers: ScoreFn<any>[];
     threshold?: number | null;
     timeout?: number;
+    beforeEach?: () => void | Promise<void>;
+    afterEach?: () => void | Promise<void>;
 }): vitest.SuiteCollector<object>;
 declare function formatScores(scores: (Score & {
     name: string;
 })[]): string;
-/**
- * Wraps text to fit within a specified width, breaking at word boundaries.
- *
- * @param text - The text to wrap
- * @param width - The maximum width in characters (default: 80)
- * @returns The wrapped text with line breaks
- *
- * @example
- * ```javascript
- * const wrapped = wrapText("This is a very long text that needs to be wrapped to fit within an 80 character width.", 20);
- * console.log(wrapped);
- * // Output:
- * // This is a very
- * // long text that
- * // needs to be
- * // wrapped to fit
- * // within an 80
- * // character width.
- * ```
- */
-declare function wrapText(text: string, width?: number): string;
 
-export { type BaseScorerOptions as B, type EvalMatchers as E, type StructuredOutputScorerOptions as S, type ToolCall as T, ToolCallScorer, type ToolCallScorerConfig, type ToolCallScorerOptions, type StructuredOutputScorerConfig as a, StructuredOutputScorer as b, type TaskResult as c, type TaskFn as d, type Score as e, type ScoreFn as f, type ToEval as g, describeEval as h, formatScores as i, wrapText as w };
+export { type BaseScorerOptions as B, type EvalMatchers as E, StructuredOutputScorer as S, type TaskFn as T, ToolCallScorer, type ToolCallScorerConfig, type ToolCallScorerOptions, type StructuredOutputScorerConfig as a, type StructuredOutputScorerOptions as b, type Score as c, type ScoreFn as d, type TaskResult as e, type ToEval as f, type ToolCall as g, describeEval as h, formatScores as i };

package/dist/scorers/toolCallScorer.d.ts

@@ -1,5 +1,6 @@
 import { BaseMatcherConfig, MatchStrategy, FuzzyMatchOptions } from './utils.js';
 import * as vitest from 'vitest';
+import '../wrapText.js';
 
 interface ToolCallScorerOptions extends BaseScorerOptions {
     expectedTools?: Array<{
@@ -294,39 +295,21 @@ declare module "vitest" {
  * });
  * ```
  */
-declare function describeEval(name: string, { data, task, skipIf, scorers, threshold, timeout, }: {
+declare function describeEval(name: string, { data, task, skipIf, scorers, threshold, timeout, beforeEach: beforeEachHook, afterEach: afterEachHook, }: {
     data: () => Promise<Array<{
         input: string;
+        name?: string;
     } & Record<string, any>>>;
     task: TaskFn;
     skipIf?: () => boolean;
     scorers: ScoreFn<any>[];
     threshold?: number | null;
     timeout?: number;
+    beforeEach?: () => void | Promise<void>;
+    afterEach?: () => void | Promise<void>;
 }): vitest.SuiteCollector<object>;
 declare function formatScores(scores: (Score & {
     name: string;
 })[]): string;
-/**
- * Wraps text to fit within a specified width, breaking at word boundaries.
- *
- * @param text - The text to wrap
- * @param width - The maximum width in characters (default: 80)
- * @returns The wrapped text with line breaks
- *
- * @example
- * ```javascript
- * const wrapped = wrapText("This is a very long text that needs to be wrapped to fit within an 80 character width.", 20);
- * console.log(wrapped);
- * // Output:
- * // This is a very
- * // long text that
- * // needs to be
- * // wrapped to fit
- * // within an 80
- * // character width.
- * ```
- */
-declare function wrapText(text: string, width?: number): string;
 
-export { type BaseScorerOptions as B, type EvalMatchers as E, type StructuredOutputScorerOptions as S, type ToolCall as T, ToolCallScorer, type ToolCallScorerConfig, type ToolCallScorerOptions, type StructuredOutputScorerConfig as a, StructuredOutputScorer as b, type TaskResult as c, type TaskFn as d, type Score as e, type ScoreFn as f, type ToEval as g, describeEval as h, formatScores as i, wrapText as w };
+export { type BaseScorerOptions as B, type EvalMatchers as E, StructuredOutputScorer as S, type TaskFn as T, ToolCallScorer, type ToolCallScorerConfig, type ToolCallScorerOptions, type StructuredOutputScorerConfig as a, type StructuredOutputScorerOptions as b, type Score as c, type ScoreFn as d, type TaskResult as e, type ToEval as f, type ToolCall as g, describeEval as h, formatScores as i };

package/dist/wrapText.d.mts

@@ -0,0 +1,23 @@
+/**
+ * Wraps text to fit within a specified width, breaking at word boundaries.
+ *
+ * @param text - The text to wrap
+ * @param width - The maximum width in characters (default: 80)
+ * @returns The wrapped text with line breaks
+ *
+ * @example
+ * ```javascript
+ * const wrapped = wrapText("This is a very long text that needs to be wrapped to fit within an 80 character width.", 20);
+ * console.log(wrapped);
+ * // Output:
+ * // This is a very
+ * // long text that
+ * // needs to be
+ * // wrapped to fit
+ * // within an 80
+ * // character width.
+ * ```
+ */
+declare function wrapText(text: string, width?: number): string;
+
+export { wrapText };

package/dist/wrapText.d.ts

@@ -0,0 +1,23 @@
+/**
+ * Wraps text to fit within a specified width, breaking at word boundaries.
+ *
+ * @param text - The text to wrap
+ * @param width - The maximum width in characters (default: 80)
+ * @returns The wrapped text with line breaks
+ *
+ * @example
+ * ```javascript
+ * const wrapped = wrapText("This is a very long text that needs to be wrapped to fit within an 80 character width.", 20);
+ * console.log(wrapped);
+ * // Output:
+ * // This is a very
+ * // long text that
+ * // needs to be
+ * // wrapped to fit
+ * // within an 80
+ * // character width.
+ * ```
+ */
+declare function wrapText(text: string, width?: number): string;
+
+export { wrapText };

package/dist/wrapText.js

@@ -0,0 +1,50 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+
+// src/wrapText.ts
+var wrapText_exports = {};
+__export(wrapText_exports, {
+  wrapText: () => wrapText
+});
+module.exports = __toCommonJS(wrapText_exports);
+function wrapText(text, width = 80) {
+  if (!text || text.length <= width) {
+    return text;
+  }
+  const words = text.split(/\s+/);
+  const lines = [];
+  let currentLine = "";
+  for (const word of words) {
+    if (currentLine.length + word.length + 1 > width) {
+      lines.push(currentLine.trim());
+      currentLine = word;
+    } else {
+      currentLine += (currentLine ? " " : "") + word;
+    }
+  }
+  if (currentLine) {
+    lines.push(currentLine);
+  }
+  return lines.join("\n");
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  wrapText
+});
+//# sourceMappingURL=wrapText.js.map

package/dist/wrapText.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/wrapText.ts"],"sourcesContent":["/**\n * Wraps text to fit within a specified width, breaking at word boundaries.\n *\n * @param text - The text to wrap\n * @param width - The maximum width in characters (default: 80)\n * @returns The wrapped text with line breaks\n *\n * @example\n * ```javascript\n * const wrapped = wrapText(\"This is a very long text that needs to be wrapped to fit within an 80 character width.\", 20);\n * console.log(wrapped);\n * // Output:\n * // This is a very\n * // long text that\n * // needs to be\n * // wrapped to fit\n * // within an 80\n * // character width.\n * ```\n */\nexport function wrapText(text: string, width = 80): string {\n  if (!text || text.length <= width) {\n    return text;\n  }\n\n  const words = text.split(/\\s+/);\n  const lines: string[] = [];\n  let currentLine = \"\";\n\n  for (const word of words) {\n    // If adding this word would exceed the width, start a new line\n    if (currentLine.length + word.length + 1 > width) {\n      lines.push(currentLine.trim());\n      currentLine = word;\n    } else {\n      // Add the word to the current line\n      currentLine += (currentLine ? \" \" : \"\") + word;\n    }\n  }\n\n  // Add the last line if it's not empty\n  if (currentLine) {\n    lines.push(currentLine);\n  }\n\n  return lines.join(\"\\n\");\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAoBO,SAAS,SAAS,MAAc,QAAQ,IAAY;AACzD,MAAI,CAAC,QAAQ,KAAK,UAAU,OAAO;AACjC,WAAO;AAAA,EACT;AAEA,QAAM,QAAQ,KAAK,MAAM,KAAK;AAC9B,QAAM,QAAkB,CAAC;AACzB,MAAI,cAAc;AAElB,aAAW,QAAQ,OAAO;AAExB,QAAI,YAAY,SAAS,KAAK,SAAS,IAAI,OAAO;AAChD,YAAM,KAAK,YAAY,KAAK,CAAC;AAC7B,oBAAc;AAAA,IAChB,OAAO;AAEL,sBAAgB,cAAc,MAAM,MAAM;AAAA,IAC5C;AAAA,EACF;AAGA,MAAI,aAAa;AACf,UAAM,KAAK,WAAW;AAAA,EACxB;AAEA,SAAO,MAAM,KAAK,IAAI;AACxB;","names":[]}

package/dist/wrapText.mjs

@@ -0,0 +1,25 @@
+// src/wrapText.ts
+function wrapText(text, width = 80) {
+  if (!text || text.length <= width) {
+    return text;
+  }
+  const words = text.split(/\s+/);
+  const lines = [];
+  let currentLine = "";
+  for (const word of words) {
+    if (currentLine.length + word.length + 1 > width) {
+      lines.push(currentLine.trim());
+      currentLine = word;
+    } else {
+      currentLine += (currentLine ? " " : "") + word;
+    }
+  }
+  if (currentLine) {
+    lines.push(currentLine);
+  }
+  return lines.join("\n");
+}
+export {
+  wrapText
+};
+//# sourceMappingURL=wrapText.mjs.map

package/dist/wrapText.mjs.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/wrapText.ts"],"sourcesContent":["/**\n * Wraps text to fit within a specified width, breaking at word boundaries.\n *\n * @param text - The text to wrap\n * @param width - The maximum width in characters (default: 80)\n * @returns The wrapped text with line breaks\n *\n * @example\n * ```javascript\n * const wrapped = wrapText(\"This is a very long text that needs to be wrapped to fit within an 80 character width.\", 20);\n * console.log(wrapped);\n * // Output:\n * // This is a very\n * // long text that\n * // needs to be\n * // wrapped to fit\n * // within an 80\n * // character width.\n * ```\n */\nexport function wrapText(text: string, width = 80): string {\n  if (!text || text.length <= width) {\n    return text;\n  }\n\n  const words = text.split(/\\s+/);\n  const lines: string[] = [];\n  let currentLine = \"\";\n\n  for (const word of words) {\n    // If adding this word would exceed the width, start a new line\n    if (currentLine.length + word.length + 1 > width) {\n      lines.push(currentLine.trim());\n      currentLine = word;\n    } else {\n      // Add the word to the current line\n      currentLine += (currentLine ? \" \" : \"\") + word;\n    }\n  }\n\n  // Add the last line if it's not empty\n  if (currentLine) {\n    lines.push(currentLine);\n  }\n\n  return lines.join(\"\\n\");\n}\n"],"mappings":";AAoBO,SAAS,SAAS,MAAc,QAAQ,IAAY;AACzD,MAAI,CAAC,QAAQ,KAAK,UAAU,OAAO;AACjC,WAAO;AAAA,EACT;AAEA,QAAM,QAAQ,KAAK,MAAM,KAAK;AAC9B,QAAM,QAAkB,CAAC;AACzB,MAAI,cAAc;AAElB,aAAW,QAAQ,OAAO;AAExB,QAAI,YAAY,SAAS,KAAK,SAAS,IAAI,OAAO;AAChD,YAAM,KAAK,YAAY,KAAK,CAAC;AAC7B,oBAAc;AAAA,IAChB,OAAO;AAEL,sBAAgB,cAAc,MAAM,MAAM;AAAA,IAC5C;AAAA,EACF;AAGA,MAAI,aAAa;AACf,UAAM,KAAK,WAAW;AAAA,EACxB;AAEA,SAAO,MAAM,KAAK,IAAI;AACxB;","names":[]}