vitest-evals 0.2.0 → 0.4.0

package/dist/wrapText.test.mjs

@@ -1,161 +0,0 @@
-var __defProp = Object.defineProperty;
-var __defProps = Object.defineProperties;
-var __getOwnPropDescs = Object.getOwnPropertyDescriptors;
-var __getOwnPropSymbols = Object.getOwnPropertySymbols;
-var __hasOwnProp = Object.prototype.hasOwnProperty;
-var __propIsEnum = Object.prototype.propertyIsEnumerable;
-var __defNormalProp = (obj, key, value) => key in obj ? __defProp(obj, key, { enumerable: true, configurable: true, writable: true, value }) : obj[key] = value;
-var __spreadValues = (a, b) => {
-  for (var prop in b || (b = {}))
-    if (__hasOwnProp.call(b, prop))
-      __defNormalProp(a, prop, b[prop]);
-  if (__getOwnPropSymbols)
-    for (var prop of __getOwnPropSymbols(b)) {
-      if (__propIsEnum.call(b, prop))
-        __defNormalProp(a, prop, b[prop]);
-    }
-  return a;
-};
-var __spreadProps = (a, b) => __defProps(a, __getOwnPropDescs(b));
-var __async = (__this, __arguments, generator) => {
-  return new Promise((resolve, reject) => {
-    var fulfilled = (value) => {
-      try {
-        step(generator.next(value));
-      } catch (e) {
-        reject(e);
-      }
-    };
-    var rejected = (value) => {
-      try {
-        step(generator.throw(value));
-      } catch (e) {
-        reject(e);
-      }
-    };
-    var step = (x) => x.done ? resolve(x.value) : Promise.resolve(x.value).then(fulfilled, rejected);
-    step((generator = generator.apply(__this, __arguments)).next());
-  });
-};
-
-// src/wrapText.test.ts
-import { describe as describe2, expect as expect2, test as test2 } from "vitest";
-
-// src/index.ts
-import { assert, describe, expect, test } from "vitest";
-import "vitest";
-expect.extend({
-  /**
-   * Evaluates a language model output against an expected answer using a scoring function.
-   *
-   * @param expected - The expected (ground truth) answer
-   * @param taskFn - Async function that processes the input and returns the model output
-   * @param scoreFn - Function that evaluates the model output against the expected answer
-   * @param threshold - Minimum acceptable score (0-1), defaults to 1.0
-   *
-   * @example
-   * ```javascript
-   * test("checks capital of France", async () => {
-   *   expect("What is the capital of France?").toEval(
-   *     "Paris",
-   *     async (input) => {
-   *       // Query LLM here
-   *       return "Paris";
-   *     },
-   *     checkFactuality,
-   *     0.8
-   *   );
-   * });
-   * ```
-   */
-  toEval: function toEval(input, expected, taskFn, scoreFn, threshold = 1) {
-    return __async(this, null, function* () {
-      var _a;
-      const { isNot } = this;
-      const output = yield taskFn(input);
-      let result = scoreFn({ input, expected, output });
-      if (result instanceof Promise) {
-        result = yield result;
-      }
-      return {
-        pass: ((_a = result.score) != null ? _a : 0) >= threshold,
-        message: () => formatScores([__spreadProps(__spreadValues({}, result), { name: scoreFn.name })])
-      };
-    });
-  }
-});
-function formatScores(scores) {
-  return scores.sort((a, b) => {
-    var _a, _b;
-    return ((_a = a.score) != null ? _a : 0) - ((_b = b.score) != null ? _b : 0);
-  }).map((s) => {
-    var _a, _b, _c, _d, _e, _f;
-    const scoreLine = `# ${s.name || "Unknown"} [${((_a = s.score) != null ? _a : 0).toFixed(1)}]`;
-    if (((_b = s.score) != null ? _b : 0) < 1 && ((_c = s.metadata) == null ? void 0 : _c.rationale) || ((_d = s.metadata) == null ? void 0 : _d.output)) {
-      return `${scoreLine}${((_e = s.metadata) == null ? void 0 : _e.rationale) ? `
-
-## Rationale
-
-${wrapText(s.metadata.rationale)}` : ""}${((_f = s.metadata) == null ? void 0 : _f.output) ? `
-
-## Response
-
-${wrapText(s.metadata.output)}` : ""}`;
-    }
-    return scoreLine;
-  }).join("\n\n");
-}
-function wrapText(text, width = 80) {
-  if (!text || text.length <= width) {
-    return text;
-  }
-  const words = text.split(/\s+/);
-  const lines = [];
-  let currentLine = "";
-  for (const word of words) {
-    if (currentLine.length + word.length + 1 > width) {
-      lines.push(currentLine.trim());
-      currentLine = word;
-    } else {
-      currentLine += (currentLine ? " " : "") + word;
-    }
-  }
-  if (currentLine) {
-    lines.push(currentLine);
-  }
-  return lines.join("\n");
-}
-
-// src/wrapText.test.ts
-describe2("wrapText", () => {
-  test2("should return the original text if it's shorter than the width", () => {
-    const text = "This is a short text";
-    expect2(wrapText(text, 30)).toBe(text);
-  });
-  test2("should wrap text at word boundaries", () => {
-    const text = "This is a very long text that needs to be wrapped";
-    const expected = "This is a very\nlong text that\nneeds to be\nwrapped";
-    expect2(wrapText(text, 15)).toBe(expected);
-  });
-  test2("should handle empty string", () => {
-    expect2(wrapText("")).toBe("");
-  });
-  test2("should handle null or undefined", () => {
-    expect2(wrapText(null)).toBe(null);
-    expect2(wrapText(void 0)).toBe(void 0);
-  });
-  test2("should use default width of 80", () => {
-    const text = "This is a very long text that needs to be wrapped to fit within an 80 character width.";
-    const result = wrapText(text);
-    const lines = result.split("\n");
-    for (const line of lines) {
-      expect2(line.length).toBeLessThanOrEqual(80);
-    }
-  });
-  test2("should handle text with multiple spaces", () => {
-    const text = "This    has    multiple    spaces";
-    const expected = "This has\nmultiple\nspaces";
-    expect2(wrapText(text, 10)).toBe(expected);
-  });
-});
-//# sourceMappingURL=wrapText.test.mjs.map

package/dist/wrapText.test.mjs.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../src/wrapText.test.ts","../src/index.ts"],"sourcesContent":["import { describe, expect, test } from \"vitest\";\nimport { wrapText } from \"./index\";\n\ndescribe(\"wrapText\", () => {\n  test(\"should return the original text if it's shorter than the width\", () => {\n    const text = \"This is a short text\";\n    expect(wrapText(text, 30)).toBe(text);\n  });\n\n  test(\"should wrap text at word boundaries\", () => {\n    const text = \"This is a very long text that needs to be wrapped\";\n    const expected = \"This is a very\\nlong text that\\nneeds to be\\nwrapped\";\n    expect(wrapText(text, 15)).toBe(expected);\n  });\n\n  test(\"should handle empty string\", () => {\n    expect(wrapText(\"\")).toBe(\"\");\n  });\n\n  test(\"should handle null or undefined\", () => {\n    expect(wrapText(null as any)).toBe(null);\n    expect(wrapText(undefined as any)).toBe(undefined);\n  });\n\n  test(\"should use default width of 80\", () => {\n    const text =\n      \"This is a very long text that needs to be wrapped to fit within an 80 character width.\";\n    const result = wrapText(text);\n    const lines = result.split(\"\\n\");\n\n    // Check that no line exceeds 80 characters\n    for (const line of lines) {\n      expect(line.length).toBeLessThanOrEqual(80);\n    }\n  });\n\n  test(\"should handle text with multiple spaces\", () => {\n    const text = \"This    has    multiple    spaces\";\n    const expected = \"This has\\nmultiple\\nspaces\";\n    expect(wrapText(text, 10)).toBe(expected);\n  });\n});\n","import { assert, describe, expect, test } from \"vitest\";\nimport \"vitest\";\n\nexport type TaskFn = (input: string) => Promise<string>;\n\nexport type Score = {\n  score: number | null;\n  metadata?: {\n    rationale?: string;\n    output?: string;\n  };\n};\n\nexport type ScoreFn = (opts: {\n  input: string;\n  output: string;\n  expected?: string;\n}) => Promise<Score> | Score;\n\nexport type ToEval<R = unknown> = (\n  expected: string,\n  taskFn: TaskFn,\n  scoreFn: ScoreFn,\n  threshold?: number,\n) => Promise<R>;\n\nexport interface EvalMatchers<R = unknown> {\n  toEval: ToEval<R>;\n}\n\ndeclare module \"vitest\" {\n  interface Assertion<T = any> extends EvalMatchers<T> {}\n  interface AsymmetricMatchersContaining extends EvalMatchers {}\n\n  interface TaskMeta {\n    eval?: {\n      scores: (Score & { name: string })[];\n      avgScore: number;\n    };\n  }\n}\n\nexpect.extend({\n  /**\n   * Evaluates a language model output against an expected answer using a scoring function.\n   *\n   * @param expected - The expected (ground truth) answer\n   * @param taskFn - Async function that processes the input and returns the model output\n   * @param scoreFn - Function that evaluates the model output against the expected answer\n   * @param threshold - Minimum acceptable score (0-1), defaults to 1.0\n   *\n   * @example\n   * ```javascript\n   * test(\"checks capital of France\", async () => {\n   *   expect(\"What is the capital of France?\").toEval(\n   *     \"Paris\",\n   *     async (input) => {\n   *       // Query LLM here\n   *       return \"Paris\";\n   *     },\n   *     checkFactuality,\n   *     0.8\n   *   );\n   * });\n   * ```\n   */\n  toEval: async function toEval(\n    input: string,\n    expected: string,\n    taskFn: TaskFn,\n    scoreFn: ScoreFn,\n    threshold = 1.0,\n  ) {\n    const { isNot } = this;\n\n    const output = await taskFn(input);\n\n    let result = scoreFn({ input, expected, output });\n    if (result instanceof Promise) {\n      result = await result;\n    }\n\n    return {\n      pass: (result.score ?? 0) >= threshold,\n      message: () => formatScores([{ ...result, name: scoreFn.name }]),\n    };\n  },\n});\n\n/**\n * Creates a test suite for evaluating language model outputs.\n *\n * @param name - The name of the test suite\n * @param options - Configuration options\n * @param options.data - Async function that returns an array of test cases with input and expected values\n * @param options.task - Function that processes the input and returns the model output\n * @param options.skipIf - Optional function that determines if tests should be skipped\n * @param options.scorers - Array of scoring functions that evaluate model outputs\n * @param options.threshold - Minimum acceptable average score (0-1), defaults to 1.0\n * @param options.timeout - Test timeout in milliseconds, defaults to 60000 (60s)\n *\n * @example\n * ```javascript\n * describeEval(\"capital cities test\", {\n *   data: async () => [{\n *     input: \"What is the capital of France?\",\n *     expected: \"Paris\"\n *   }],\n *   task: async (input) => {\n *     // Query LLM here\n *     return \"Paris\";\n *   },\n *   scorers: [checkFactuality],\n *   threshold: 0.8\n * });\n * ```\n */\nexport function describeEval(\n  name: string,\n  {\n    data,\n    task,\n    skipIf,\n    scorers,\n    threshold = 1.0,\n    // increase default test timeout as 5s is usually not enough for\n    // a single factuality check\n    timeout = 60000,\n  }: {\n    data: () => Promise<{ input: string; expected: string }[]>;\n    task: TaskFn;\n    skipIf?: () => boolean;\n    scorers: ScoreFn[];\n    threshold?: number | null;\n    timeout?: number;\n  },\n) {\n  return describe(name, async () => {\n    const testFn = skipIf ? test.skipIf(skipIf()) : test;\n    // TODO: should data just be a generator?\n    for (const { input, expected } of await data()) {\n      testFn(\n        input,\n        {\n          timeout,\n        },\n        async ({ task: testTask }) => {\n          const output = await task(input);\n\n          const scores = await Promise.all(\n            scorers.map((scorer) => {\n              const result = scorer({ input, expected, output });\n              if (result instanceof Promise) {\n                return result;\n              }\n              return new Promise<Score>((resolve) => resolve(result));\n            }),\n          );\n          const scoresWithName = scores.map((s, i) => ({\n            ...s,\n            name: scorers[i].name,\n          }));\n\n          const avgScore =\n            scores.reduce((acc, s) => acc + (s.score ?? 0), 0) / scores.length;\n\n          testTask.meta.eval = {\n            scores: scoresWithName,\n            avgScore,\n          };\n\n          if (threshold) {\n            assert(\n              avgScore >= threshold,\n              `Score: ${avgScore} below threshold: ${threshold}\\n\\n## Output:\\n${wrapText(output)}\\n\\n${formatScores(\n                scoresWithName,\n              )}`,\n            );\n          }\n        },\n      );\n    }\n  });\n}\n\nexport function formatScores(scores: (Score & { name: string })[]) {\n  return scores\n    .sort((a, b) => (a.score ?? 0) - (b.score ?? 0))\n    .map((s) => {\n      const scoreLine = `# ${s.name || \"Unknown\"} [${(s.score ?? 0).toFixed(1)}]`;\n      if (\n        ((s.score ?? 0) < 1.0 && s.metadata?.rationale) ||\n        s.metadata?.output\n      ) {\n        return `${scoreLine}${\n          s.metadata?.rationale\n            ? `\\n\\n## Rationale\\n\\n${wrapText(s.metadata.rationale)}`\n            : \"\"\n        }${s.metadata?.output ? `\\n\\n## Response\\n\\n${wrapText(s.metadata.output)}` : \"\"}`;\n      }\n      return scoreLine;\n    })\n    .join(\"\\n\\n\");\n}\n\n/**\n * Wraps text to fit within a specified width, breaking at word boundaries.\n *\n * @param text - The text to wrap\n * @param width - The maximum width in characters (default: 80)\n * @returns The wrapped text with line breaks\n *\n * @example\n * ```javascript\n * const wrapped = wrapText(\"This is a very long text that needs to be wrapped to fit within an 80 character width.\", 20);\n * console.log(wrapped);\n * // Output:\n * // This is a very\n * // long text that\n * // needs to be\n * // wrapped to fit\n * // within an 80\n * // character width.\n * ```\n */\nexport function wrapText(text: string, width = 80): string {\n  if (!text || text.length <= width) {\n    return text;\n  }\n\n  const words = text.split(/\\s+/);\n  const lines: string[] = [];\n  let currentLine = \"\";\n\n  for (const word of words) {\n    // If adding this word would exceed the width, start a new line\n    if (currentLine.length + word.length + 1 > width) {\n      lines.push(currentLine.trim());\n      currentLine = word;\n    } else {\n      // Add the word to the current line\n      currentLine += (currentLine ? \" \" : \"\") + word;\n    }\n  }\n\n  // Add the last line if it's not empty\n  if (currentLine) {\n    lines.push(currentLine);\n  }\n\n  return lines.join(\"\\n\");\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,SAAS,YAAAA,WAAU,UAAAC,SAAQ,QAAAC,aAAY;;;ACAvC,SAAS,QAAQ,UAAU,QAAQ,YAAY;AAC/C,OAAO;AAyCP,OAAO,OAAO;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAwBZ,QAAQ,SAAe,OACrB,OACA,UACA,QACA,SACA,YAAY,GACZ;AAAA;AAxEJ;AAyEI,YAAM,EAAE,MAAM,IAAI;AAElB,YAAM,SAAS,MAAM,OAAO,KAAK;AAEjC,UAAI,SAAS,QAAQ,EAAE,OAAO,UAAU,OAAO,CAAC;AAChD,UAAI,kBAAkB,SAAS;AAC7B,iBAAS,MAAM;AAAA,MACjB;AAEA,aAAO;AAAA,QACL,QAAO,YAAO,UAAP,YAAgB,MAAM;AAAA,QAC7B,SAAS,MAAM,aAAa,CAAC,iCAAK,SAAL,EAAa,MAAM,QAAQ,KAAK,EAAC,CAAC;AAAA,MACjE;AAAA,IACF;AAAA;AACF,CAAC;AAkGM,SAAS,aAAa,QAAsC;AACjE,SAAO,OACJ,KAAK,CAAC,GAAG,MAAG;AA3LjB;AA2LqB,oBAAE,UAAF,YAAW,OAAM,OAAE,UAAF,YAAW;AAAA,GAAE,EAC9C,IAAI,CAAC,MAAM;AA5LhB;AA6LM,UAAM,YAAY,KAAK,EAAE,QAAQ,SAAS,OAAM,OAAE,UAAF,YAAW,GAAG,QAAQ,CAAC,CAAC;AACxE,UACI,OAAE,UAAF,YAAW,KAAK,OAAO,OAAE,aAAF,mBAAY,gBACrC,OAAE,aAAF,mBAAY,SACZ;AACA,aAAO,GAAG,SAAS,KACjB,OAAE,aAAF,mBAAY,aACR;AAAA;AAAA;AAAA;AAAA,EAAuB,SAAS,EAAE,SAAS,SAAS,CAAC,KACrD,EACN,KAAG,OAAE,aAAF,mBAAY,UAAS;AAAA;AAAA;AAAA;AAAA,EAAsB,SAAS,EAAE,SAAS,MAAM,CAAC,KAAK,EAAE;AAAA,IAClF;AACA,WAAO;AAAA,EACT,CAAC,EACA,KAAK,MAAM;AAChB;AAsBO,SAAS,SAAS,MAAc,QAAQ,IAAY;AACzD,MAAI,CAAC,QAAQ,KAAK,UAAU,OAAO;AACjC,WAAO;AAAA,EACT;AAEA,QAAM,QAAQ,KAAK,MAAM,KAAK;AAC9B,QAAM,QAAkB,CAAC;AACzB,MAAI,cAAc;AAElB,aAAW,QAAQ,OAAO;AAExB,QAAI,YAAY,SAAS,KAAK,SAAS,IAAI,OAAO;AAChD,YAAM,KAAK,YAAY,KAAK,CAAC;AAC7B,oBAAc;AAAA,IAChB,OAAO;AAEL,sBAAgB,cAAc,MAAM,MAAM;AAAA,IAC5C;AAAA,EACF;AAGA,MAAI,aAAa;AACf,UAAM,KAAK,WAAW;AAAA,EACxB;AAEA,SAAO,MAAM,KAAK,IAAI;AACxB;;;ADxPAC,UAAS,YAAY,MAAM;AACzB,EAAAC,MAAK,kEAAkE,MAAM;AAC3E,UAAM,OAAO;AACb,IAAAC,QAAO,SAAS,MAAM,EAAE,CAAC,EAAE,KAAK,IAAI;AAAA,EACtC,CAAC;AAED,EAAAD,MAAK,uCAAuC,MAAM;AAChD,UAAM,OAAO;AACb,UAAM,WAAW;AACjB,IAAAC,QAAO,SAAS,MAAM,EAAE,CAAC,EAAE,KAAK,QAAQ;AAAA,EAC1C,CAAC;AAED,EAAAD,MAAK,8BAA8B,MAAM;AACvC,IAAAC,QAAO,SAAS,EAAE,CAAC,EAAE,KAAK,EAAE;AAAA,EAC9B,CAAC;AAED,EAAAD,MAAK,mCAAmC,MAAM;AAC5C,IAAAC,QAAO,SAAS,IAAW,CAAC,EAAE,KAAK,IAAI;AACvC,IAAAA,QAAO,SAAS,MAAgB,CAAC,EAAE,KAAK,MAAS;AAAA,EACnD,CAAC;AAED,EAAAD,MAAK,kCAAkC,MAAM;AAC3C,UAAM,OACJ;AACF,UAAM,SAAS,SAAS,IAAI;AAC5B,UAAM,QAAQ,OAAO,MAAM,IAAI;AAG/B,eAAW,QAAQ,OAAO;AACxB,MAAAC,QAAO,KAAK,MAAM,EAAE,oBAAoB,EAAE;AAAA,IAC5C;AAAA,EACF,CAAC;AAED,EAAAD,MAAK,2CAA2C,MAAM;AACpD,UAAM,OAAO;AACb,UAAM,WAAW;AACjB,IAAAC,QAAO,SAAS,MAAM,EAAE,CAAC,EAAE,KAAK,QAAQ;AAAA,EAC1C,CAAC;AACH,CAAC;","names":["describe","expect","test","describe","test","expect"]}