peerbench 0.0.2-alpha.0 → 0.0.2-alpha.2

package/dist/chunk-ZEWI24CV.js

@@ -0,0 +1,365 @@
+import {
+  PEERBENCH_NAMESPACE
+} from "./chunk-UHHHSYVE.js";
+import {
+  parseResponseAsJSON
+} from "./chunk-4UBK6452.js";
+
+// src/scorers/abstract.ts
+var AbstractScorer = class {
+};
+
+// src/scorers/regex.ts
+var RegexScorer = class extends AbstractScorer {
+  kind = `${PEERBENCH_NAMESPACE}/regex`;
+  async score(params) {
+    const allGroupNames = /* @__PURE__ */ new Set();
+    for (const pattern of params.patterns) {
+      const regexSource = pattern.regex.source;
+      const namedGroupRegex = /\(\?<(\w+)>/g;
+      while (true) {
+        const match = namedGroupRegex.exec(regexSource);
+        if (match === null) {
+          break;
+        }
+        if (match[1]) {
+          allGroupNames.add(match[1]);
+        }
+      }
+    }
+    const extractedValues = {};
+    for (const groupName of allGroupNames) {
+      extractedValues[groupName] = null;
+    }
+    const matchPreference = params.matchPreference ?? "last";
+    for (const pattern of params.patterns) {
+      const matches = Array.from(params.input.matchAll(pattern.regex));
+      const match = matchPreference === "first" ? matches[0] : matches.at(-1);
+      if (match && match.groups) {
+        let hasExtractedValue = false;
+        for (const [groupName, groupValue] of Object.entries(match.groups)) {
+          if (groupValue !== void 0) {
+            let value = groupValue;
+            if (pattern.transform) {
+              const transformed = pattern.transform(value);
+              if (transformed === void 0) {
+                continue;
+              }
+              value = transformed;
+            }
+            extractedValues[groupName] = value;
+            hasExtractedValue = true;
+          }
+        }
+        if (hasExtractedValue) {
+          break;
+        }
+      } else if (match) {
+        const captureGroupIndex = pattern.captureGroupIndex ?? 1;
+        const extractedValue = match[captureGroupIndex];
+        if (extractedValue !== void 0) {
+          if (pattern.transform) {
+            const transformed = pattern.transform(extractedValue);
+            if (transformed === void 0) {
+              continue;
+            }
+          }
+          break;
+        }
+      }
+    }
+    const allowPartial = params.allowPartialScoring ?? false;
+    let score = 0;
+    if (typeof params.expectedValue === "function") {
+      const validator = params.expectedValue;
+      const extractedEntries = Object.entries(extractedValues).filter(
+        ([, value]) => value !== null
+      );
+      if (extractedEntries.length === 0) {
+        score = 0;
+      } else {
+        if (allowPartial) {
+          const passingCount = extractedEntries.filter(
+            ([groupName, extractedValue]) => validator(groupName, extractedValue)
+          ).length;
+          score = passingCount / extractedEntries.length;
+        } else {
+          const allMatch = extractedEntries.every(
+            ([groupName, extractedValue]) => validator(groupName, extractedValue)
+          );
+          score = allMatch ? 1 : 0;
+        }
+      }
+    } else {
+      const expectedEntries = Object.entries(params.expectedValue);
+      const totalExpected = expectedEntries.length;
+      if (allowPartial) {
+        const matchingCount = expectedEntries.filter(([key, expectedValue]) => {
+          const extractedValue = extractedValues[key];
+          return extractedValue !== null && extractedValue === expectedValue;
+        }).length;
+        score = totalExpected > 0 ? matchingCount / totalExpected : 0;
+      } else {
+        const allMatch = expectedEntries.every(([key, expectedValue]) => {
+          const extractedValue = extractedValues[key];
+          return extractedValue !== null && extractedValue === expectedValue;
+        });
+        score = allMatch ? 1 : 0;
+      }
+    }
+    return {
+      value: score,
+      extractedAnswers: Object.fromEntries(
+        Object.entries(extractedValues).filter(
+          (entry) => entry[1] !== null
+        )
+      )
+    };
+  }
+};
+
+// src/scorers/mcq.ts
+var MCQScorer = class extends AbstractScorer {
+  kind = `${PEERBENCH_NAMESPACE}/mcq`;
+  regexScorer = new RegexScorer();
+  async score(params) {
+    const { response, choices, correctAnswers } = params;
+    const normalizedCorrectAnswers = correctAnswers.map(
+      (ca) => ca.toUpperCase()
+    );
+    const normalizedResponse = response.trim().toUpperCase();
+    if (normalizedCorrectAnswers.includes(normalizedResponse)) {
+      return {
+        value: 1,
+        extractedAnswers: [normalizedResponse]
+      };
+    }
+    const json = parseResponseAsJSON(response);
+    if (json !== void 0 && typeof json === "object") {
+      const extractedAnswer = json.answer !== void 0 ? getFirstLetter(json.answer) : void 0;
+      if (extractedAnswer !== void 0) {
+        const normalizedExtracted = extractedAnswer.trim().toUpperCase();
+        if (normalizedCorrectAnswers.includes(normalizedExtracted)) {
+          return {
+            value: 1,
+            extractedAnswers: [extractedAnswer]
+          };
+        }
+        return {
+          value: 0,
+          extractedAnswers: json.answer === void 0 ? [] : [extractedAnswer ?? String(json.answer)]
+        };
+      }
+    }
+    const patterns = [];
+    for (const answer of Object.values(params.choices)) {
+      const answerPatterns = this.buildPatternsForAnswer(answer);
+      patterns.push(...answerPatterns);
+    }
+    const validateAnswer = (groupName, extracted) => {
+      const normalizedExtracted = extracted.trim().toUpperCase();
+      if (normalizedCorrectAnswers.includes(normalizedExtracted)) {
+        return true;
+      }
+      const answerOption = Object.entries(choices).find(
+        ([, value]) => value.trim().toUpperCase() === extracted.trim().toUpperCase()
+      );
+      if (answerOption && normalizedCorrectAnswers.includes(answerOption[0].toUpperCase())) {
+        return true;
+      }
+      return false;
+    };
+    const regexParams = {
+      input: response,
+      patterns,
+      expectedValue: validateAnswer,
+      matchPreference: "last"
+    };
+    const result = await this.regexScorer.score(regexParams);
+    return {
+      value: result.value,
+      extractedAnswers: Object.entries(result.extractedAnswers).map(
+        ([, value]) => value
+      )
+    };
+  }
+  buildPatternsForAnswer(answerText) {
+    const escapedAnswer = escapeRegex(answerText);
+    return [
+      {
+        // "<!NO ANSWER!>" - This pattern matches but has no capture group, so it won't extract anything
+        regex: /<!NO ANSWER!>/g
+      },
+      // Specific patterns for the full answer text (checked first)
+      {
+        // "Answer is $\boxed{answer text}$"
+        regex: new RegExp(
+          `[Aa]nswer is \\$\\\\boxed\\{(?<answer>${escapedAnswer})\\}\\$`,
+          "g"
+        ),
+        transform: (value) => value.toUpperCase()
+      },
+      {
+        // "Answer is answer text"
+        regex: new RegExp(`[Aa]nswer is\\s+(?<answer>${escapedAnswer})`, "g"),
+        transform: (value) => value.toUpperCase()
+      },
+      {
+        // "Answer is **answer text**"
+        regex: new RegExp(
+          `[Aa]nswer is\\s+\\**(?<answer>${escapedAnswer})\\**`,
+          "g"
+        ),
+        transform: (value) => value.toUpperCase()
+      },
+      // Generic patterns (checked after specific patterns)
+      {
+        // "Answer is $\boxed{A}$."
+        regex: /[Aa]nswer is \$\\boxed\{(?<answer>[A-Z])\}\$\.?/g
+      },
+      {
+        // "Answer is A" - match single letter only when it's a complete standalone answer.
+        // Pattern matches: "Answer is" + whitespace + single letter + end or punctuation
+        regex: /[Aa]nswer is\s+(?<answer>[A-Z])(?=\s*$|[.,;:!?])/g
+      },
+      {
+        // "Answer is **A**"
+        regex: /[Aa]nswer is\s+\**(?<answer>[A-Z])\**/g
+      },
+      {
+        // "A: answer text"
+        regex: /(?<answer>[A-Z]):.+/g
+      },
+      {
+        // "A) answer text"
+        regex: /(?<answer>[A-Z])\)\s*.+/g
+      },
+      {
+        // "A)"
+        regex: /(?<answer>[A-Z])\)/g
+      }
+    ];
+  }
+};
+function getFirstLetter(text) {
+  const match = text.match(/[A-Za-z]/);
+  return match ? match[0].toUpperCase() : void 0;
+}
+function escapeRegex(str) {
+  return str.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}
+
+// src/scorers/llm-judge.ts
+import z from "zod";
+var LLMAsAJudgeScorer = class extends AbstractScorer {
+  kind = `${PEERBENCH_NAMESPACE}/llm-as-a-judge`;
+  provider;
+  constructor(config) {
+    super();
+    this.provider = config.provider;
+  }
+  async score(params) {
+    const criteria = normalizeWeights(params.criteria);
+    const systemPrompt = [];
+    const responseSchema = z.object({
+      results: z.array(
+        z.object({
+          id: z.string().describe("The id of the criterion"),
+          score: z.number().describe("The score of the criterion"),
+          explanation: z.string().describe("The explanation of the criterion")
+        })
+      ).describe("The results of the evaluation per criterion"),
+      explanation: z.string().describe(
+        `The overall explanation for the score (less than ${params.maxExplanationLength ?? 200} characters)`
+      ),
+      ...params.fieldsToExtract ?? {}
+    });
+    systemPrompt.push("You are a strict, fair evaluation judge.");
+    if (params.systemPrompt) {
+      systemPrompt.push(params.systemPrompt);
+    } else {
+      systemPrompt.push("Only use information from the rubric");
+    }
+    systemPrompt.push(
+      "For each criterion return an integer score within the provided scale and a very brief justification (less than 2 sentences)."
+    );
+    systemPrompt.push(
+      [
+        `Rubric: ${params.rubric}`,
+        `Criteria:`,
+        ...criteria.map(
+          (criterion) => `- ${criterion.id}: ${criterion.description} (weight: ${criterion.weight}, scale: ${criterion.scale?.min ?? 0}..${criterion.scale?.max ?? 5})`
+        )
+      ].join("\n")
+    );
+    const responseJSONSchema = responseSchema.toJSONSchema();
+    systemPrompt.push(
+      `Reply back with the following JSON schema (strict):
+${JSON.stringify(responseJSONSchema, null, 2)}
+`
+    );
+    const userPrompt = [`Answer: ${params.response}`];
+    const providerResponse = await this.provider.forward({
+      messages: [
+        {
+          role: "system",
+          content: systemPrompt.join("\n")
+        },
+        {
+          role: "user",
+          content: userPrompt.join("\n")
+        }
+      ],
+      model: params.model,
+      responseFormat: {
+        type: "json_schema",
+        json_schema: {
+          name: "judgeResult",
+          schema: responseJSONSchema
+        }
+      }
+    });
+    const parsed = responseSchema.parse(
+      parseResponseAsJSON(providerResponse.data)
+    );
+    const { explanation, results, ...extractedFields } = parsed;
+    return {
+      explanation,
+      results,
+      value: computeOverallScore(results, criteria),
+      extractedFields,
+      provider: this.provider.kind,
+      inputTokensUsed: providerResponse.inputTokensUsed,
+      outputTokensUsed: providerResponse.outputTokensUsed,
+      inputCost: providerResponse.inputCost,
+      outputCost: providerResponse.outputCost
+    };
+  }
+};
+function normalizeWeights(criteria) {
+  const sum = criteria.reduce((a, c) => a + (c.weight ?? 1), 0) || 1;
+  return criteria.map((c) => ({ ...c, weight: (c.weight ?? 1) / sum }));
+}
+function computeOverallScore(results, criteria) {
+  let total = 0;
+  for (const pc of results) {
+    const criterion = criteria.find((c) => c.id === pc.id);
+    const min = criterion?.scale?.min ?? 0;
+    const max = criterion?.scale?.max ?? 5;
+    const weight = criterion?.weight ?? 0;
+    const score = Number(pc.score);
+    if (!Number.isFinite(score)) continue;
+    const clamped = Math.max(min, Math.min(max, score));
+    const normalized01 = max === min ? 0 : (clamped - min) / (max - min);
+    total += normalized01 * weight;
+  }
+  return Math.max(0, Math.min(1, total));
+}
+
+export {
+  AbstractScorer,
+  RegexScorer,
+  MCQScorer,
+  LLMAsAJudgeScorer
+};
+//# sourceMappingURL=chunk-ZEWI24CV.js.map

package/dist/chunk-ZEWI24CV.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/scorers/abstract.ts","../src/scorers/regex.ts","../src/scorers/mcq.ts","../src/scorers/llm-judge.ts"],"sourcesContent":["export abstract class AbstractScorer {\n  abstract readonly kind: string;\n\n  abstract score(params: any): Promise<BaseScorerResult | null>;\n}\n\nexport type BaseScorerResult = {\n  value: number;\n  explanation?: string;\n  metadata?: Record<string, unknown>;\n  [key: string]: unknown;\n};\n","import { PEERBENCH_NAMESPACE } from \"@/constants\";\nimport { AbstractScorer, BaseScorerResult } from \"./abstract\";\n\nexport type RegexPattern = {\n  /**\n   * The regex pattern to match against the response\n   */\n  regex: RegExp;\n\n  /**\n   * The index of the capture group to extract (1-based, like match[1])\n   * If not provided, defaults to 1 (first capture group)\n   */\n  captureGroupIndex?: number;\n\n  /**\n   * Optional function to transform the extracted value before validation\n   */\n  transform?: (value: string) => string | undefined;\n};\n\nexport type RegexScorerParams = {\n  /**\n   * The input text to score\n   */\n  input: string;\n\n  /**\n   * Array of regex patterns to try (in order, first match wins)\n   */\n  patterns: RegexPattern[];\n\n  /**\n   * Expected value(s) to match against. Can be a record of expected values for named groups, or a validation function\n   */\n  expectedValue:\n    | Record<string, string>\n    | ((groupName: string, match: string) => boolean);\n\n  /**\n   * Optional: Which match to use when multiple matches are found\n   * Defaults to \"last\" (uses the last match found)\n   */\n  matchPreference?: \"first\" | \"last\";\n\n  /**\n   * Optional: If true, allows partial scoring based on how many groups match\n   * For example, if 2 groups are expected and only 1 matches, score would be 0.5\n   * Defaults to false (all-or-nothing scoring)\n   */\n  allowPartialScoring?: boolean;\n};\n\n/**\n * Generic Regex scorer. It scores the given input against a set of regex patterns.\n */\nexport class RegexScorer extends AbstractScorer {\n  override readonly kind = `${PEERBENCH_NAMESPACE}/regex` as const;\n\n  override async score(params: RegexScorerParams) {\n    // Collect all named group names from all patterns\n    const allGroupNames = new Set<string>();\n    for (const pattern of params.patterns) {\n      const regexSource = pattern.regex.source;\n      const namedGroupRegex = /\\(\\?<(\\w+)>/g;\n\n      while (true) {\n        const match = namedGroupRegex.exec(regexSource);\n        if (match === null) {\n          break;\n        }\n\n        if (match[1]) {\n          allGroupNames.add(match[1]);\n        }\n      }\n    }\n\n    // Initialize result object with all group names set to null (aka not found yet)\n    const extractedValues: Record<string, string | null> = {};\n    for (const groupName of allGroupNames) {\n      extractedValues[groupName] = null;\n    }\n\n    // Try regex patterns in order, stop at first successful match\n    const matchPreference = params.matchPreference ?? \"last\";\n    for (const pattern of params.patterns) {\n      const matches = Array.from(params.input.matchAll(pattern.regex));\n      const match = matchPreference === \"first\" ? matches[0] : matches.at(-1);\n\n      if (match && match.groups) {\n        // Extract all named groups from this match\n        let hasExtractedValue = false;\n        for (const [groupName, groupValue] of Object.entries(match.groups)) {\n          if (groupValue !== undefined) {\n            let value = groupValue;\n\n            // Apply transformation if provided\n            if (pattern.transform) {\n              const transformed = pattern.transform(value);\n              if (transformed === undefined) {\n                continue;\n              }\n              value = transformed;\n            }\n\n            extractedValues[groupName] = value;\n            hasExtractedValue = true;\n          }\n        }\n        // If we extracted at least one value, stop processing further patterns\n        if (hasExtractedValue) {\n          break;\n        }\n      } else if (match) {\n        // Fallback to captureGroupIndex if no named groups\n        // For unnamed groups, we can't add to extractedValues by name\n        // They are only used for scoring if no named groups are found\n        const captureGroupIndex = pattern.captureGroupIndex ?? 1;\n        const extractedValue = match[captureGroupIndex];\n\n        if (extractedValue !== undefined) {\n          // Apply transformation if provided\n          if (pattern.transform) {\n            const transformed = pattern.transform(extractedValue);\n            if (transformed === undefined) {\n              continue;\n            }\n            // For unnamed groups, we can't store in extractedValues\n            // but we can use it for scoring if no named groups were found\n          }\n          // If we have an extracted value (even if unnamed), stop processing\n          // Note: This is a fallback case, so we break here too\n          break;\n        }\n      }\n    }\n\n    // Calculate score based on matched value\n    const allowPartial = params.allowPartialScoring ?? false;\n    let score = 0;\n\n    if (typeof params.expectedValue === \"function\") {\n      // To get proper type inference, cast it using \"as\"\n      const validator = params.expectedValue as typeof params.expectedValue;\n      const extractedEntries = Object.entries(extractedValues).filter(\n        ([, value]) => value !== null\n      );\n\n      // If no values were extracted, score is 0\n      if (extractedEntries.length === 0) {\n        score = 0;\n      } else {\n        if (allowPartial) {\n          // Count how many extracted values pass validation\n          const passingCount = extractedEntries.filter(\n            ([groupName, extractedValue]) =>\n              validator(groupName, extractedValue as string)\n          ).length;\n          score = passingCount / extractedEntries.length;\n        } else {\n          // All extracted values must pass validation\n          const allMatch = extractedEntries.every(\n            ([groupName, extractedValue]) =>\n              validator(groupName, extractedValue as string)\n          );\n          score = allMatch ? 1 : 0;\n        }\n      }\n    } else {\n      const expectedEntries = Object.entries(params.expectedValue);\n      const totalExpected = expectedEntries.length;\n\n      if (allowPartial) {\n        // Count how many expected values match their extracted values\n        const matchingCount = expectedEntries.filter(([key, expectedValue]) => {\n          const extractedValue = extractedValues[key];\n          return extractedValue !== null && extractedValue === expectedValue;\n        }).length;\n        score = totalExpected > 0 ? matchingCount / totalExpected : 0;\n      } else {\n        // All expected values must match\n        const allMatch = expectedEntries.every(([key, expectedValue]) => {\n          const extractedValue = extractedValues[key];\n          return extractedValue !== null && extractedValue === expectedValue;\n        });\n        score = allMatch ? 1 : 0;\n      }\n    }\n\n    return {\n      value: score,\n      extractedAnswers: Object.fromEntries(\n        Object.entries(extractedValues).filter(\n          (entry): entry is [string, string] => entry[1] !== null\n        )\n      ),\n    } satisfies BaseScorerResult;\n  }\n}\n","import { parseResponseAsJSON } from \"@/utils\";\nimport { AbstractScorer, BaseScorerResult } from \"./abstract\";\nimport { RegexScorer, RegexPattern, RegexScorerParams } from \"./regex\";\nimport { PEERBENCH_NAMESPACE } from \"@/constants\";\n\nexport type MCQScorerParams = {\n  response: string;\n  choices: Record<string, string>;\n  correctAnswers: string[];\n};\n\nexport class MCQScorer extends AbstractScorer {\n  override readonly kind = `${PEERBENCH_NAMESPACE}/mcq` as const;\n  private regexScorer = new RegexScorer();\n\n  async score(params: MCQScorerParams): Promise<\n    BaseScorerResult & {\n      extractedAnswers: string[];\n    }\n  > {\n    const { response, choices, correctAnswers } = params;\n    const normalizedCorrectAnswers = correctAnswers.map((ca) =>\n      ca.toUpperCase()\n    );\n\n    // Direct answer comparison\n    const normalizedResponse = response.trim().toUpperCase();\n    if (normalizedCorrectAnswers.includes(normalizedResponse)) {\n      return {\n        value: 1,\n        extractedAnswers: [normalizedResponse],\n      };\n    }\n\n    // Try to parse the response as JSON (original behavior: returns early if parsed)\n    const json = parseResponseAsJSON<{ answer: string }>(response);\n    if (json !== undefined && typeof json === \"object\") {\n      const extractedAnswer =\n        json.answer !== undefined ? getFirstLetter(json.answer) : undefined;\n\n      if (extractedAnswer !== undefined) {\n        const normalizedExtracted = extractedAnswer.trim().toUpperCase();\n        if (normalizedCorrectAnswers.includes(normalizedExtracted)) {\n          return {\n            value: 1,\n            extractedAnswers: [extractedAnswer],\n          };\n        }\n\n        // Response parsed as JSON but does not represent the correct answer\n        return {\n          value: 0,\n          extractedAnswers:\n            json.answer === undefined\n              ? []\n              : [extractedAnswer ?? String(json.answer)],\n        };\n      }\n    }\n\n    // Build patterns for all correctAnswers\n    const patterns: RegexPattern[] = [];\n    for (const answer of Object.values(params.choices)) {\n      const answerPatterns = this.buildPatternsForAnswer(answer);\n      patterns.push(...answerPatterns);\n    }\n\n    // Create validation function that handles choices matching\n    // New RegexScorer API expects (groupName: string, match: string) => boolean\n    const validateAnswer = (groupName: string, extracted: string): boolean => {\n      const normalizedExtracted = extracted.trim().toUpperCase();\n\n      // Check if extracted value is in correctAnswers\n      if (normalizedCorrectAnswers.includes(normalizedExtracted)) {\n        return true;\n      }\n\n      // Check if extracted text matches any choice value, and that choice key is correct\n      // Note: When transform uppercases the value, we need to compare case-insensitively\n      const answerOption = Object.entries(choices).find(\n        ([, value]) =>\n          value.trim().toUpperCase() === extracted.trim().toUpperCase()\n      );\n\n      if (\n        answerOption &&\n        normalizedCorrectAnswers.includes(answerOption[0].toUpperCase())\n      ) {\n        return true;\n      }\n\n      return false;\n    };\n\n    // Build regex scorer params\n    const regexParams: RegexScorerParams = {\n      input: response,\n      patterns,\n      expectedValue: validateAnswer,\n      matchPreference: \"last\",\n    };\n\n    // Call regex scorer\n    const result = await this.regexScorer.score(regexParams);\n\n    return {\n      value: result.value,\n      extractedAnswers: Object.entries(result.extractedAnswers).map(\n        ([, value]) => value\n      ),\n    };\n  }\n\n  private buildPatternsForAnswer(answerText: string): RegexPattern[] {\n    const escapedAnswer = escapeRegex(answerText);\n\n    return [\n      {\n        // \"<!NO ANSWER!>\" - This pattern matches but has no capture group, so it won't extract anything\n        regex: /<!NO ANSWER!>/g,\n      },\n      // Specific patterns for the full answer text (checked first)\n      {\n        // \"Answer is $\\boxed{answer text}$\"\n        regex: new RegExp(\n          `[Aa]nswer is \\\\$\\\\\\\\boxed\\\\{(?<answer>${escapedAnswer})\\\\}\\\\$`,\n          \"g\"\n        ),\n        transform: (value: string) => value.toUpperCase(),\n      },\n      {\n        // \"Answer is answer text\"\n        regex: new RegExp(`[Aa]nswer is\\\\s+(?<answer>${escapedAnswer})`, \"g\"),\n        transform: (value: string) => value.toUpperCase(),\n      },\n      {\n        // \"Answer is **answer text**\"\n        regex: new RegExp(\n          `[Aa]nswer is\\\\s+\\\\**(?<answer>${escapedAnswer})\\\\**`,\n          \"g\"\n        ),\n        transform: (value: string) => value.toUpperCase(),\n      },\n      // Generic patterns (checked after specific patterns)\n      {\n        // \"Answer is $\\boxed{A}$.\"\n        regex: /[Aa]nswer is \\$\\\\boxed\\{(?<answer>[A-Z])\\}\\$\\.?/g,\n      },\n      {\n        // \"Answer is A\" - match single letter only when it's a complete standalone answer.\n        // Pattern matches: \"Answer is\" + whitespace + single letter + end or punctuation\n        regex: /[Aa]nswer is\\s+(?<answer>[A-Z])(?=\\s*$|[.,;:!?])/g,\n      },\n      {\n        // \"Answer is **A**\"\n        regex: /[Aa]nswer is\\s+\\**(?<answer>[A-Z])\\**/g,\n      },\n      {\n        // \"A: answer text\"\n        regex: /(?<answer>[A-Z]):.+/g,\n      },\n      {\n        // \"A) answer text\"\n        regex: /(?<answer>[A-Z])\\)\\s*.+/g,\n      },\n      {\n        // \"A)\"\n        regex: /(?<answer>[A-Z])\\)/g,\n      },\n    ];\n  }\n}\n\nfunction getFirstLetter(text: string): string | undefined {\n  const match = text.match(/[A-Za-z]/);\n  return match ? match[0].toUpperCase() : undefined;\n}\n\nfunction escapeRegex(str: string): string {\n  return str.replace(/[.*+?^${}()|[\\]\\\\]/g, \"\\\\$&\");\n}\n","import { AbstractLLMProvider } from \"@/providers/abstract/llm\";\nimport { parseResponseAsJSON } from \"@/utils\";\nimport { RateLimiter } from \"@/utils/rate-limiter\";\nimport { AbstractScorer, BaseScorerResult } from \"./abstract\";\nimport { PEERBENCH_NAMESPACE } from \"@/constants\";\nimport z from \"zod\";\n\nexport class LLMAsAJudgeScorer extends AbstractScorer {\n  override readonly kind = `${PEERBENCH_NAMESPACE}/llm-as-a-judge` as const;\n\n  private provider: AbstractLLMProvider;\n\n  constructor(config: {\n    provider: AbstractLLMProvider;\n    rateLimiter?: RateLimiter;\n  }) {\n    super();\n    this.provider = config.provider;\n  }\n\n  override async score<T extends z.ZodRawShape>(\n    params: LLMAsAJudgeScoreParams & { fieldsToExtract: T }\n  ): Promise<ScorerResultWithExtractedFields<T> | null>;\n  override async score(\n    params: LLMAsAJudgeScoreParams & { fieldsToExtract?: never }\n  ): Promise<ScorerResultWithoutExtractedFields | null>;\n  override async score<T extends z.ZodRawShape>(\n    params: LLMAsAJudgeScoreParams & { fieldsToExtract?: T }\n  ): Promise<\n    | ScorerResultWithoutExtractedFields\n    | ScorerResultWithExtractedFields<T>\n    | null\n  > {\n    const criteria = normalizeWeights(params.criteria);\n    const systemPrompt = [];\n    const responseSchema = z.object({\n      results: z\n        .array(\n          z.object({\n            id: z.string().describe(\"The id of the criterion\"),\n            score: z.number().describe(\"The score of the criterion\"),\n            explanation: z\n              .string()\n              .describe(\"The explanation of the criterion\"),\n          })\n        )\n        .describe(\"The results of the evaluation per criterion\"),\n\n      explanation: z\n        .string()\n        .describe(\n          `The overall explanation for the score (less than ${params.maxExplanationLength ?? 200} characters)`\n        ),\n\n      ...(params.fieldsToExtract ?? {}),\n    });\n\n    systemPrompt.push(\"You are a strict, fair evaluation judge.\");\n\n    if (params.systemPrompt) {\n      systemPrompt.push(params.systemPrompt);\n    } else {\n      systemPrompt.push(\"Only use information from the rubric\");\n    }\n\n    systemPrompt.push(\n      \"For each criterion return an integer score within the provided scale and a very brief justification (less than 2 sentences).\"\n    );\n    systemPrompt.push(\n      [\n        `Rubric: ${params.rubric}`,\n        `Criteria:`,\n        ...criteria.map(\n          (criterion) =>\n            `- ${criterion.id}: ${criterion.description} (weight: ${criterion.weight}, scale: ${criterion.scale?.min ?? 0}..${criterion.scale?.max ?? 5})`\n        ),\n      ].join(\"\\n\")\n    );\n\n    const responseJSONSchema = responseSchema.toJSONSchema();\n    systemPrompt.push(\n      `Reply back with the following JSON schema (strict):\\n${JSON.stringify(responseJSONSchema, null, 2)}\\n`\n    );\n\n    const userPrompt = [`Answer: ${params.response}`];\n    const providerResponse = await this.provider.forward({\n      messages: [\n        {\n          role: \"system\",\n          content: systemPrompt.join(\"\\n\"),\n        },\n        {\n          role: \"user\",\n          content: userPrompt.join(\"\\n\"),\n        },\n      ],\n      model: params.model,\n      responseFormat: {\n        type: \"json_schema\",\n        json_schema: {\n          name: \"judgeResult\",\n          schema: responseJSONSchema,\n        },\n      },\n    });\n\n    const parsed = responseSchema.parse(\n      parseResponseAsJSON(providerResponse.data)\n    );\n\n    const { explanation, results, ...extractedFields } = parsed;\n\n    return {\n      explanation,\n      results,\n      value: computeOverallScore(results, criteria),\n      extractedFields: extractedFields as z.infer<z.ZodObject<T>>,\n\n      provider: this.provider.kind,\n      inputTokensUsed: providerResponse.inputTokensUsed,\n      outputTokensUsed: providerResponse.outputTokensUsed,\n      inputCost: providerResponse.inputCost,\n      outputCost: providerResponse.outputCost,\n    };\n  }\n}\n\nexport type LLMAsAJudgeCriterion = {\n  id: string;\n  description: string;\n  weight: number;\n  scale?: {\n    min: number;\n    max: number;\n  };\n};\n\nexport type LLMAsAJudgeScoreParams = {\n  model: string;\n  response: string;\n  rubric: string;\n  criteria: LLMAsAJudgeCriterion[];\n\n  systemPrompt?: string;\n  maxExplanationLength?: number;\n};\n\ntype ScorerResultWithoutExtractedFields = BaseScorerResult & {\n  results: {\n    id: string;\n    score: number;\n    explanation: string;\n  }[];\n\n  provider: string;\n  inputTokensUsed?: number;\n  outputTokensUsed?: number;\n  inputCost?: string;\n  outputCost?: string;\n};\n\ntype ScorerResultWithExtractedFields<T extends z.ZodRawShape> =\n  ScorerResultWithoutExtractedFields & {\n    extractedFields: z.infer<z.ZodObject<T>>;\n  };\n\nfunction normalizeWeights(\n  criteria: LLMAsAJudgeCriterion[]\n): LLMAsAJudgeCriterion[] {\n  const sum = criteria.reduce((a, c) => a + (c.weight ?? 1), 0) || 1;\n  return criteria.map((c) => ({ ...c, weight: (c.weight ?? 1) / sum }));\n}\n\nfunction computeOverallScore(\n  results: ScorerResultWithoutExtractedFields[\"results\"],\n  criteria: LLMAsAJudgeCriterion[]\n): number {\n  let total = 0;\n  for (const pc of results) {\n    const criterion = criteria.find((c) => c.id === pc.id);\n    const min = criterion?.scale?.min ?? 0;\n    const max = criterion?.scale?.max ?? 5;\n    const weight = criterion?.weight ?? 0;\n\n    const score = Number(pc.score);\n    if (!Number.isFinite(score)) continue;\n\n    const clamped = Math.max(min, Math.min(max, score));\n    const normalized01 = max === min ? 0 : (clamped - min) / (max - min);\n    total += normalized01 * weight;\n  }\n  return Math.max(0, Math.min(1, total));\n}\n"],"mappings":";;;;;;;;AAAO,IAAe,iBAAf,MAA8B;AAIrC;;;ACoDO,IAAM,cAAN,cAA0B,eAAe;AAAA,EAC5B,OAAO,GAAG,mBAAmB;AAAA,EAE/C,MAAe,MAAM,QAA2B;AAE9C,UAAM,gBAAgB,oBAAI,IAAY;AACtC,eAAW,WAAW,OAAO,UAAU;AACrC,YAAM,cAAc,QAAQ,MAAM;AAClC,YAAM,kBAAkB;AAExB,aAAO,MAAM;AACX,cAAM,QAAQ,gBAAgB,KAAK,WAAW;AAC9C,YAAI,UAAU,MAAM;AAClB;AAAA,QACF;AAEA,YAAI,MAAM,CAAC,GAAG;AACZ,wBAAc,IAAI,MAAM,CAAC,CAAC;AAAA,QAC5B;AAAA,MACF;AAAA,IACF;AAGA,UAAM,kBAAiD,CAAC;AACxD,eAAW,aAAa,eAAe;AACrC,sBAAgB,SAAS,IAAI;AAAA,IAC/B;AAGA,UAAM,kBAAkB,OAAO,mBAAmB;AAClD,eAAW,WAAW,OAAO,UAAU;AACrC,YAAM,UAAU,MAAM,KAAK,OAAO,MAAM,SAAS,QAAQ,KAAK,CAAC;AAC/D,YAAM,QAAQ,oBAAoB,UAAU,QAAQ,CAAC,IAAI,QAAQ,GAAG,EAAE;AAEtE,UAAI,SAAS,MAAM,QAAQ;AAEzB,YAAI,oBAAoB;AACxB,mBAAW,CAAC,WAAW,UAAU,KAAK,OAAO,QAAQ,MAAM,MAAM,GAAG;AAClE,cAAI,eAAe,QAAW;AAC5B,gBAAI,QAAQ;AAGZ,gBAAI,QAAQ,WAAW;AACrB,oBAAM,cAAc,QAAQ,UAAU,KAAK;AAC3C,kBAAI,gBAAgB,QAAW;AAC7B;AAAA,cACF;AACA,sBAAQ;AAAA,YACV;AAEA,4BAAgB,SAAS,IAAI;AAC7B,gCAAoB;AAAA,UACtB;AAAA,QACF;AAEA,YAAI,mBAAmB;AACrB;AAAA,QACF;AAAA,MACF,WAAW,OAAO;AAIhB,cAAM,oBAAoB,QAAQ,qBAAqB;AACvD,cAAM,iBAAiB,MAAM,iBAAiB;AAE9C,YAAI,mBAAmB,QAAW;AAEhC,cAAI,QAAQ,WAAW;AACrB,kBAAM,cAAc,QAAQ,UAAU,cAAc;AACpD,gBAAI,gBAAgB,QAAW;AAC7B;AAAA,YACF;AAAA,UAGF;AAGA;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAGA,UAAM,eAAe,OAAO,uBAAuB;AACnD,QAAI,QAAQ;AAEZ,QAAI,OAAO,OAAO,kBAAkB,YAAY;AAE9C,YAAM,YAAY,OAAO;AACzB,YAAM,mBAAmB,OAAO,QAAQ,eAAe,EAAE;AAAA,QACvD,CAAC,CAAC,EAAE,KAAK,MAAM,UAAU;AAAA,MAC3B;AAGA,UAAI,iBAAiB,WAAW,GAAG;AACjC,gBAAQ;AAAA,MACV,OAAO;AACL,YAAI,cAAc;AAEhB,gBAAM,eAAe,iBAAiB;AAAA,YACpC,CAAC,CAAC,WAAW,cAAc,MACzB,UAAU,WAAW,cAAwB;AAAA,UACjD,EAAE;AACF,kBAAQ,eAAe,iBAAiB;AAAA,QAC1C,OAAO;AAEL,gBAAM,WAAW,iBAAiB;AAAA,YAChC,CAAC,CAAC,WAAW,cAAc,MACzB,UAAU,WAAW,cAAwB;AAAA,UACjD;AACA,kBAAQ,WAAW,IAAI;AAAA,QACzB;AAAA,MACF;AAAA,IACF,OAAO;AACL,YAAM,kBAAkB,OAAO,QAAQ,OAAO,aAAa;AAC3D,YAAM,gBAAgB,gBAAgB;AAEtC,UAAI,cAAc;AAEhB,cAAM,gBAAgB,gBAAgB,OAAO,CAAC,CAAC,KAAK,aAAa,MAAM;AACrE,gBAAM,iBAAiB,gBAAgB,GAAG;AAC1C,iBAAO,mBAAmB,QAAQ,mBAAmB;AAAA,QACvD,CAAC,EAAE;AACH,gBAAQ,gBAAgB,IAAI,gBAAgB,gBAAgB;AAAA,MAC9D,OAAO;AAEL,cAAM,WAAW,gBAAgB,MAAM,CAAC,CAAC,KAAK,aAAa,MAAM;AAC/D,gBAAM,iBAAiB,gBAAgB,GAAG;AAC1C,iBAAO,mBAAmB,QAAQ,mBAAmB;AAAA,QACvD,CAAC;AACD,gBAAQ,WAAW,IAAI;AAAA,MACzB;AAAA,IACF;AAEA,WAAO;AAAA,MACL,OAAO;AAAA,MACP,kBAAkB,OAAO;AAAA,QACvB,OAAO,QAAQ,eAAe,EAAE;AAAA,UAC9B,CAAC,UAAqC,MAAM,CAAC,MAAM;AAAA,QACrD;AAAA,MACF;AAAA,IACF;AAAA,EACF;AACF;;;AC5LO,IAAM,YAAN,cAAwB,eAAe;AAAA,EAC1B,OAAO,GAAG,mBAAmB;AAAA,EACvC,cAAc,IAAI,YAAY;AAAA,EAEtC,MAAM,MAAM,QAIV;AACA,UAAM,EAAE,UAAU,SAAS,eAAe,IAAI;AAC9C,UAAM,2BAA2B,eAAe;AAAA,MAAI,CAAC,OACnD,GAAG,YAAY;AAAA,IACjB;AAGA,UAAM,qBAAqB,SAAS,KAAK,EAAE,YAAY;AACvD,QAAI,yBAAyB,SAAS,kBAAkB,GAAG;AACzD,aAAO;AAAA,QACL,OAAO;AAAA,QACP,kBAAkB,CAAC,kBAAkB;AAAA,MACvC;AAAA,IACF;AAGA,UAAM,OAAO,oBAAwC,QAAQ;AAC7D,QAAI,SAAS,UAAa,OAAO,SAAS,UAAU;AAClD,YAAM,kBACJ,KAAK,WAAW,SAAY,eAAe,KAAK,MAAM,IAAI;AAE5D,UAAI,oBAAoB,QAAW;AACjC,cAAM,sBAAsB,gBAAgB,KAAK,EAAE,YAAY;AAC/D,YAAI,yBAAyB,SAAS,mBAAmB,GAAG;AAC1D,iBAAO;AAAA,YACL,OAAO;AAAA,YACP,kBAAkB,CAAC,eAAe;AAAA,UACpC;AAAA,QACF;AAGA,eAAO;AAAA,UACL,OAAO;AAAA,UACP,kBACE,KAAK,WAAW,SACZ,CAAC,IACD,CAAC,mBAAmB,OAAO,KAAK,MAAM,CAAC;AAAA,QAC/C;AAAA,MACF;AAAA,IACF;AAGA,UAAM,WAA2B,CAAC;AAClC,eAAW,UAAU,OAAO,OAAO,OAAO,OAAO,GAAG;AAClD,YAAM,iBAAiB,KAAK,uBAAuB,MAAM;AACzD,eAAS,KAAK,GAAG,cAAc;AAAA,IACjC;AAIA,UAAM,iBAAiB,CAAC,WAAmB,cAA+B;AACxE,YAAM,sBAAsB,UAAU,KAAK,EAAE,YAAY;AAGzD,UAAI,yBAAyB,SAAS,mBAAmB,GAAG;AAC1D,eAAO;AAAA,MACT;AAIA,YAAM,eAAe,OAAO,QAAQ,OAAO,EAAE;AAAA,QAC3C,CAAC,CAAC,EAAE,KAAK,MACP,MAAM,KAAK,EAAE,YAAY,MAAM,UAAU,KAAK,EAAE,YAAY;AAAA,MAChE;AAEA,UACE,gBACA,yBAAyB,SAAS,aAAa,CAAC,EAAE,YAAY,CAAC,GAC/D;AACA,eAAO;AAAA,MACT;AAEA,aAAO;AAAA,IACT;AAGA,UAAM,cAAiC;AAAA,MACrC,OAAO;AAAA,MACP;AAAA,MACA,eAAe;AAAA,MACf,iBAAiB;AAAA,IACnB;AAGA,UAAM,SAAS,MAAM,KAAK,YAAY,MAAM,WAAW;AAEvD,WAAO;AAAA,MACL,OAAO,OAAO;AAAA,MACd,kBAAkB,OAAO,QAAQ,OAAO,gBAAgB,EAAE;AAAA,QACxD,CAAC,CAAC,EAAE,KAAK,MAAM;AAAA,MACjB;AAAA,IACF;AAAA,EACF;AAAA,EAEQ,uBAAuB,YAAoC;AACjE,UAAM,gBAAgB,YAAY,UAAU;AAE5C,WAAO;AAAA,MACL;AAAA;AAAA,QAEE,OAAO;AAAA,MACT;AAAA;AAAA,MAEA;AAAA;AAAA,QAEE,OAAO,IAAI;AAAA,UACT,yCAAyC,aAAa;AAAA,UACtD;AAAA,QACF;AAAA,QACA,WAAW,CAAC,UAAkB,MAAM,YAAY;AAAA,MAClD;AAAA,MACA;AAAA;AAAA,QAEE,OAAO,IAAI,OAAO,6BAA6B,aAAa,KAAK,GAAG;AAAA,QACpE,WAAW,CAAC,UAAkB,MAAM,YAAY;AAAA,MAClD;AAAA,MACA;AAAA;AAAA,QAEE,OAAO,IAAI;AAAA,UACT,iCAAiC,aAAa;AAAA,UAC9C;AAAA,QACF;AAAA,QACA,WAAW,CAAC,UAAkB,MAAM,YAAY;AAAA,MAClD;AAAA;AAAA,MAEA;AAAA;AAAA,QAEE,OAAO;AAAA,MACT;AAAA,MACA;AAAA;AAAA;AAAA,QAGE,OAAO;AAAA,MACT;AAAA,MACA;AAAA;AAAA,QAEE,OAAO;AAAA,MACT;AAAA,MACA;AAAA;AAAA,QAEE,OAAO;AAAA,MACT;AAAA,MACA;AAAA;AAAA,QAEE,OAAO;AAAA,MACT;AAAA,MACA;AAAA;AAAA,QAEE,OAAO;AAAA,MACT;AAAA,IACF;AAAA,EACF;AACF;AAEA,SAAS,eAAe,MAAkC;AACxD,QAAM,QAAQ,KAAK,MAAM,UAAU;AACnC,SAAO,QAAQ,MAAM,CAAC,EAAE,YAAY,IAAI;AAC1C;AAEA,SAAS,YAAY,KAAqB;AACxC,SAAO,IAAI,QAAQ,uBAAuB,MAAM;AAClD;;;AC/KA,OAAO,OAAO;AAEP,IAAM,oBAAN,cAAgC,eAAe;AAAA,EAClC,OAAO,GAAG,mBAAmB;AAAA,EAEvC;AAAA,EAER,YAAY,QAGT;AACD,UAAM;AACN,SAAK,WAAW,OAAO;AAAA,EACzB;AAAA,EAQA,MAAe,MACb,QAKA;AACA,UAAM,WAAW,iBAAiB,OAAO,QAAQ;AACjD,UAAM,eAAe,CAAC;AACtB,UAAM,iBAAiB,EAAE,OAAO;AAAA,MAC9B,SAAS,EACN;AAAA,QACC,EAAE,OAAO;AAAA,UACP,IAAI,EAAE,OAAO,EAAE,SAAS,yBAAyB;AAAA,UACjD,OAAO,EAAE,OAAO,EAAE,SAAS,4BAA4B;AAAA,UACvD,aAAa,EACV,OAAO,EACP,SAAS,kCAAkC;AAAA,QAChD,CAAC;AAAA,MACH,EACC,SAAS,6CAA6C;AAAA,MAEzD,aAAa,EACV,OAAO,EACP;AAAA,QACC,oDAAoD,OAAO,wBAAwB,GAAG;AAAA,MACxF;AAAA,MAEF,GAAI,OAAO,mBAAmB,CAAC;AAAA,IACjC,CAAC;AAED,iBAAa,KAAK,0CAA0C;AAE5D,QAAI,OAAO,cAAc;AACvB,mBAAa,KAAK,OAAO,YAAY;AAAA,IACvC,OAAO;AACL,mBAAa,KAAK,sCAAsC;AAAA,IAC1D;AAEA,iBAAa;AAAA,MACX;AAAA,IACF;AACA,iBAAa;AAAA,MACX;AAAA,QACE,WAAW,OAAO,MAAM;AAAA,QACxB;AAAA,QACA,GAAG,SAAS;AAAA,UACV,CAAC,cACC,KAAK,UAAU,EAAE,KAAK,UAAU,WAAW,aAAa,UAAU,MAAM,YAAY,UAAU,OAAO,OAAO,CAAC,KAAK,UAAU,OAAO,OAAO,CAAC;AAAA,QAC/I;AAAA,MACF,EAAE,KAAK,IAAI;AAAA,IACb;AAEA,UAAM,qBAAqB,eAAe,aAAa;AACvD,iBAAa;AAAA,MACX;AAAA,EAAwD,KAAK,UAAU,oBAAoB,MAAM,CAAC,CAAC;AAAA;AAAA,IACrG;AAEA,UAAM,aAAa,CAAC,WAAW,OAAO,QAAQ,EAAE;AAChD,UAAM,mBAAmB,MAAM,KAAK,SAAS,QAAQ;AAAA,MACnD,UAAU;AAAA,QACR;AAAA,UACE,MAAM;AAAA,UACN,SAAS,aAAa,KAAK,IAAI;AAAA,QACjC;AAAA,QACA;AAAA,UACE,MAAM;AAAA,UACN,SAAS,WAAW,KAAK,IAAI;AAAA,QAC/B;AAAA,MACF;AAAA,MACA,OAAO,OAAO;AAAA,MACd,gBAAgB;AAAA,QACd,MAAM;AAAA,QACN,aAAa;AAAA,UACX,MAAM;AAAA,UACN,QAAQ;AAAA,QACV;AAAA,MACF;AAAA,IACF,CAAC;AAED,UAAM,SAAS,eAAe;AAAA,MAC5B,oBAAoB,iBAAiB,IAAI;AAAA,IAC3C;AAEA,UAAM,EAAE,aAAa,SAAS,GAAG,gBAAgB,IAAI;AAErD,WAAO;AAAA,MACL;AAAA,MACA;AAAA,MACA,OAAO,oBAAoB,SAAS,QAAQ;AAAA,MAC5C;AAAA,MAEA,UAAU,KAAK,SAAS;AAAA,MACxB,iBAAiB,iBAAiB;AAAA,MAClC,kBAAkB,iBAAiB;AAAA,MACnC,WAAW,iBAAiB;AAAA,MAC5B,YAAY,iBAAiB;AAAA,IAC/B;AAAA,EACF;AACF;AAyCA,SAAS,iBACP,UACwB;AACxB,QAAM,MAAM,SAAS,OAAO,CAAC,GAAG,MAAM,KAAK,EAAE,UAAU,IAAI,CAAC,KAAK;AACjE,SAAO,SAAS,IAAI,CAAC,OAAO,EAAE,GAAG,GAAG,SAAS,EAAE,UAAU,KAAK,IAAI,EAAE;AACtE;AAEA,SAAS,oBACP,SACA,UACQ;AACR,MAAI,QAAQ;AACZ,aAAW,MAAM,SAAS;AACxB,UAAM,YAAY,SAAS,KAAK,CAAC,MAAM,EAAE,OAAO,GAAG,EAAE;AACrD,UAAM,MAAM,WAAW,OAAO,OAAO;AACrC,UAAM,MAAM,WAAW,OAAO,OAAO;AACrC,UAAM,SAAS,WAAW,UAAU;AAEpC,UAAM,QAAQ,OAAO,GAAG,KAAK;AAC7B,QAAI,CAAC,OAAO,SAAS,KAAK,EAAG;AAE7B,UAAM,UAAU,KAAK,IAAI,KAAK,KAAK,IAAI,KAAK,KAAK,CAAC;AAClD,UAAM,eAAe,QAAQ,MAAM,KAAK,UAAU,QAAQ,MAAM;AAChE,aAAS,eAAe;AAAA,EAC1B;AACA,SAAO,KAAK,IAAI,GAAG,KAAK,IAAI,GAAG,KAAK,CAAC;AACvC;","names":[]}

package/dist/chunk-ZXTQJFGL.js

@@ -0,0 +1,44 @@
+import {
+  CATEGORIES,
+  PEERBENCH_NAMESPACE
+} from "./chunk-UHHHSYVE.js";
+import {
+  buildSchemaDefiner
+} from "./chunk-OQE6TQXZ.js";
+import {
+  IdSchema
+} from "./chunk-NUEOE3K5.js";
+
+// src/schemas/llm/system-prompt.ts
+import { z } from "zod";
+var BaseSystemPromptSchemaV1 = z.object({
+  id: IdSchema,
+  namespace: z.string(),
+  kind: z.string(),
+  schemaVersion: z.number(),
+  version: z.number(),
+  metadata: z.record(z.string(), z.unknown()).optional()
+});
+var defineSystemPromptSchema = buildSchemaDefiner(
+  BaseSystemPromptSchemaV1,
+  "sys-prompt"
+);
+
+// src/schemas/llm/simple-system-prompt.ts
+import { z as z2 } from "zod";
+var SimpleSystemPromptSchemaV1 = defineSystemPromptSchema({
+  baseSchema: BaseSystemPromptSchemaV1,
+  namespace: PEERBENCH_NAMESPACE,
+  kind: `${CATEGORIES.LLM}/simple`,
+  schemaVersion: 1,
+  fields: {
+    content: z2.string()
+  }
+});
+
+export {
+  BaseSystemPromptSchemaV1,
+  defineSystemPromptSchema,
+  SimpleSystemPromptSchemaV1
+};
+//# sourceMappingURL=chunk-ZXTQJFGL.js.map

package/dist/chunk-ZXTQJFGL.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/schemas/llm/system-prompt.ts","../src/schemas/llm/simple-system-prompt.ts"],"sourcesContent":["import { IdSchema } from \"../id\";\nimport { z } from \"zod\";\nimport { buildSchemaDefiner } from \"../schema-definer\";\n\nexport const BaseSystemPromptSchemaV1 = z.object({\n  id: IdSchema,\n  namespace: z.string(),\n  kind: z.string(),\n  schemaVersion: z.number(),\n  version: z.number(),\n  metadata: z.record(z.string(), z.unknown()).optional(),\n});\nexport type BaseSystemPromptV1 = z.infer<typeof BaseSystemPromptSchemaV1>;\n\nexport const defineSystemPromptSchema = buildSchemaDefiner(\n  BaseSystemPromptSchemaV1,\n  \"sys-prompt\"\n);\n","import { z } from \"zod\";\nimport { defineSystemPromptSchema } from \"./system-prompt\";\nimport { BaseSystemPromptSchemaV1 } from \"./system-prompt\";\nimport { CATEGORIES, PEERBENCH_NAMESPACE } from \"@/constants\";\n\nexport const SimpleSystemPromptSchemaV1 = defineSystemPromptSchema({\n  baseSchema: BaseSystemPromptSchemaV1,\n  namespace: PEERBENCH_NAMESPACE,\n  kind: `${CATEGORIES.LLM}/simple`,\n  schemaVersion: 1,\n  fields: {\n    content: z.string(),\n  },\n});\nexport type SimpleSystemPromptV1 = z.infer<typeof SimpleSystemPromptSchemaV1>;\n"],"mappings":";;;;;;;;;;;;AACA,SAAS,SAAS;AAGX,IAAM,2BAA2B,EAAE,OAAO;AAAA,EAC/C,IAAI;AAAA,EACJ,WAAW,EAAE,OAAO;AAAA,EACpB,MAAM,EAAE,OAAO;AAAA,EACf,eAAe,EAAE,OAAO;AAAA,EACxB,SAAS,EAAE,OAAO;AAAA,EAClB,UAAU,EAAE,OAAO,EAAE,OAAO,GAAG,EAAE,QAAQ,CAAC,EAAE,SAAS;AACvD,CAAC;AAGM,IAAM,2BAA2B;AAAA,EACtC;AAAA,EACA;AACF;;;ACjBA,SAAS,KAAAA,UAAS;AAKX,IAAM,6BAA6B,yBAAyB;AAAA,EACjE,YAAY;AAAA,EACZ,WAAW;AAAA,EACX,MAAM,GAAG,WAAW,GAAG;AAAA,EACvB,eAAe;AAAA,EACf,QAAQ;AAAA,IACN,SAASC,GAAE,OAAO;AAAA,EACpB;AACF,CAAC;","names":["z","z"]}

package/dist/index-BAioQhp2.d.ts

@@ -0,0 +1,27 @@
+import { A as AbstractProvider } from './provider-BDjGp2y-.js';
+import z__default, { z } from 'zod';
+import { A as AbstractScorer } from './abstract-Dec9Sc5O.js';
+
+declare const IdSchema: z__default.ZodString;
+
+type ClassConstructor<T> = new (...args: any[]) => T;
+type AbstractClassConstructor<T> = abstract new (...args: any[]) => T;
+type InferExtension<TExtension extends z.ZodRawShape, TBase = unknown> = z.infer<z.ZodObject<TExtension>> & TBase;
+type WidenLiteralSchema<S> = S extends z.ZodString ? z.ZodString | z.ZodLiteral<string> : S extends z.ZodNumber ? z.ZodNumber | z.ZodLiteral<number> : S extends z.ZodBoolean ? z.ZodBoolean | z.ZodLiteral<boolean> : S extends z.ZodLiteral<infer L> ? L extends string ? z.ZodString | z.ZodLiteral<L> : L extends number ? z.ZodNumber | z.ZodLiteral<L> : L extends boolean ? z.ZodBoolean | z.ZodLiteral<L> : S : S extends z.ZodOptional<infer O> ? z.ZodOptional<WidenLiteralSchema<O>> : S extends z.ZodType ? S : never;
+type WidenZodObject<T extends z.ZodObject> = z.ZodObject<{
+    [K in keyof T["shape"]]: WidenLiteralSchema<T["shape"][K]>;
+}>;
+
+type Id = z__default.infer<typeof IdSchema>;
+type IdGenerator<TInput = unknown> = (input: TInput) => MaybePromise<Id>;
+type MaybePromise<T> = T | Promise<T>;
+declare const ScoringMethod: {
+    readonly ai: "ai";
+    readonly human: "human";
+    readonly algo: "algo";
+};
+type ScoringMethod = (typeof ScoringMethod)[keyof typeof ScoringMethod];
+type ScorerCtor = ClassConstructor<AbstractScorer> | AbstractClassConstructor<AbstractScorer>;
+type ProviderCtor = ClassConstructor<AbstractProvider> | AbstractClassConstructor<AbstractProvider>;
+
+export { type AbstractClassConstructor as A, type ClassConstructor as C, type IdGenerator as I, type MaybePromise as M, type ProviderCtor as P, type ScorerCtor as S, type WidenZodObject as W, type InferExtension as a, type Id as b, ScoringMethod as c, IdSchema as d };

package/dist/index.d.ts

@@ -1,15 +1,9 @@
-import { I as IdGenerator } from './index-WiPjF2AL.js';
-export { a as Id, M as MaybePromise, S as ScoringMethod } from './index-WiPjF2AL.js';
-export { A as AbstractDataLoader, a as AbstractLoader, e as LLMChatRunner, d as LLMChatRunnerParams, L as LoaderResult, R as Runner, b as RunnerParams, c as RunnerResult } from './data-BmN5WjZ4.js';
-export { A as AbstractFileLoader, a as AbstractHttpLoader, C as ClassConstructor } from './utilities-BrRH32rD.js';
-export { a as GenericJSONArrayDataLoader, G as GenericJSONArrayLoaderResult } from './generic-array-DLHWSvf1.js';
+import { I as IdGenerator, P as ProviderCtor, S as ScorerCtor } from './index-BAioQhp2.js';
+export { A as AbstractClassConstructor, C as ClassConstructor, b as Id, a as InferExtension, M as MaybePromise, c as ScoringMethod, W as WidenZodObject } from './index-BAioQhp2.js';
 export { R as RateLimiter, b as RateLimiterCallOptions, a as RateLimiterOptions } from './rate-limiter-CSmVIRsM.js';
-import 'zod';
-import './schemas/index.js';
-import './provider-BDjGp2y-.js';
-import './abstract-Dec9Sc5O.js';
-import './system-prompt--0FdPWqK.js';
-import 'node:fs';
+import z__default from 'zod';
+import { A as AbstractProvider } from './provider-BDjGp2y-.js';
+import { A as AbstractScorer } from './abstract-Dec9Sc5O.js';
 
 declare function sleep(ms: number, signal?: AbortSignal): Promise<void>;
 
@@ -24,7 +18,7 @@ declare function parseResponseAsJSON<T>(response: string): T | undefined;
 /**
  * Converts the given byte array to a string
  */
-declare function bufferToString(buffer: Uint8Array): string;
+declare function bufferToString(buffer: Uint8Array, encoding?: BufferEncoding): string;
 /**
  * Converts the given string to a byte array
  */
@@ -32,6 +26,21 @@ declare function stringToBuffer(str: string): Uint8Array;
 
 declare const idGeneratorUUIDv7: IdGenerator;
 
+type Runner<TTestCase extends z__default.ZodObject, TResponse extends z__default.ZodObject, TScore extends z__default.ZodObject, TProvider extends AbstractProvider, TScorer extends AbstractScorer, TRunConfig extends Record<string, unknown>> = (params: {
+    testCase: z__default.infer<TTestCase>;
+    provider: TProvider;
+    scorer?: TScorer;
+    runConfig: TRunConfig;
+    idGenerators?: {
+        response?: IdGenerator;
+        score?: IdGenerator;
+    };
+}) => Promise<{
+    response: z__default.infer<TResponse>;
+    score?: z__default.infer<TScore>;
+}>;
+type InferRunConfig<TRunConfigSchema extends z__default.ZodRawShape> = z__default.infer<z__default.ZodObject<TRunConfigSchema>>;
+
 declare class PeerbenchError extends Error {
     code: number;
     constructor(message?: string, options?: ErrorOptions & {
@@ -39,19 +48,35 @@ declare class PeerbenchError extends Error {
     });
 }
 
-declare class CatalogItemNotFoundError extends PeerbenchError {
-    readonly itemName: string | {
-        message: string;
-    };
-    constructor(itemName: string | {
-        message: string;
-    });
-}
-declare class CatalogItemHasNoInstantiateMethodError extends PeerbenchError {
-    readonly itemName: string;
-    constructor(itemName: string);
-}
-
 declare function captureStackTrace(error: Error, constructor: Function): void;
 
-export { CatalogItemHasNoInstantiateMethodError, CatalogItemNotFoundError, IdGenerator, PeerbenchError, bufferToString, captureStackTrace, idGeneratorUUIDv7, parseResponseAsJSON, sleep, stringToBuffer };
+declare const PEERBENCH_NAMESPACE: "peerbench.ai";
+declare const CATEGORIES: {
+    LLM: string;
+};
+
+declare function defineRunner<const TProviders extends ProviderCtor[], const TScorers extends ScorerCtor[], const TSchemaSets extends SchemaSetDefinition[], const TRunConfigSchema extends z__default.ZodRawShape = {}>(config: {
+    schemaSets: TSchemaSets;
+    providers: TProviders;
+    scorers: TScorers;
+    runConfigSchema?: TRunConfigSchema;
+    /**
+     * @default true
+     */
+    parseRunConfig?: boolean;
+    defaults?: {
+        scorer?: InstanceType<TScorers[number]>;
+        responseIdGenerator?: IdGenerator;
+        scoreIdGenerator?: IdGenerator;
+    };
+}, fn: Runner<TSchemaSets[number]["testCase"], TSchemaSets[number]["response"], TSchemaSets[number]["score"], InstanceType<TProviders[number]>, InstanceType<TScorers[number]>, InferRunConfig<TRunConfigSchema>>): (params: Parameters<typeof fn>[0]) => Promise<{
+    response: z__default.core.output<TSchemaSets[number]["response"]>;
+    score?: z__default.core.output<TSchemaSets[number]["score"]> | undefined;
+}>;
+type SchemaSetDefinition<TTestCase extends z__default.ZodObject = z__default.ZodObject, TResponse extends z__default.ZodObject = z__default.ZodObject, TScore extends z__default.ZodObject = z__default.ZodObject> = {
+    testCase: TTestCase;
+    response: TResponse;
+    score: TScore;
+};
+
+export { CATEGORIES, IdGenerator, type InferRunConfig, PEERBENCH_NAMESPACE, PeerbenchError, ProviderCtor, type Runner, ScorerCtor, bufferToString, captureStackTrace, defineRunner, idGeneratorUUIDv7, parseResponseAsJSON, sleep, stringToBuffer };