peerbench 0.0.9 → 0.0.11

package/dist/providers/openrouter.d.ts

@@ -1,6 +1,7 @@
-import { AbstractLLMProvider, ChatResponse, LLMProviderForwardArgs } from "./abstract/llm";
+import { AbstractProvider } from "./abstract";
+import { type CallableLLM } from "./callables/llm";
 import { RateLimiter } from "../utils";
-declare const OpenRouterProvider_base: (new () => AbstractLLMProvider & {
+declare const OpenRouterProvider_base: (new () => AbstractProvider & {
     readonly kind: "peerbench.ai/llm/openrouter.ai";
 }) & {
     readonly kind: "peerbench.ai/llm/openrouter.ai";
@@ -16,12 +17,9 @@ export declare class OpenRouterProvider extends OpenRouterProvider_base {
         timeout?: number;
         rateLimiter?: RateLimiter;
     });
-    forward(args: LLMProviderForwardArgs): Promise<ChatResponse>;
-    /**
-     * Updates the cache that holds information about OpenRouter models
-     * including pricing information. It will be valid for 24 hours as
-     * long as the instance of this Provider object is alive.
-     */
+    model(config: {
+        model: string;
+    }): CallableLLM<OpenRouterProvider>;
     private updateModelsCache;
 }
 export {};

package/dist/schemas/index.js

@@ -5,9 +5,9 @@ import {
   defineResponseSchema,
   defineScoreSchema,
   defineTestCaseSchema
-} from "../chunk-YY33MNMV.js";
+} from "../chunk-7KMGLEYP.js";
 import "../chunk-OQE6TQXZ.js";
-import "../chunk-HMQYGCKI.js";
+import "../chunk-ZJWSK4VO.js";
 import {
   IdSchema
 } from "../chunk-NUEOE3K5.js";

package/dist/schemas/llm/index.js

@@ -1,12 +1,41 @@
 import {
-  BaseSystemPromptSchemaV1,
-  SimpleSystemPromptSchemaV1,
-  defineSystemPromptSchema
-} from "../../chunk-Q6GSOHOP.js";
-import "../../chunk-OQE6TQXZ.js";
-import "../../chunk-UHHHSYVE.js";
-import "../../chunk-NUEOE3K5.js";
+  buildSchemaDefiner
+} from "../../chunk-OQE6TQXZ.js";
+import {
+  CATEGORIES,
+  PEERBENCH_NAMESPACE
+} from "../../chunk-UHHHSYVE.js";
+import {
+  IdSchema
+} from "../../chunk-NUEOE3K5.js";
 import "../../chunk-PZ5AY32C.js";
+
+// src/schemas/llm/system-prompt.ts
+import { z } from "zod";
+var BaseSystemPromptSchemaV1 = z.object({
+  id: IdSchema,
+  namespace: z.string(),
+  kind: z.string(),
+  schemaVersion: z.number(),
+  version: z.number(),
+  metadata: z.record(z.string(), z.unknown()).optional()
+});
+var defineSystemPromptSchema = buildSchemaDefiner(
+  BaseSystemPromptSchemaV1,
+  "sys-prompt"
+);
+
+// src/schemas/llm/simple-system-prompt.ts
+import { z as z2 } from "zod";
+var SimpleSystemPromptSchemaV1 = defineSystemPromptSchema({
+  baseSchema: BaseSystemPromptSchemaV1,
+  namespace: PEERBENCH_NAMESPACE,
+  kind: `${CATEGORIES.LLM}/simple`,
+  schemaVersion: 1,
+  fields: {
+    content: z2.string()
+  }
+});
 export {
   BaseSystemPromptSchemaV1,
   SimpleSystemPromptSchemaV1,

package/dist/schemas/llm/index.js.map

@@ -1 +1 @@
-{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}
+{"version":3,"sources":["../../../src/schemas/llm/system-prompt.ts","../../../src/schemas/llm/simple-system-prompt.ts"],"sourcesContent":["import { IdSchema } from \"../id\";\nimport { z } from \"zod\";\nimport { buildSchemaDefiner } from \"../schema-definer\";\n\nexport const BaseSystemPromptSchemaV1 = z.object({\n  id: IdSchema,\n  namespace: z.string(),\n  kind: z.string(),\n  schemaVersion: z.number(),\n  version: z.number(),\n  metadata: z.record(z.string(), z.unknown()).optional(),\n});\nexport type BaseSystemPromptV1 = z.infer<typeof BaseSystemPromptSchemaV1>;\n\nexport const defineSystemPromptSchema = buildSchemaDefiner(\n  BaseSystemPromptSchemaV1,\n  \"sys-prompt\"\n);\n","import { z } from \"zod\";\nimport { defineSystemPromptSchema } from \"./system-prompt\";\nimport { BaseSystemPromptSchemaV1 } from \"./system-prompt\";\nimport { CATEGORIES, PEERBENCH_NAMESPACE } from \"@/constants\";\n\nexport const SimpleSystemPromptSchemaV1 = defineSystemPromptSchema({\n  baseSchema: BaseSystemPromptSchemaV1,\n  namespace: PEERBENCH_NAMESPACE,\n  kind: `${CATEGORIES.LLM}/simple`,\n  schemaVersion: 1,\n  fields: {\n    content: z.string(),\n  },\n});\nexport type SimpleSystemPromptV1 = z.infer<typeof SimpleSystemPromptSchemaV1>;\n"],"mappings":";;;;;;;;;;;;;AACA,SAAS,SAAS;AAGX,IAAM,2BAA2B,EAAE,OAAO;AAAA,EAC/C,IAAI;AAAA,EACJ,WAAW,EAAE,OAAO;AAAA,EACpB,MAAM,EAAE,OAAO;AAAA,EACf,eAAe,EAAE,OAAO;AAAA,EACxB,SAAS,EAAE,OAAO;AAAA,EAClB,UAAU,EAAE,OAAO,EAAE,OAAO,GAAG,EAAE,QAAQ,CAAC,EAAE,SAAS;AACvD,CAAC;AAGM,IAAM,2BAA2B;AAAA,EACtC;AAAA,EACA;AACF;;;ACjBA,SAAS,KAAAA,UAAS;AAKX,IAAM,6BAA6B,yBAAyB;AAAA,EACjE,YAAY;AAAA,EACZ,WAAW;AAAA,EACX,MAAM,GAAG,WAAW,GAAG;AAAA,EACvB,eAAe;AAAA,EACf,QAAQ;AAAA,IACN,SAASC,GAAE,OAAO;AAAA,EACpB;AACF,CAAC;","names":["z","z"]}

package/dist/scorers/abstract.d.ts

@@ -1,7 +1,7 @@
 export declare abstract class AbstractScorer {
     readonly kind: string;
     constructor();
-    abstract score(params: any): Promise<BaseScorerResult | null>;
+    abstract score(params: any): Promise<BaseScorerResult>;
     static withKind<TKind extends string, TThis extends abstract new (...args: any[]) => AbstractScorer>(this: TThis, kind: TKind): (new () => InstanceType<TThis> & {
         readonly kind: TKind;
     }) & {

package/dist/scorers/index.js

@@ -1,12 +1,382 @@
 import {
-  AbstractScorer,
-  LLMAsAJudgeScorer,
-  MCQScorer,
-  RegexScorer
-} from "../chunk-3JHDJEY3.js";
-import "../chunk-UHHHSYVE.js";
-import "../chunk-4UBK6452.js";
+  PEERBENCH_NAMESPACE
+} from "../chunk-UHHHSYVE.js";
+import {
+  parseResponseAsJSON
+} from "../chunk-4UBK6452.js";
 import "../chunk-PZ5AY32C.js";
+
+// src/scorers/abstract.ts
+var AbstractScorer = class {
+  kind;
+  constructor() {
+    this.kind = this.constructor.kind;
+    if (!this.kind)
+      throw new Error(
+        `${this.constructor.name} must define "static readonly kind" as a constant property.`
+      );
+  }
+  static withKind(kind) {
+    const base = this;
+    const derived = class extends base {
+      static kind = kind;
+    };
+    return derived;
+  }
+};
+
+// src/scorers/regex.ts
+var RegexScorer = class extends AbstractScorer.withKind(
+  `${PEERBENCH_NAMESPACE}/regex`
+) {
+  async score(params) {
+    const allGroupNames = /* @__PURE__ */ new Set();
+    for (const pattern of params.patterns) {
+      const regexSource = pattern.regex.source;
+      const namedGroupRegex = /\(\?<(\w+)>/g;
+      while (true) {
+        const match = namedGroupRegex.exec(regexSource);
+        if (match === null) {
+          break;
+        }
+        if (match[1]) {
+          allGroupNames.add(match[1]);
+        }
+      }
+    }
+    const extractedValues = {};
+    for (const groupName of allGroupNames) {
+      extractedValues[groupName] = null;
+    }
+    const matchPreference = params.matchPreference ?? "last";
+    for (const pattern of params.patterns) {
+      const matches = Array.from(params.input.matchAll(pattern.regex));
+      const match = matchPreference === "first" ? matches[0] : matches.at(-1);
+      if (match && match.groups) {
+        let hasExtractedValue = false;
+        for (const [groupName, groupValue] of Object.entries(match.groups)) {
+          if (groupValue !== void 0) {
+            let value = groupValue;
+            if (pattern.transform) {
+              const transformed = pattern.transform(value);
+              if (transformed === void 0) {
+                continue;
+              }
+              value = transformed;
+            }
+            extractedValues[groupName] = value;
+            hasExtractedValue = true;
+          }
+        }
+        if (hasExtractedValue) {
+          break;
+        }
+      } else if (match) {
+        const captureGroupIndex = pattern.captureGroupIndex ?? 1;
+        const extractedValue = match[captureGroupIndex];
+        if (extractedValue !== void 0) {
+          if (pattern.transform) {
+            const transformed = pattern.transform(extractedValue);
+            if (transformed === void 0) {
+              continue;
+            }
+          }
+          break;
+        }
+      }
+    }
+    const allowPartial = params.allowPartialScoring ?? false;
+    let score = 0;
+    if (typeof params.expectedValue === "function") {
+      const validator = params.expectedValue;
+      const extractedEntries = Object.entries(extractedValues).filter(
+        ([, value]) => value !== null
+      );
+      if (extractedEntries.length === 0) {
+        score = 0;
+      } else {
+        if (allowPartial) {
+          const passingCount = extractedEntries.filter(
+            ([groupName, extractedValue]) => validator(groupName, extractedValue)
+          ).length;
+          score = passingCount / extractedEntries.length;
+        } else {
+          const allMatch = extractedEntries.every(
+            ([groupName, extractedValue]) => validator(groupName, extractedValue)
+          );
+          score = allMatch ? 1 : 0;
+        }
+      }
+    } else {
+      const expectedEntries = Object.entries(params.expectedValue);
+      const totalExpected = expectedEntries.length;
+      if (allowPartial) {
+        const matchingCount = expectedEntries.filter(([key, expectedValue]) => {
+          const extractedValue = extractedValues[key];
+          return extractedValue !== null && extractedValue === expectedValue;
+        }).length;
+        score = totalExpected > 0 ? matchingCount / totalExpected : 0;
+      } else {
+        const allMatch = expectedEntries.every(([key, expectedValue]) => {
+          const extractedValue = extractedValues[key];
+          return extractedValue !== null && extractedValue === expectedValue;
+        });
+        score = allMatch ? 1 : 0;
+      }
+    }
+    return {
+      value: score,
+      extractedAnswers: Object.fromEntries(
+        Object.entries(extractedValues).filter(
+          (entry) => entry[1] !== null
+        )
+      )
+    };
+  }
+};
+
+// src/scorers/mcq.ts
+var MCQScorer = class extends AbstractScorer.withKind(
+  `${PEERBENCH_NAMESPACE}/mcq`
+) {
+  regexScorer = new RegexScorer();
+  async score(params) {
+    const { response, choices, correctAnswers } = params;
+    const normalizedCorrectAnswers = correctAnswers.map(
+      (ca) => ca.toUpperCase()
+    );
+    const normalizedResponse = response.trim().toUpperCase();
+    if (normalizedCorrectAnswers.includes(normalizedResponse)) {
+      return {
+        value: 1,
+        extractedAnswers: [normalizedResponse]
+      };
+    }
+    const json = parseResponseAsJSON(response);
+    if (json !== void 0 && typeof json === "object") {
+      const extractedAnswer = json.answer !== void 0 ? getFirstLetter(json.answer) : void 0;
+      if (extractedAnswer !== void 0) {
+        const normalizedExtracted = extractedAnswer.trim().toUpperCase();
+        if (normalizedCorrectAnswers.includes(normalizedExtracted)) {
+          return {
+            value: 1,
+            extractedAnswers: [extractedAnswer]
+          };
+        }
+        return {
+          value: 0,
+          extractedAnswers: json.answer === void 0 ? [] : [extractedAnswer ?? String(json.answer)]
+        };
+      }
+    }
+    const patterns = [];
+    for (const answer of Object.values(params.choices)) {
+      const answerPatterns = this.buildPatternsForAnswer(answer);
+      patterns.push(...answerPatterns);
+    }
+    const validateAnswer = (groupName, extracted) => {
+      const normalizedExtracted = extracted.trim().toUpperCase();
+      if (normalizedCorrectAnswers.includes(normalizedExtracted)) {
+        return true;
+      }
+      const answerOption = Object.entries(choices).find(
+        ([, value]) => value.trim().toUpperCase() === extracted.trim().toUpperCase()
+      );
+      if (answerOption && normalizedCorrectAnswers.includes(answerOption[0].toUpperCase())) {
+        return true;
+      }
+      return false;
+    };
+    const regexParams = {
+      input: response,
+      patterns,
+      expectedValue: validateAnswer,
+      matchPreference: "last"
+    };
+    const result = await this.regexScorer.score(regexParams);
+    return {
+      value: result.value,
+      extractedAnswers: Object.entries(result.extractedAnswers).map(
+        ([, value]) => value
+      )
+    };
+  }
+  buildPatternsForAnswer(answerText) {
+    const escapedAnswer = escapeRegex(answerText);
+    return [
+      {
+        // "<!NO ANSWER!>" - This pattern matches but has no capture group, so it won't extract anything
+        regex: /<!NO ANSWER!>/g
+      },
+      // Specific patterns for the full answer text (checked first)
+      {
+        // "Answer is $\boxed{answer text}$"
+        regex: new RegExp(
+          `[Aa]nswer is \\$\\\\boxed\\{(?<answer>${escapedAnswer})\\}\\$`,
+          "g"
+        ),
+        transform: (value) => value.toUpperCase()
+      },
+      {
+        // "Answer is answer text"
+        regex: new RegExp(`[Aa]nswer is\\s+(?<answer>${escapedAnswer})`, "g"),
+        transform: (value) => value.toUpperCase()
+      },
+      {
+        // "Answer is **answer text**"
+        regex: new RegExp(
+          `[Aa]nswer is\\s+\\**(?<answer>${escapedAnswer})\\**`,
+          "g"
+        ),
+        transform: (value) => value.toUpperCase()
+      },
+      // Generic patterns (checked after specific patterns)
+      {
+        // "Answer is $\boxed{A}$."
+        regex: /[Aa]nswer is \$\\boxed\{(?<answer>[A-Z])\}\$\.?/g
+      },
+      {
+        // "Answer is A" - match single letter only when it's a complete standalone answer.
+        // Pattern matches: "Answer is" + whitespace + single letter + end or punctuation
+        regex: /[Aa]nswer is\s+(?<answer>[A-Z])(?=\s*$|[.,;:!?])/g
+      },
+      {
+        // "Answer is **A**"
+        regex: /[Aa]nswer is\s+\**(?<answer>[A-Z])\**/g
+      },
+      {
+        // "A: answer text"
+        regex: /(?<answer>[A-Z]):.+/g
+      },
+      {
+        // "A) answer text"
+        regex: /(?<answer>[A-Z])\)\s*.+/g
+      },
+      {
+        // "A)"
+        regex: /(?<answer>[A-Z])\)/g
+      }
+    ];
+  }
+};
+function getFirstLetter(text) {
+  const match = text.match(/[A-Za-z]/);
+  return match ? match[0].toUpperCase() : void 0;
+}
+function escapeRegex(str) {
+  return str.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}
+
+// src/scorers/llm-judge.ts
+import z from "zod";
+var LLMAsAJudgeScorer = class extends AbstractScorer.withKind(
+  `${PEERBENCH_NAMESPACE}/llm-as-a-judge`
+) {
+  model;
+  constructor(config) {
+    super();
+    this.model = config.callable;
+  }
+  async score(params) {
+    const criteria = normalizeWeights(params.criteria);
+    const systemPrompt = [];
+    const responseSchema = z.object({
+      results: z.array(
+        z.object({
+          id: z.string().describe("The id of the criterion"),
+          score: z.number().describe("The score of the criterion"),
+          explanation: z.string().describe("The explanation of the criterion")
+        })
+      ).describe("The results of the evaluation per criterion"),
+      explanation: z.string().describe(
+        `The overall explanation for the score (less than ${params.maxExplanationLength ?? 200} characters)`
+      ),
+      ...params.fieldsToExtract ?? {}
+    });
+    if (params.systemPrompt) {
+      systemPrompt.push(params.systemPrompt);
+    } else {
+      systemPrompt.push("You are a strict, fair evaluation judge.");
+      systemPrompt.push("Only use information from the rubric");
+    }
+    systemPrompt.push(
+      "For each criterion return an integer score within the provided scale and a very brief justification (less than 2 sentences)."
+    );
+    systemPrompt.push(
+      [
+        `Rubric: ${params.rubric}`,
+        `Criteria:`,
+        ...criteria.map(
+          (criterion) => `- ${criterion.id}: ${criterion.description} (weight: ${criterion.weight}, scale: ${criterion.scale?.min ?? 0}..${criterion.scale?.max ?? 5})`
+        )
+      ].join("\n")
+    );
+    const responseJSONSchema = responseSchema.toJSONSchema();
+    systemPrompt.push(
+      `Reply back with the following JSON schema (strict):
+${JSON.stringify(responseJSONSchema, null, 2)}
+`
+    );
+    const userPrompt = [`Answer: ${params.response}`];
+    const providerResponse = await this.model.forward({
+      messages: [
+        {
+          role: "system",
+          content: systemPrompt.join("\n")
+        },
+        {
+          role: "user",
+          content: userPrompt.join("\n")
+        }
+      ],
+      responseFormat: {
+        type: "json_schema",
+        json_schema: {
+          name: "judgeResult",
+          schema: responseJSONSchema
+        }
+      }
+    });
+    const parsed = responseSchema.parse(
+      parseResponseAsJSON(providerResponse.data)
+    );
+    const { explanation, results, ...extractedFields } = parsed;
+    return {
+      explanation,
+      results,
+      value: computeOverallScore(results, criteria),
+      extractedFields,
+      provider: this.model.provider.kind,
+      modelSlug: this.model.slug,
+      inputTokensUsed: providerResponse.inputTokensUsed,
+      outputTokensUsed: providerResponse.outputTokensUsed,
+      inputCost: providerResponse.inputCost,
+      outputCost: providerResponse.outputCost,
+      metadata: {
+        judgeResponseMetadata: providerResponse.metadata
+      }
+    };
+  }
+};
+function normalizeWeights(criteria) {
+  const sum = criteria.reduce((a, c) => a + (c.weight ?? 1), 0) || 1;
+  return criteria.map((c) => ({ ...c, weight: (c.weight ?? 1) / sum }));
+}
+function computeOverallScore(results, criteria) {
+  let total = 0;
+  for (const pc of results) {
+    const criterion = criteria.find((c) => c.id === pc.id);
+    const min = criterion?.scale?.min ?? 0;
+    const max = criterion?.scale?.max ?? 5;
+    const weight = criterion?.weight ?? 0;
+    const score = Number(pc.score);
+    if (!Number.isFinite(score)) continue;
+    const clamped = Math.max(min, Math.min(max, score));
+    const normalized01 = max === min ? 0 : (clamped - min) / (max - min);
+    total += normalized01 * weight;
+  }
+  return Math.max(0, Math.min(1, total));
+}
 export {
   AbstractScorer,
   LLMAsAJudgeScorer,

package/dist/scorers/index.js.map

@@ -1 +1 @@
-{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}
+{"version":3,"sources":["../../src/scorers/abstract.ts","../../src/scorers/regex.ts","../../src/scorers/mcq.ts","../../src/scorers/llm-judge.ts"],"sourcesContent":["export abstract class AbstractScorer {\n  readonly kind: string;\n\n  constructor() {\n    this.kind = (this.constructor as any).kind;\n    if (!this.kind)\n      throw new Error(\n        `${this.constructor.name} must define \"static readonly kind\" as a constant property.`\n      );\n  }\n\n  abstract score(params: any): Promise<BaseScorerResult>;\n\n  static withKind<\n    TKind extends string,\n    TThis extends abstract new (...args: any[]) => AbstractScorer,\n  >(this: TThis, kind: TKind) {\n    const base = this as unknown as new (...args: any[]) => any;\n    const derived = class extends base {\n      static readonly kind: TKind = kind;\n      declare readonly kind: TKind;\n    };\n\n    return derived as unknown as (new () => InstanceType<TThis> & {\n      readonly kind: TKind;\n    }) & {\n      readonly kind: TKind;\n    };\n  }\n}\n\nexport type BaseScorerResult = {\n  value: number;\n  explanation?: string;\n  metadata?: Record<string, unknown>;\n  [key: string]: unknown;\n};\n","import { PEERBENCH_NAMESPACE } from \"@/constants\";\nimport { AbstractScorer, BaseScorerResult } from \"./abstract\";\n\nexport type RegexPattern = {\n  /**\n   * The regex pattern to match against the response\n   */\n  regex: RegExp;\n\n  /**\n   * The index of the capture group to extract (1-based, like match[1])\n   * If not provided, defaults to 1 (first capture group)\n   */\n  captureGroupIndex?: number;\n\n  /**\n   * Optional function to transform the extracted value before validation\n   */\n  transform?: (value: string) => string | undefined;\n};\n\nexport type RegexScorerParams = {\n  /**\n   * The input text to score\n   */\n  input: string;\n\n  /**\n   * Array of regex patterns to try (in order, first match wins)\n   */\n  patterns: RegexPattern[];\n\n  /**\n   * Expected value(s) to match against. Can be a record of expected values for named groups, or a validation function\n   */\n  expectedValue:\n    | Record<string, string>\n    | ((groupName: string, match: string) => boolean);\n\n  /**\n   * Optional: Which match to use when multiple matches are found\n   * Defaults to \"last\" (uses the last match found)\n   */\n  matchPreference?: \"first\" | \"last\";\n\n  /**\n   * Optional: If true, allows partial scoring based on how many groups match\n   * For example, if 2 groups are expected and only 1 matches, score would be 0.5\n   * Defaults to false (all-or-nothing scoring)\n   */\n  allowPartialScoring?: boolean;\n};\n\n/**\n * Generic Regex scorer. It scores the given input against a set of regex patterns.\n */\nexport class RegexScorer extends AbstractScorer.withKind(\n  `${PEERBENCH_NAMESPACE}/regex`\n) {\n  override async score(params: RegexScorerParams) {\n    // Collect all named group names from all patterns\n    const allGroupNames = new Set<string>();\n    for (const pattern of params.patterns) {\n      const regexSource = pattern.regex.source;\n      const namedGroupRegex = /\\(\\?<(\\w+)>/g;\n\n      while (true) {\n        const match = namedGroupRegex.exec(regexSource);\n        if (match === null) {\n          break;\n        }\n\n        if (match[1]) {\n          allGroupNames.add(match[1]);\n        }\n      }\n    }\n\n    // Initialize result object with all group names set to null (aka not found yet)\n    const extractedValues: Record<string, string | null> = {};\n    for (const groupName of allGroupNames) {\n      extractedValues[groupName] = null;\n    }\n\n    // Try regex patterns in order, stop at first successful match\n    const matchPreference = params.matchPreference ?? \"last\";\n    for (const pattern of params.patterns) {\n      const matches = Array.from(params.input.matchAll(pattern.regex));\n      const match = matchPreference === \"first\" ? matches[0] : matches.at(-1);\n\n      if (match && match.groups) {\n        // Extract all named groups from this match\n        let hasExtractedValue = false;\n        for (const [groupName, groupValue] of Object.entries(match.groups)) {\n          if (groupValue !== undefined) {\n            let value = groupValue;\n\n            // Apply transformation if provided\n            if (pattern.transform) {\n              const transformed = pattern.transform(value);\n              if (transformed === undefined) {\n                continue;\n              }\n              value = transformed;\n            }\n\n            extractedValues[groupName] = value;\n            hasExtractedValue = true;\n          }\n        }\n        // If we extracted at least one value, stop processing further patterns\n        if (hasExtractedValue) {\n          break;\n        }\n      } else if (match) {\n        // Fallback to captureGroupIndex if no named groups\n        // For unnamed groups, we can't add to extractedValues by name\n        // They are only used for scoring if no named groups are found\n        const captureGroupIndex = pattern.captureGroupIndex ?? 1;\n        const extractedValue = match[captureGroupIndex];\n\n        if (extractedValue !== undefined) {\n          // Apply transformation if provided\n          if (pattern.transform) {\n            const transformed = pattern.transform(extractedValue);\n            if (transformed === undefined) {\n              continue;\n            }\n            // For unnamed groups, we can't store in extractedValues\n            // but we can use it for scoring if no named groups were found\n          }\n          // If we have an extracted value (even if unnamed), stop processing\n          // Note: This is a fallback case, so we break here too\n          break;\n        }\n      }\n    }\n\n    // Calculate score based on matched value\n    const allowPartial = params.allowPartialScoring ?? false;\n    let score = 0;\n\n    if (typeof params.expectedValue === \"function\") {\n      // To get proper type inference, cast it using \"as\"\n      const validator = params.expectedValue as typeof params.expectedValue;\n      const extractedEntries = Object.entries(extractedValues).filter(\n        ([, value]) => value !== null\n      );\n\n      // If no values were extracted, score is 0\n      if (extractedEntries.length === 0) {\n        score = 0;\n      } else {\n        if (allowPartial) {\n          // Count how many extracted values pass validation\n          const passingCount = extractedEntries.filter(\n            ([groupName, extractedValue]) =>\n              validator(groupName, extractedValue as string)\n          ).length;\n          score = passingCount / extractedEntries.length;\n        } else {\n          // All extracted values must pass validation\n          const allMatch = extractedEntries.every(\n            ([groupName, extractedValue]) =>\n              validator(groupName, extractedValue as string)\n          );\n          score = allMatch ? 1 : 0;\n        }\n      }\n    } else {\n      const expectedEntries = Object.entries(params.expectedValue);\n      const totalExpected = expectedEntries.length;\n\n      if (allowPartial) {\n        // Count how many expected values match their extracted values\n        const matchingCount = expectedEntries.filter(([key, expectedValue]) => {\n          const extractedValue = extractedValues[key];\n          return extractedValue !== null && extractedValue === expectedValue;\n        }).length;\n        score = totalExpected > 0 ? matchingCount / totalExpected : 0;\n      } else {\n        // All expected values must match\n        const allMatch = expectedEntries.every(([key, expectedValue]) => {\n          const extractedValue = extractedValues[key];\n          return extractedValue !== null && extractedValue === expectedValue;\n        });\n        score = allMatch ? 1 : 0;\n      }\n    }\n\n    return {\n      value: score,\n      extractedAnswers: Object.fromEntries(\n        Object.entries(extractedValues).filter(\n          (entry): entry is [string, string] => entry[1] !== null\n        )\n      ),\n    } satisfies BaseScorerResult;\n  }\n}\n","import { parseResponseAsJSON } from \"@/utils\";\nimport { AbstractScorer, BaseScorerResult } from \"./abstract\";\nimport { RegexScorer, RegexPattern, RegexScorerParams } from \"./regex\";\nimport { PEERBENCH_NAMESPACE } from \"@/constants\";\n\nexport type MCQScorerParams = {\n  response: string;\n  choices: Record<string, string>;\n  correctAnswers: string[];\n};\n\nexport class MCQScorer extends AbstractScorer.withKind(\n  `${PEERBENCH_NAMESPACE}/mcq`\n) {\n  private regexScorer = new RegexScorer();\n\n  async score(params: MCQScorerParams): Promise<\n    BaseScorerResult & {\n      extractedAnswers: string[];\n    }\n  > {\n    const { response, choices, correctAnswers } = params;\n    const normalizedCorrectAnswers = correctAnswers.map((ca) =>\n      ca.toUpperCase()\n    );\n\n    // Direct answer comparison\n    const normalizedResponse = response.trim().toUpperCase();\n    if (normalizedCorrectAnswers.includes(normalizedResponse)) {\n      return {\n        value: 1,\n        extractedAnswers: [normalizedResponse],\n      };\n    }\n\n    // Try to parse the response as JSON (original behavior: returns early if parsed)\n    const json = parseResponseAsJSON<{ answer: string }>(response);\n    if (json !== undefined && typeof json === \"object\") {\n      const extractedAnswer =\n        json.answer !== undefined ? getFirstLetter(json.answer) : undefined;\n\n      if (extractedAnswer !== undefined) {\n        const normalizedExtracted = extractedAnswer.trim().toUpperCase();\n        if (normalizedCorrectAnswers.includes(normalizedExtracted)) {\n          return {\n            value: 1,\n            extractedAnswers: [extractedAnswer],\n          };\n        }\n\n        // Response parsed as JSON but does not represent the correct answer\n        return {\n          value: 0,\n          extractedAnswers:\n            json.answer === undefined\n              ? []\n              : [extractedAnswer ?? String(json.answer)],\n        };\n      }\n    }\n\n    // Build patterns for all correctAnswers\n    const patterns: RegexPattern[] = [];\n    for (const answer of Object.values(params.choices)) {\n      const answerPatterns = this.buildPatternsForAnswer(answer);\n      patterns.push(...answerPatterns);\n    }\n\n    // Create validation function that handles choices matching\n    // New RegexScorer API expects (groupName: string, match: string) => boolean\n    const validateAnswer = (groupName: string, extracted: string): boolean => {\n      const normalizedExtracted = extracted.trim().toUpperCase();\n\n      // Check if extracted value is in correctAnswers\n      if (normalizedCorrectAnswers.includes(normalizedExtracted)) {\n        return true;\n      }\n\n      // Check if extracted text matches any choice value, and that choice key is correct\n      // Note: When transform uppercases the value, we need to compare case-insensitively\n      const answerOption = Object.entries(choices).find(\n        ([, value]) =>\n          value.trim().toUpperCase() === extracted.trim().toUpperCase()\n      );\n\n      if (\n        answerOption &&\n        normalizedCorrectAnswers.includes(answerOption[0].toUpperCase())\n      ) {\n        return true;\n      }\n\n      return false;\n    };\n\n    // Build regex scorer params\n    const regexParams: RegexScorerParams = {\n      input: response,\n      patterns,\n      expectedValue: validateAnswer,\n      matchPreference: \"last\",\n    };\n\n    // Call regex scorer\n    const result = await this.regexScorer.score(regexParams);\n\n    return {\n      value: result.value,\n      extractedAnswers: Object.entries(result.extractedAnswers).map(\n        ([, value]) => value\n      ),\n    };\n  }\n\n  private buildPatternsForAnswer(answerText: string): RegexPattern[] {\n    const escapedAnswer = escapeRegex(answerText);\n\n    return [\n      {\n        // \"<!NO ANSWER!>\" - This pattern matches but has no capture group, so it won't extract anything\n        regex: /<!NO ANSWER!>/g,\n      },\n      // Specific patterns for the full answer text (checked first)\n      {\n        // \"Answer is $\\boxed{answer text}$\"\n        regex: new RegExp(\n          `[Aa]nswer is \\\\$\\\\\\\\boxed\\\\{(?<answer>${escapedAnswer})\\\\}\\\\$`,\n          \"g\"\n        ),\n        transform: (value: string) => value.toUpperCase(),\n      },\n      {\n        // \"Answer is answer text\"\n        regex: new RegExp(`[Aa]nswer is\\\\s+(?<answer>${escapedAnswer})`, \"g\"),\n        transform: (value: string) => value.toUpperCase(),\n      },\n      {\n        // \"Answer is **answer text**\"\n        regex: new RegExp(\n          `[Aa]nswer is\\\\s+\\\\**(?<answer>${escapedAnswer})\\\\**`,\n          \"g\"\n        ),\n        transform: (value: string) => value.toUpperCase(),\n      },\n      // Generic patterns (checked after specific patterns)\n      {\n        // \"Answer is $\\boxed{A}$.\"\n        regex: /[Aa]nswer is \\$\\\\boxed\\{(?<answer>[A-Z])\\}\\$\\.?/g,\n      },\n      {\n        // \"Answer is A\" - match single letter only when it's a complete standalone answer.\n        // Pattern matches: \"Answer is\" + whitespace + single letter + end or punctuation\n        regex: /[Aa]nswer is\\s+(?<answer>[A-Z])(?=\\s*$|[.,;:!?])/g,\n      },\n      {\n        // \"Answer is **A**\"\n        regex: /[Aa]nswer is\\s+\\**(?<answer>[A-Z])\\**/g,\n      },\n      {\n        // \"A: answer text\"\n        regex: /(?<answer>[A-Z]):.+/g,\n      },\n      {\n        // \"A) answer text\"\n        regex: /(?<answer>[A-Z])\\)\\s*.+/g,\n      },\n      {\n        // \"A)\"\n        regex: /(?<answer>[A-Z])\\)/g,\n      },\n    ];\n  }\n}\n\nfunction getFirstLetter(text: string): string | undefined {\n  const match = text.match(/[A-Za-z]/);\n  return match ? match[0].toUpperCase() : undefined;\n}\n\nfunction escapeRegex(str: string): string {\n  return str.replace(/[.*+?^${}()|[\\]\\\\]/g, \"\\\\$&\");\n}\n","import { CallableLLM } from \"@/providers/callables/llm\";\nimport { parseResponseAsJSON } from \"@/utils\";\nimport { RateLimiter } from \"@/utils/rate-limiter\";\nimport { AbstractScorer, BaseScorerResult } from \"./abstract\";\nimport { PEERBENCH_NAMESPACE } from \"@/constants\";\nimport z from \"zod\";\n\nexport class LLMAsAJudgeScorer extends AbstractScorer.withKind(\n  `${PEERBENCH_NAMESPACE}/llm-as-a-judge`\n) {\n  private model: CallableLLM;\n\n  constructor(config: {\n    callable: CallableLLM;\n    rateLimiter?: RateLimiter;\n  }) {\n    super();\n    this.model = config.callable;\n  }\n\n  override async score<T extends z.ZodRawShape>(\n    params: LLMAsAJudgeScoreParams & { fieldsToExtract: T }\n  ): Promise<ScorerResultWithExtractedFields<T>>;\n  override async score(\n    params: LLMAsAJudgeScoreParams & { fieldsToExtract?: never }\n  ): Promise<ScorerResultWithoutExtractedFields>;\n  override async score<T extends z.ZodRawShape>(\n    params: LLMAsAJudgeScoreParams & { fieldsToExtract?: T }\n  ): Promise<\n    | ScorerResultWithoutExtractedFields\n    | ScorerResultWithExtractedFields<T>\n    | null\n  > {\n    const criteria = normalizeWeights(params.criteria);\n    const systemPrompt = [];\n    const responseSchema = z.object({\n      results: z\n        .array(\n          z.object({\n            id: z.string().describe(\"The id of the criterion\"),\n            score: z.number().describe(\"The score of the criterion\"),\n            explanation: z\n              .string()\n              .describe(\"The explanation of the criterion\"),\n          })\n        )\n        .describe(\"The results of the evaluation per criterion\"),\n\n      explanation: z\n        .string()\n        .describe(\n          `The overall explanation for the score (less than ${params.maxExplanationLength ?? 200} characters)`\n        ),\n\n      ...(params.fieldsToExtract ?? {}),\n    });\n\n    if (params.systemPrompt) {\n      systemPrompt.push(params.systemPrompt);\n    } else {\n      systemPrompt.push(\"You are a strict, fair evaluation judge.\");\n      systemPrompt.push(\"Only use information from the rubric\");\n    }\n\n    systemPrompt.push(\n      \"For each criterion return an integer score within the provided scale and a very brief justification (less than 2 sentences).\"\n    );\n    systemPrompt.push(\n      [\n        `Rubric: ${params.rubric}`,\n        `Criteria:`,\n        ...criteria.map(\n          (criterion) =>\n            `- ${criterion.id}: ${criterion.description} (weight: ${criterion.weight}, scale: ${criterion.scale?.min ?? 0}..${criterion.scale?.max ?? 5})`\n        ),\n      ].join(\"\\n\")\n    );\n\n    const responseJSONSchema = responseSchema.toJSONSchema();\n    systemPrompt.push(\n      `Reply back with the following JSON schema (strict):\\n${JSON.stringify(responseJSONSchema, null, 2)}\\n`\n    );\n\n    const userPrompt = [`Answer: ${params.response}`];\n    const providerResponse = await this.model.forward({\n      messages: [\n        {\n          role: \"system\",\n          content: systemPrompt.join(\"\\n\"),\n        },\n        {\n          role: \"user\",\n          content: userPrompt.join(\"\\n\"),\n        },\n      ],\n      responseFormat: {\n        type: \"json_schema\",\n        json_schema: {\n          name: \"judgeResult\",\n          schema: responseJSONSchema,\n        },\n      },\n    });\n\n    const parsed = responseSchema.parse(\n      parseResponseAsJSON(providerResponse.data)\n    );\n\n    const { explanation, results, ...extractedFields } = parsed;\n\n    return {\n      explanation,\n      results,\n      value: computeOverallScore(results, criteria),\n      extractedFields: extractedFields as z.infer<z.ZodObject<T>>,\n\n      provider: this.model.provider.kind,\n      modelSlug: this.model.slug,\n      inputTokensUsed: providerResponse.inputTokensUsed,\n      outputTokensUsed: providerResponse.outputTokensUsed,\n      inputCost: providerResponse.inputCost,\n      outputCost: providerResponse.outputCost,\n      metadata: {\n        judgeResponseMetadata: providerResponse.metadata,\n      }\n    };\n  }\n}\n\nexport type LLMAsAJudgeCriterion = {\n  id: string;\n  description: string;\n  weight: number;\n  scale?: {\n    min: number;\n    max: number;\n  };\n};\n\nexport type LLMAsAJudgeScoreParams = {\n  response: string;\n  rubric: string;\n  criteria: LLMAsAJudgeCriterion[];\n\n  systemPrompt?: string;\n  maxExplanationLength?: number;\n};\n\ntype ScorerResultWithoutExtractedFields = BaseScorerResult & {\n  results: {\n    id: string;\n    score: number;\n    explanation: string;\n  }[];\n\n  provider: string;\n  modelSlug: string;\n  inputTokensUsed?: number;\n  outputTokensUsed?: number;\n  inputCost?: string;\n  outputCost?: string;\n};\n\ntype ScorerResultWithExtractedFields<T extends z.ZodRawShape> =\n  ScorerResultWithoutExtractedFields & {\n    extractedFields: z.infer<z.ZodObject<T>>;\n  };\n\nfunction normalizeWeights(\n  criteria: LLMAsAJudgeCriterion[]\n): LLMAsAJudgeCriterion[] {\n  const sum = criteria.reduce((a, c) => a + (c.weight ?? 1), 0) || 1;\n  return criteria.map((c) => ({ ...c, weight: (c.weight ?? 1) / sum }));\n}\n\nfunction computeOverallScore(\n  results: ScorerResultWithoutExtractedFields[\"results\"],\n  criteria: LLMAsAJudgeCriterion[]\n): number {\n  let total = 0;\n  for (const pc of results) {\n    const criterion = criteria.find((c) => c.id === pc.id);\n    const min = criterion?.scale?.min ?? 0;\n    const max = criterion?.scale?.max ?? 5;\n    const weight = criterion?.weight ?? 0;\n\n    const score = Number(pc.score);\n    if (!Number.isFinite(score)) continue;\n\n    const clamped = Math.max(min, Math.min(max, score));\n    const normalized01 = max === min ? 0 : (clamped - min) / (max - min);\n    total += normalized01 * weight;\n  }\n  return Math.max(0, Math.min(1, total));\n}\n"],"mappings":";;;;;;;;;AAAO,IAAe,iBAAf,MAA8B;AAAA,EAC1B;AAAA,EAET,cAAc;AACZ,SAAK,OAAQ,KAAK,YAAoB;AACtC,QAAI,CAAC,KAAK;AACR,YAAM,IAAI;AAAA,QACR,GAAG,KAAK,YAAY,IAAI;AAAA,MAC1B;AAAA,EACJ;AAAA,EAIA,OAAO,SAGQ,MAAa;AAC1B,UAAM,OAAO;AACb,UAAM,UAAU,cAAc,KAAK;AAAA,MACjC,OAAgB,OAAc;AAAA,IAEhC;AAEA,WAAO;AAAA,EAKT;AACF;;;AC2BO,IAAM,cAAN,cAA0B,eAAe;AAAA,EAC9C,GAAG,mBAAmB;AACxB,EAAE;AAAA,EACA,MAAe,MAAM,QAA2B;AAE9C,UAAM,gBAAgB,oBAAI,IAAY;AACtC,eAAW,WAAW,OAAO,UAAU;AACrC,YAAM,cAAc,QAAQ,MAAM;AAClC,YAAM,kBAAkB;AAExB,aAAO,MAAM;AACX,cAAM,QAAQ,gBAAgB,KAAK,WAAW;AAC9C,YAAI,UAAU,MAAM;AAClB;AAAA,QACF;AAEA,YAAI,MAAM,CAAC,GAAG;AACZ,wBAAc,IAAI,MAAM,CAAC,CAAC;AAAA,QAC5B;AAAA,MACF;AAAA,IACF;AAGA,UAAM,kBAAiD,CAAC;AACxD,eAAW,aAAa,eAAe;AACrC,sBAAgB,SAAS,IAAI;AAAA,IAC/B;AAGA,UAAM,kBAAkB,OAAO,mBAAmB;AAClD,eAAW,WAAW,OAAO,UAAU;AACrC,YAAM,UAAU,MAAM,KAAK,OAAO,MAAM,SAAS,QAAQ,KAAK,CAAC;AAC/D,YAAM,QAAQ,oBAAoB,UAAU,QAAQ,CAAC,IAAI,QAAQ,GAAG,EAAE;AAEtE,UAAI,SAAS,MAAM,QAAQ;AAEzB,YAAI,oBAAoB;AACxB,mBAAW,CAAC,WAAW,UAAU,KAAK,OAAO,QAAQ,MAAM,MAAM,GAAG;AAClE,cAAI,eAAe,QAAW;AAC5B,gBAAI,QAAQ;AAGZ,gBAAI,QAAQ,WAAW;AACrB,oBAAM,cAAc,QAAQ,UAAU,KAAK;AAC3C,kBAAI,gBAAgB,QAAW;AAC7B;AAAA,cACF;AACA,sBAAQ;AAAA,YACV;AAEA,4BAAgB,SAAS,IAAI;AAC7B,gCAAoB;AAAA,UACtB;AAAA,QACF;AAEA,YAAI,mBAAmB;AACrB;AAAA,QACF;AAAA,MACF,WAAW,OAAO;AAIhB,cAAM,oBAAoB,QAAQ,qBAAqB;AACvD,cAAM,iBAAiB,MAAM,iBAAiB;AAE9C,YAAI,mBAAmB,QAAW;AAEhC,cAAI,QAAQ,WAAW;AACrB,kBAAM,cAAc,QAAQ,UAAU,cAAc;AACpD,gBAAI,gBAAgB,QAAW;AAC7B;AAAA,YACF;AAAA,UAGF;AAGA;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAGA,UAAM,eAAe,OAAO,uBAAuB;AACnD,QAAI,QAAQ;AAEZ,QAAI,OAAO,OAAO,kBAAkB,YAAY;AAE9C,YAAM,YAAY,OAAO;AACzB,YAAM,mBAAmB,OAAO,QAAQ,eAAe,EAAE;AAAA,QACvD,CAAC,CAAC,EAAE,KAAK,MAAM,UAAU;AAAA,MAC3B;AAGA,UAAI,iBAAiB,WAAW,GAAG;AACjC,gBAAQ;AAAA,MACV,OAAO;AACL,YAAI,cAAc;AAEhB,gBAAM,eAAe,iBAAiB;AAAA,YACpC,CAAC,CAAC,WAAW,cAAc,MACzB,UAAU,WAAW,cAAwB;AAAA,UACjD,EAAE;AACF,kBAAQ,eAAe,iBAAiB;AAAA,QAC1C,OAAO;AAEL,gBAAM,WAAW,iBAAiB;AAAA,YAChC,CAAC,CAAC,WAAW,cAAc,MACzB,UAAU,WAAW,cAAwB;AAAA,UACjD;AACA,kBAAQ,WAAW,IAAI;AAAA,QACzB;AAAA,MACF;AAAA,IACF,OAAO;AACL,YAAM,kBAAkB,OAAO,QAAQ,OAAO,aAAa;AAC3D,YAAM,gBAAgB,gBAAgB;AAEtC,UAAI,cAAc;AAEhB,cAAM,gBAAgB,gBAAgB,OAAO,CAAC,CAAC,KAAK,aAAa,MAAM;AACrE,gBAAM,iBAAiB,gBAAgB,GAAG;AAC1C,iBAAO,mBAAmB,QAAQ,mBAAmB;AAAA,QACvD,CAAC,EAAE;AACH,gBAAQ,gBAAgB,IAAI,gBAAgB,gBAAgB;AAAA,MAC9D,OAAO;AAEL,cAAM,WAAW,gBAAgB,MAAM,CAAC,CAAC,KAAK,aAAa,MAAM;AAC/D,gBAAM,iBAAiB,gBAAgB,GAAG;AAC1C,iBAAO,mBAAmB,QAAQ,mBAAmB;AAAA,QACvD,CAAC;AACD,gBAAQ,WAAW,IAAI;AAAA,MACzB;AAAA,IACF;AAEA,WAAO;AAAA,MACL,OAAO;AAAA,MACP,kBAAkB,OAAO;AAAA,QACvB,OAAO,QAAQ,eAAe,EAAE;AAAA,UAC9B,CAAC,UAAqC,MAAM,CAAC,MAAM;AAAA,QACrD;AAAA,MACF;AAAA,IACF;AAAA,EACF;AACF;;;AC5LO,IAAM,YAAN,cAAwB,eAAe;AAAA,EAC5C,GAAG,mBAAmB;AACxB,EAAE;AAAA,EACQ,cAAc,IAAI,YAAY;AAAA,EAEtC,MAAM,MAAM,QAIV;AACA,UAAM,EAAE,UAAU,SAAS,eAAe,IAAI;AAC9C,UAAM,2BAA2B,eAAe;AAAA,MAAI,CAAC,OACnD,GAAG,YAAY;AAAA,IACjB;AAGA,UAAM,qBAAqB,SAAS,KAAK,EAAE,YAAY;AACvD,QAAI,yBAAyB,SAAS,kBAAkB,GAAG;AACzD,aAAO;AAAA,QACL,OAAO;AAAA,QACP,kBAAkB,CAAC,kBAAkB;AAAA,MACvC;AAAA,IACF;AAGA,UAAM,OAAO,oBAAwC,QAAQ;AAC7D,QAAI,SAAS,UAAa,OAAO,SAAS,UAAU;AAClD,YAAM,kBACJ,KAAK,WAAW,SAAY,eAAe,KAAK,MAAM,IAAI;AAE5D,UAAI,oBAAoB,QAAW;AACjC,cAAM,sBAAsB,gBAAgB,KAAK,EAAE,YAAY;AAC/D,YAAI,yBAAyB,SAAS,mBAAmB,GAAG;AAC1D,iBAAO;AAAA,YACL,OAAO;AAAA,YACP,kBAAkB,CAAC,eAAe;AAAA,UACpC;AAAA,QACF;AAGA,eAAO;AAAA,UACL,OAAO;AAAA,UACP,kBACE,KAAK,WAAW,SACZ,CAAC,IACD,CAAC,mBAAmB,OAAO,KAAK,MAAM,CAAC;AAAA,QAC/C;AAAA,MACF;AAAA,IACF;AAGA,UAAM,WAA2B,CAAC;AAClC,eAAW,UAAU,OAAO,OAAO,OAAO,OAAO,GAAG;AAClD,YAAM,iBAAiB,KAAK,uBAAuB,MAAM;AACzD,eAAS,KAAK,GAAG,cAAc;AAAA,IACjC;AAIA,UAAM,iBAAiB,CAAC,WAAmB,cAA+B;AACxE,YAAM,sBAAsB,UAAU,KAAK,EAAE,YAAY;AAGzD,UAAI,yBAAyB,SAAS,mBAAmB,GAAG;AAC1D,eAAO;AAAA,MACT;AAIA,YAAM,eAAe,OAAO,QAAQ,OAAO,EAAE;AAAA,QAC3C,CAAC,CAAC,EAAE,KAAK,MACP,MAAM,KAAK,EAAE,YAAY,MAAM,UAAU,KAAK,EAAE,YAAY;AAAA,MAChE;AAEA,UACE,gBACA,yBAAyB,SAAS,aAAa,CAAC,EAAE,YAAY,CAAC,GAC/D;AACA,eAAO;AAAA,MACT;AAEA,aAAO;AAAA,IACT;AAGA,UAAM,cAAiC;AAAA,MACrC,OAAO;AAAA,MACP;AAAA,MACA,eAAe;AAAA,MACf,iBAAiB;AAAA,IACnB;AAGA,UAAM,SAAS,MAAM,KAAK,YAAY,MAAM,WAAW;AAEvD,WAAO;AAAA,MACL,OAAO,OAAO;AAAA,MACd,kBAAkB,OAAO,QAAQ,OAAO,gBAAgB,EAAE;AAAA,QACxD,CAAC,CAAC,EAAE,KAAK,MAAM;AAAA,MACjB;AAAA,IACF;AAAA,EACF;AAAA,EAEQ,uBAAuB,YAAoC;AACjE,UAAM,gBAAgB,YAAY,UAAU;AAE5C,WAAO;AAAA,MACL;AAAA;AAAA,QAEE,OAAO;AAAA,MACT;AAAA;AAAA,MAEA;AAAA;AAAA,QAEE,OAAO,IAAI;AAAA,UACT,yCAAyC,aAAa;AAAA,UACtD;AAAA,QACF;AAAA,QACA,WAAW,CAAC,UAAkB,MAAM,YAAY;AAAA,MAClD;AAAA,MACA;AAAA;AAAA,QAEE,OAAO,IAAI,OAAO,6BAA6B,aAAa,KAAK,GAAG;AAAA,QACpE,WAAW,CAAC,UAAkB,MAAM,YAAY;AAAA,MAClD;AAAA,MACA;AAAA;AAAA,QAEE,OAAO,IAAI;AAAA,UACT,iCAAiC,aAAa;AAAA,UAC9C;AAAA,QACF;AAAA,QACA,WAAW,CAAC,UAAkB,MAAM,YAAY;AAAA,MAClD;AAAA;AAAA,MAEA;AAAA;AAAA,QAEE,OAAO;AAAA,MACT;AAAA,MACA;AAAA;AAAA;AAAA,QAGE,OAAO;AAAA,MACT;AAAA,MACA;AAAA;AAAA,QAEE,OAAO;AAAA,MACT;AAAA,MACA;AAAA;AAAA,QAEE,OAAO;AAAA,MACT;AAAA,MACA;AAAA;AAAA,QAEE,OAAO;AAAA,MACT;AAAA,MACA;AAAA;AAAA,QAEE,OAAO;AAAA,MACT;AAAA,IACF;AAAA,EACF;AACF;AAEA,SAAS,eAAe,MAAkC;AACxD,QAAM,QAAQ,KAAK,MAAM,UAAU;AACnC,SAAO,QAAQ,MAAM,CAAC,EAAE,YAAY,IAAI;AAC1C;AAEA,SAAS,YAAY,KAAqB;AACxC,SAAO,IAAI,QAAQ,uBAAuB,MAAM;AAClD;;;AChLA,OAAO,OAAO;AAEP,IAAM,oBAAN,cAAgC,eAAe;AAAA,EACpD,GAAG,mBAAmB;AACxB,EAAE;AAAA,EACQ;AAAA,EAER,YAAY,QAGT;AACD,UAAM;AACN,SAAK,QAAQ,OAAO;AAAA,EACtB;AAAA,EAQA,MAAe,MACb,QAKA;AACA,UAAM,WAAW,iBAAiB,OAAO,QAAQ;AACjD,UAAM,eAAe,CAAC;AACtB,UAAM,iBAAiB,EAAE,OAAO;AAAA,MAC9B,SAAS,EACN;AAAA,QACC,EAAE,OAAO;AAAA,UACP,IAAI,EAAE,OAAO,EAAE,SAAS,yBAAyB;AAAA,UACjD,OAAO,EAAE,OAAO,EAAE,SAAS,4BAA4B;AAAA,UACvD,aAAa,EACV,OAAO,EACP,SAAS,kCAAkC;AAAA,QAChD,CAAC;AAAA,MACH,EACC,SAAS,6CAA6C;AAAA,MAEzD,aAAa,EACV,OAAO,EACP;AAAA,QACC,oDAAoD,OAAO,wBAAwB,GAAG;AAAA,MACxF;AAAA,MAEF,GAAI,OAAO,mBAAmB,CAAC;AAAA,IACjC,CAAC;AAED,QAAI,OAAO,cAAc;AACvB,mBAAa,KAAK,OAAO,YAAY;AAAA,IACvC,OAAO;AACL,mBAAa,KAAK,0CAA0C;AAC5D,mBAAa,KAAK,sCAAsC;AAAA,IAC1D;AAEA,iBAAa;AAAA,MACX;AAAA,IACF;AACA,iBAAa;AAAA,MACX;AAAA,QACE,WAAW,OAAO,MAAM;AAAA,QACxB;AAAA,QACA,GAAG,SAAS;AAAA,UACV,CAAC,cACC,KAAK,UAAU,EAAE,KAAK,UAAU,WAAW,aAAa,UAAU,MAAM,YAAY,UAAU,OAAO,OAAO,CAAC,KAAK,UAAU,OAAO,OAAO,CAAC;AAAA,QAC/I;AAAA,MACF,EAAE,KAAK,IAAI;AAAA,IACb;AAEA,UAAM,qBAAqB,eAAe,aAAa;AACvD,iBAAa;AAAA,MACX;AAAA,EAAwD,KAAK,UAAU,oBAAoB,MAAM,CAAC,CAAC;AAAA;AAAA,IACrG;AAEA,UAAM,aAAa,CAAC,WAAW,OAAO,QAAQ,EAAE;AAChD,UAAM,mBAAmB,MAAM,KAAK,MAAM,QAAQ;AAAA,MAChD,UAAU;AAAA,QACR;AAAA,UACE,MAAM;AAAA,UACN,SAAS,aAAa,KAAK,IAAI;AAAA,QACjC;AAAA,QACA;AAAA,UACE,MAAM;AAAA,UACN,SAAS,WAAW,KAAK,IAAI;AAAA,QAC/B;AAAA,MACF;AAAA,MACA,gBAAgB;AAAA,QACd,MAAM;AAAA,QACN,aAAa;AAAA,UACX,MAAM;AAAA,UACN,QAAQ;AAAA,QACV;AAAA,MACF;AAAA,IACF,CAAC;AAED,UAAM,SAAS,eAAe;AAAA,MAC5B,oBAAoB,iBAAiB,IAAI;AAAA,IAC3C;AAEA,UAAM,EAAE,aAAa,SAAS,GAAG,gBAAgB,IAAI;AAErD,WAAO;AAAA,MACL;AAAA,MACA;AAAA,MACA,OAAO,oBAAoB,SAAS,QAAQ;AAAA,MAC5C;AAAA,MAEA,UAAU,KAAK,MAAM,SAAS;AAAA,MAC9B,WAAW,KAAK,MAAM;AAAA,MACtB,iBAAiB,iBAAiB;AAAA,MAClC,kBAAkB,iBAAiB;AAAA,MACnC,WAAW,iBAAiB;AAAA,MAC5B,YAAY,iBAAiB;AAAA,MAC7B,UAAU;AAAA,QACR,uBAAuB,iBAAiB;AAAA,MAC1C;AAAA,IACF;AAAA,EACF;AACF;AAyCA,SAAS,iBACP,UACwB;AACxB,QAAM,MAAM,SAAS,OAAO,CAAC,GAAG,MAAM,KAAK,EAAE,UAAU,IAAI,CAAC,KAAK;AACjE,SAAO,SAAS,IAAI,CAAC,OAAO,EAAE,GAAG,GAAG,SAAS,EAAE,UAAU,KAAK,IAAI,EAAE;AACtE;AAEA,SAAS,oBACP,SACA,UACQ;AACR,MAAI,QAAQ;AACZ,aAAW,MAAM,SAAS;AACxB,UAAM,YAAY,SAAS,KAAK,CAAC,MAAM,EAAE,OAAO,GAAG,EAAE;AACrD,UAAM,MAAM,WAAW,OAAO,OAAO;AACrC,UAAM,MAAM,WAAW,OAAO,OAAO;AACrC,UAAM,SAAS,WAAW,UAAU;AAEpC,UAAM,QAAQ,OAAO,GAAG,KAAK;AAC7B,QAAI,CAAC,OAAO,SAAS,KAAK,EAAG;AAE7B,UAAM,UAAU,KAAK,IAAI,KAAK,KAAK,IAAI,KAAK,KAAK,CAAC;AAClD,UAAM,eAAe,QAAQ,MAAM,KAAK,UAAU,QAAQ,MAAM;AAChE,aAAS,eAAe;AAAA,EAC1B;AACA,SAAO,KAAK,IAAI,GAAG,KAAK,IAAI,GAAG,KAAK,CAAC;AACvC;","names":[]}

package/dist/scorers/llm-judge.d.ts

@@ -1,4 +1,4 @@
-import { AbstractLLMProvider } from "../providers/abstract/llm";
+import { CallableLLM } from "../providers/callables/llm";
 import { RateLimiter } from "../utils/rate-limiter";
 import { AbstractScorer, BaseScorerResult } from "./abstract";
 import z from "zod";
@@ -8,17 +8,17 @@ declare const LLMAsAJudgeScorer_base: (new () => AbstractScorer & {
     readonly kind: "peerbench.ai/llm-as-a-judge";
 };
 export declare class LLMAsAJudgeScorer extends LLMAsAJudgeScorer_base {
-    private provider;
+    private model;
     constructor(config: {
-        provider: AbstractLLMProvider;
+        callable: CallableLLM;
         rateLimiter?: RateLimiter;
     });
     score<T extends z.ZodRawShape>(params: LLMAsAJudgeScoreParams & {
         fieldsToExtract: T;
-    }): Promise<ScorerResultWithExtractedFields<T> | null>;
+    }): Promise<ScorerResultWithExtractedFields<T>>;
     score(params: LLMAsAJudgeScoreParams & {
         fieldsToExtract?: never;
-    }): Promise<ScorerResultWithoutExtractedFields | null>;
+    }): Promise<ScorerResultWithoutExtractedFields>;
 }
 export type LLMAsAJudgeCriterion = {
     id: string;
@@ -30,7 +30,6 @@ export type LLMAsAJudgeCriterion = {
     };
 };
 export type LLMAsAJudgeScoreParams = {
-    model: string;
     response: string;
     rubric: string;
     criteria: LLMAsAJudgeCriterion[];
@@ -44,6 +43,7 @@ type ScorerResultWithoutExtractedFields = BaseScorerResult & {
         explanation: string;
     }[];
     provider: string;
+    modelSlug: string;
     inputTokensUsed?: number;
     outputTokensUsed?: number;
     inputCost?: string;

package/dist/types/index.d.ts

@@ -1,8 +1,5 @@
 export * from "./runner";
-import { AbstractProvider } from "../providers/index.js";
 import { IdSchema } from "../schemas/id";
-import { AbstractScorer } from "../scorers/abstract";
-import { AbstractClassConstructor, ClassConstructor } from "../utilities";
 import z from "zod";
 export type Id = z.infer<typeof IdSchema>;
 export type IdGenerator<TInput = unknown> = (input: TInput) => MaybePromise<Id>;
@@ -13,5 +10,3 @@ export declare const ScoringMethod: {
     readonly algo: "algo";
 };
 export type ScoringMethod = (typeof ScoringMethod)[keyof typeof ScoringMethod];
-export type ScorerCtor = ClassConstructor<AbstractScorer> | AbstractClassConstructor<AbstractScorer>;
-export type ProviderCtor = ClassConstructor<AbstractProvider> | AbstractClassConstructor<AbstractProvider>;

package/dist/types/runner.d.ts

@@ -1,18 +1,14 @@
-import { AbstractProvider } from "../providers/index.js";
+import { Callable } from "../providers/callables/callable";
 import { AbstractScorer } from "../scorers/index.js";
-import { IdGenerator } from ".";
-import z from "zod";
-export type Runner<TTestCase extends z.ZodObject, TResponse extends z.ZodObject, TScore extends z.ZodObject, TProvider extends AbstractProvider, TScorer extends AbstractScorer, TRunConfig extends Record<string, unknown>> = (params: {
-    testCase: z.infer<TTestCase>;
-    provider: TProvider;
-    scorer?: TScorer;
-    runConfig: TRunConfig;
-    idGenerators?: {
-        response?: IdGenerator;
-        score?: IdGenerator;
-    };
-}) => Promise<{
-    response: z.infer<TResponse>;
-    score?: z.infer<TScore>;
-}>;
-export type InferRunConfig<TRunConfigSchema extends z.ZodRawShape> = z.infer<z.ZodObject<TRunConfigSchema>>;
+import { BaseTestCaseV1 } from "../schemas/test-case";
+import { BaseResponseV1 } from "../schemas/response";
+import { BaseScoreV1 } from "../schemas/score";
+export type RunnerParams = {
+    testCase: BaseTestCaseV1;
+    target: Callable;
+    scorer?: AbstractScorer;
+};
+export type RunnerResult = {
+    response: BaseResponseV1;
+    score?: BaseScoreV1;
+};