peerbench 0.0.5 → 0.0.7

package/dist/scorers/index.d.ts

@@ -1,8 +1,8 @@
-import { A as AbstractScorer } from '../abstract-Dec9Sc5O.js';
-export { B as BaseScorerResult } from '../abstract-Dec9Sc5O.js';
-export { b as LLMAsAJudgeCriterion, c as LLMAsAJudgeScoreParams, L as LLMAsAJudgeScorer, a as MCQScorer, M as MCQScorerParams } from '../llm-judge-BS_oNYUK.js';
-import '../llm-BND163ns.js';
-import '../provider-BDjGp2y-.js';
+import { A as AbstractScorer } from '../abstract-BdgLjkNC.js';
+export { B as BaseScorerResult } from '../abstract-BdgLjkNC.js';
+export { b as LLMAsAJudgeCriterion, c as LLMAsAJudgeScoreParams, L as LLMAsAJudgeScorer, a as MCQScorer, M as MCQScorerParams } from '../llm-judge-BuF80-5-.js';
+import '../llm-8ecJmwKJ.js';
+import '../provider-DnEBdl1n.js';
 import 'openai/resources/shared';
 import 'openai/resources/chat/completions';
 import '../rate-limiter-CSmVIRsM.js';
@@ -48,11 +48,15 @@ type RegexScorerParams = {
      */
     allowPartialScoring?: boolean;
 };
+declare const RegexScorer_base: (new () => AbstractScorer & {
+    readonly kind: "peerbench.ai/regex";
+}) & {
+    readonly kind: "peerbench.ai/regex";
+};
 /**
  * Generic Regex scorer. It scores the given input against a set of regex patterns.
  */
-declare class RegexScorer extends AbstractScorer {
-    readonly kind: "peerbench.ai/regex";
+declare class RegexScorer extends RegexScorer_base {
     score(params: RegexScorerParams): Promise<{
         value: number;
         extractedAnswers: {

package/dist/scorers/index.js

@@ -3,7 +3,7 @@ import {
   LLMAsAJudgeScorer,
   MCQScorer,
   RegexScorer
-} from "../chunk-DNGT4SJC.js";
+} from "../chunk-3JHDJEY3.js";
 import "../chunk-UHHHSYVE.js";
 import "../chunk-4UBK6452.js";
 import "../chunk-PZ5AY32C.js";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "peerbench",
-  "version": "0.0.5",
+  "version": "0.0.7",
   "description": "Tooling help work with AI benchmarking data and create trustworthy AI",
   "keywords": [
     "ai",

package/dist/chunk-DNGT4SJC.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../src/scorers/abstract.ts","../src/scorers/regex.ts","../src/scorers/mcq.ts","../src/scorers/llm-judge.ts"],"sourcesContent":["export abstract class AbstractScorer {\n  abstract readonly kind: string;\n\n  abstract score(params: any): Promise<BaseScorerResult | null>;\n}\n\nexport type BaseScorerResult = {\n  value: number;\n  explanation?: string;\n  metadata?: Record<string, unknown>;\n  [key: string]: unknown;\n};\n","import { PEERBENCH_NAMESPACE } from \"@/constants\";\nimport { AbstractScorer, BaseScorerResult } from \"./abstract\";\n\nexport type RegexPattern = {\n  /**\n   * The regex pattern to match against the response\n   */\n  regex: RegExp;\n\n  /**\n   * The index of the capture group to extract (1-based, like match[1])\n   * If not provided, defaults to 1 (first capture group)\n   */\n  captureGroupIndex?: number;\n\n  /**\n   * Optional function to transform the extracted value before validation\n   */\n  transform?: (value: string) => string | undefined;\n};\n\nexport type RegexScorerParams = {\n  /**\n   * The input text to score\n   */\n  input: string;\n\n  /**\n   * Array of regex patterns to try (in order, first match wins)\n   */\n  patterns: RegexPattern[];\n\n  /**\n   * Expected value(s) to match against. Can be a record of expected values for named groups, or a validation function\n   */\n  expectedValue:\n    | Record<string, string>\n    | ((groupName: string, match: string) => boolean);\n\n  /**\n   * Optional: Which match to use when multiple matches are found\n   * Defaults to \"last\" (uses the last match found)\n   */\n  matchPreference?: \"first\" | \"last\";\n\n  /**\n   * Optional: If true, allows partial scoring based on how many groups match\n   * For example, if 2 groups are expected and only 1 matches, score would be 0.5\n   * Defaults to false (all-or-nothing scoring)\n   */\n  allowPartialScoring?: boolean;\n};\n\n/**\n * Generic Regex scorer. It scores the given input against a set of regex patterns.\n */\nexport class RegexScorer extends AbstractScorer {\n  override readonly kind = `${PEERBENCH_NAMESPACE}/regex` as const;\n\n  override async score(params: RegexScorerParams) {\n    // Collect all named group names from all patterns\n    const allGroupNames = new Set<string>();\n    for (const pattern of params.patterns) {\n      const regexSource = pattern.regex.source;\n      const namedGroupRegex = /\\(\\?<(\\w+)>/g;\n\n      while (true) {\n        const match = namedGroupRegex.exec(regexSource);\n        if (match === null) {\n          break;\n        }\n\n        if (match[1]) {\n          allGroupNames.add(match[1]);\n        }\n      }\n    }\n\n    // Initialize result object with all group names set to null (aka not found yet)\n    const extractedValues: Record<string, string | null> = {};\n    for (const groupName of allGroupNames) {\n      extractedValues[groupName] = null;\n    }\n\n    // Try regex patterns in order, stop at first successful match\n    const matchPreference = params.matchPreference ?? \"last\";\n    for (const pattern of params.patterns) {\n      const matches = Array.from(params.input.matchAll(pattern.regex));\n      const match = matchPreference === \"first\" ? matches[0] : matches.at(-1);\n\n      if (match && match.groups) {\n        // Extract all named groups from this match\n        let hasExtractedValue = false;\n        for (const [groupName, groupValue] of Object.entries(match.groups)) {\n          if (groupValue !== undefined) {\n            let value = groupValue;\n\n            // Apply transformation if provided\n            if (pattern.transform) {\n              const transformed = pattern.transform(value);\n              if (transformed === undefined) {\n                continue;\n              }\n              value = transformed;\n            }\n\n            extractedValues[groupName] = value;\n            hasExtractedValue = true;\n          }\n        }\n        // If we extracted at least one value, stop processing further patterns\n        if (hasExtractedValue) {\n          break;\n        }\n      } else if (match) {\n        // Fallback to captureGroupIndex if no named groups\n        // For unnamed groups, we can't add to extractedValues by name\n        // They are only used for scoring if no named groups are found\n        const captureGroupIndex = pattern.captureGroupIndex ?? 1;\n        const extractedValue = match[captureGroupIndex];\n\n        if (extractedValue !== undefined) {\n          // Apply transformation if provided\n          if (pattern.transform) {\n            const transformed = pattern.transform(extractedValue);\n            if (transformed === undefined) {\n              continue;\n            }\n            // For unnamed groups, we can't store in extractedValues\n            // but we can use it for scoring if no named groups were found\n          }\n          // If we have an extracted value (even if unnamed), stop processing\n          // Note: This is a fallback case, so we break here too\n          break;\n        }\n      }\n    }\n\n    // Calculate score based on matched value\n    const allowPartial = params.allowPartialScoring ?? false;\n    let score = 0;\n\n    if (typeof params.expectedValue === \"function\") {\n      // To get proper type inference, cast it using \"as\"\n      const validator = params.expectedValue as typeof params.expectedValue;\n      const extractedEntries = Object.entries(extractedValues).filter(\n        ([, value]) => value !== null\n      );\n\n      // If no values were extracted, score is 0\n      if (extractedEntries.length === 0) {\n        score = 0;\n      } else {\n        if (allowPartial) {\n          // Count how many extracted values pass validation\n          const passingCount = extractedEntries.filter(\n            ([groupName, extractedValue]) =>\n              validator(groupName, extractedValue as string)\n          ).length;\n          score = passingCount / extractedEntries.length;\n        } else {\n          // All extracted values must pass validation\n          const allMatch = extractedEntries.every(\n            ([groupName, extractedValue]) =>\n              validator(groupName, extractedValue as string)\n          );\n          score = allMatch ? 1 : 0;\n        }\n      }\n    } else {\n      const expectedEntries = Object.entries(params.expectedValue);\n      const totalExpected = expectedEntries.length;\n\n      if (allowPartial) {\n        // Count how many expected values match their extracted values\n        const matchingCount = expectedEntries.filter(([key, expectedValue]) => {\n          const extractedValue = extractedValues[key];\n          return extractedValue !== null && extractedValue === expectedValue;\n        }).length;\n        score = totalExpected > 0 ? matchingCount / totalExpected : 0;\n      } else {\n        // All expected values must match\n        const allMatch = expectedEntries.every(([key, expectedValue]) => {\n          const extractedValue = extractedValues[key];\n          return extractedValue !== null && extractedValue === expectedValue;\n        });\n        score = allMatch ? 1 : 0;\n      }\n    }\n\n    return {\n      value: score,\n      extractedAnswers: Object.fromEntries(\n        Object.entries(extractedValues).filter(\n          (entry): entry is [string, string] => entry[1] !== null\n        )\n      ),\n    } satisfies BaseScorerResult;\n  }\n}\n","import { parseResponseAsJSON } from \"@/utils\";\nimport { AbstractScorer, BaseScorerResult } from \"./abstract\";\nimport { RegexScorer, RegexPattern, RegexScorerParams } from \"./regex\";\nimport { PEERBENCH_NAMESPACE } from \"@/constants\";\n\nexport type MCQScorerParams = {\n  response: string;\n  choices: Record<string, string>;\n  correctAnswers: string[];\n};\n\nexport class MCQScorer extends AbstractScorer {\n  override readonly kind = `${PEERBENCH_NAMESPACE}/mcq` as const;\n  private regexScorer = new RegexScorer();\n\n  async score(params: MCQScorerParams): Promise<\n    BaseScorerResult & {\n      extractedAnswers: string[];\n    }\n  > {\n    const { response, choices, correctAnswers } = params;\n    const normalizedCorrectAnswers = correctAnswers.map((ca) =>\n      ca.toUpperCase()\n    );\n\n    // Direct answer comparison\n    const normalizedResponse = response.trim().toUpperCase();\n    if (normalizedCorrectAnswers.includes(normalizedResponse)) {\n      return {\n        value: 1,\n        extractedAnswers: [normalizedResponse],\n      };\n    }\n\n    // Try to parse the response as JSON (original behavior: returns early if parsed)\n    const json = parseResponseAsJSON<{ answer: string }>(response);\n    if (json !== undefined && typeof json === \"object\") {\n      const extractedAnswer =\n        json.answer !== undefined ? getFirstLetter(json.answer) : undefined;\n\n      if (extractedAnswer !== undefined) {\n        const normalizedExtracted = extractedAnswer.trim().toUpperCase();\n        if (normalizedCorrectAnswers.includes(normalizedExtracted)) {\n          return {\n            value: 1,\n            extractedAnswers: [extractedAnswer],\n          };\n        }\n\n        // Response parsed as JSON but does not represent the correct answer\n        return {\n          value: 0,\n          extractedAnswers:\n            json.answer === undefined\n              ? []\n              : [extractedAnswer ?? String(json.answer)],\n        };\n      }\n    }\n\n    // Build patterns for all correctAnswers\n    const patterns: RegexPattern[] = [];\n    for (const answer of Object.values(params.choices)) {\n      const answerPatterns = this.buildPatternsForAnswer(answer);\n      patterns.push(...answerPatterns);\n    }\n\n    // Create validation function that handles choices matching\n    // New RegexScorer API expects (groupName: string, match: string) => boolean\n    const validateAnswer = (groupName: string, extracted: string): boolean => {\n      const normalizedExtracted = extracted.trim().toUpperCase();\n\n      // Check if extracted value is in correctAnswers\n      if (normalizedCorrectAnswers.includes(normalizedExtracted)) {\n        return true;\n      }\n\n      // Check if extracted text matches any choice value, and that choice key is correct\n      // Note: When transform uppercases the value, we need to compare case-insensitively\n      const answerOption = Object.entries(choices).find(\n        ([, value]) =>\n          value.trim().toUpperCase() === extracted.trim().toUpperCase()\n      );\n\n      if (\n        answerOption &&\n        normalizedCorrectAnswers.includes(answerOption[0].toUpperCase())\n      ) {\n        return true;\n      }\n\n      return false;\n    };\n\n    // Build regex scorer params\n    const regexParams: RegexScorerParams = {\n      input: response,\n      patterns,\n      expectedValue: validateAnswer,\n      matchPreference: \"last\",\n    };\n\n    // Call regex scorer\n    const result = await this.regexScorer.score(regexParams);\n\n    return {\n      value: result.value,\n      extractedAnswers: Object.entries(result.extractedAnswers).map(\n        ([, value]) => value\n      ),\n    };\n  }\n\n  private buildPatternsForAnswer(answerText: string): RegexPattern[] {\n    const escapedAnswer = escapeRegex(answerText);\n\n    return [\n      {\n        // \"<!NO ANSWER!>\" - This pattern matches but has no capture group, so it won't extract anything\n        regex: /<!NO ANSWER!>/g,\n      },\n      // Specific patterns for the full answer text (checked first)\n      {\n        // \"Answer is $\\boxed{answer text}$\"\n        regex: new RegExp(\n          `[Aa]nswer is \\\\$\\\\\\\\boxed\\\\{(?<answer>${escapedAnswer})\\\\}\\\\$`,\n          \"g\"\n        ),\n        transform: (value: string) => value.toUpperCase(),\n      },\n      {\n        // \"Answer is answer text\"\n        regex: new RegExp(`[Aa]nswer is\\\\s+(?<answer>${escapedAnswer})`, \"g\"),\n        transform: (value: string) => value.toUpperCase(),\n      },\n      {\n        // \"Answer is **answer text**\"\n        regex: new RegExp(\n          `[Aa]nswer is\\\\s+\\\\**(?<answer>${escapedAnswer})\\\\**`,\n          \"g\"\n        ),\n        transform: (value: string) => value.toUpperCase(),\n      },\n      // Generic patterns (checked after specific patterns)\n      {\n        // \"Answer is $\\boxed{A}$.\"\n        regex: /[Aa]nswer is \\$\\\\boxed\\{(?<answer>[A-Z])\\}\\$\\.?/g,\n      },\n      {\n        // \"Answer is A\" - match single letter only when it's a complete standalone answer.\n        // Pattern matches: \"Answer is\" + whitespace + single letter + end or punctuation\n        regex: /[Aa]nswer is\\s+(?<answer>[A-Z])(?=\\s*$|[.,;:!?])/g,\n      },\n      {\n        // \"Answer is **A**\"\n        regex: /[Aa]nswer is\\s+\\**(?<answer>[A-Z])\\**/g,\n      },\n      {\n        // \"A: answer text\"\n        regex: /(?<answer>[A-Z]):.+/g,\n      },\n      {\n        // \"A) answer text\"\n        regex: /(?<answer>[A-Z])\\)\\s*.+/g,\n      },\n      {\n        // \"A)\"\n        regex: /(?<answer>[A-Z])\\)/g,\n      },\n    ];\n  }\n}\n\nfunction getFirstLetter(text: string): string | undefined {\n  const match = text.match(/[A-Za-z]/);\n  return match ? match[0].toUpperCase() : undefined;\n}\n\nfunction escapeRegex(str: string): string {\n  return str.replace(/[.*+?^${}()|[\\]\\\\]/g, \"\\\\$&\");\n}\n","import { AbstractLLMProvider } from \"@/providers/abstract/llm\";\nimport { parseResponseAsJSON } from \"@/utils\";\nimport { RateLimiter } from \"@/utils/rate-limiter\";\nimport { AbstractScorer, BaseScorerResult } from \"./abstract\";\nimport { PEERBENCH_NAMESPACE } from \"@/constants\";\nimport z from \"zod\";\n\nexport class LLMAsAJudgeScorer extends AbstractScorer {\n  override readonly kind = `${PEERBENCH_NAMESPACE}/llm-as-a-judge` as const;\n\n  private provider: AbstractLLMProvider;\n\n  constructor(config: {\n    provider: AbstractLLMProvider;\n    rateLimiter?: RateLimiter;\n  }) {\n    super();\n    this.provider = config.provider;\n  }\n\n  override async score<T extends z.ZodRawShape>(\n    params: LLMAsAJudgeScoreParams & { fieldsToExtract: T }\n  ): Promise<ScorerResultWithExtractedFields<T> | null>;\n  override async score(\n    params: LLMAsAJudgeScoreParams & { fieldsToExtract?: never }\n  ): Promise<ScorerResultWithoutExtractedFields | null>;\n  override async score<T extends z.ZodRawShape>(\n    params: LLMAsAJudgeScoreParams & { fieldsToExtract?: T }\n  ): Promise<\n    | ScorerResultWithoutExtractedFields\n    | ScorerResultWithExtractedFields<T>\n    | null\n  > {\n    const criteria = normalizeWeights(params.criteria);\n    const systemPrompt = [];\n    const responseSchema = z.object({\n      results: z\n        .array(\n          z.object({\n            id: z.string().describe(\"The id of the criterion\"),\n            score: z.number().describe(\"The score of the criterion\"),\n            explanation: z\n              .string()\n              .describe(\"The explanation of the criterion\"),\n          })\n        )\n        .describe(\"The results of the evaluation per criterion\"),\n\n      explanation: z\n        .string()\n        .describe(\n          `The overall explanation for the score (less than ${params.maxExplanationLength ?? 200} characters)`\n        ),\n\n      ...(params.fieldsToExtract ?? {}),\n    });\n\n    if (params.systemPrompt) {\n      systemPrompt.push(params.systemPrompt);\n    } else {\n      systemPrompt.push(\"You are a strict, fair evaluation judge.\");\n      systemPrompt.push(\"Only use information from the rubric\");\n    }\n\n    systemPrompt.push(\n      \"For each criterion return an integer score within the provided scale and a very brief justification (less than 2 sentences).\"\n    );\n    systemPrompt.push(\n      [\n        `Rubric: ${params.rubric}`,\n        `Criteria:`,\n        ...criteria.map(\n          (criterion) =>\n            `- ${criterion.id}: ${criterion.description} (weight: ${criterion.weight}, scale: ${criterion.scale?.min ?? 0}..${criterion.scale?.max ?? 5})`\n        ),\n      ].join(\"\\n\")\n    );\n\n    const responseJSONSchema = responseSchema.toJSONSchema();\n    systemPrompt.push(\n      `Reply back with the following JSON schema (strict):\\n${JSON.stringify(responseJSONSchema, null, 2)}\\n`\n    );\n\n    const userPrompt = [`Answer: ${params.response}`];\n    const providerResponse = await this.provider.forward({\n      messages: [\n        {\n          role: \"system\",\n          content: systemPrompt.join(\"\\n\"),\n        },\n        {\n          role: \"user\",\n          content: userPrompt.join(\"\\n\"),\n        },\n      ],\n      model: params.model,\n      responseFormat: {\n        type: \"json_schema\",\n        json_schema: {\n          name: \"judgeResult\",\n          schema: responseJSONSchema,\n        },\n      },\n    });\n\n    const parsed = responseSchema.parse(\n      parseResponseAsJSON(providerResponse.data)\n    );\n\n    const { explanation, results, ...extractedFields } = parsed;\n\n    return {\n      explanation,\n      results,\n      value: computeOverallScore(results, criteria),\n      extractedFields: extractedFields as z.infer<z.ZodObject<T>>,\n\n      provider: this.provider.kind,\n      inputTokensUsed: providerResponse.inputTokensUsed,\n      outputTokensUsed: providerResponse.outputTokensUsed,\n      inputCost: providerResponse.inputCost,\n      outputCost: providerResponse.outputCost,\n    };\n  }\n}\n\nexport type LLMAsAJudgeCriterion = {\n  id: string;\n  description: string;\n  weight: number;\n  scale?: {\n    min: number;\n    max: number;\n  };\n};\n\nexport type LLMAsAJudgeScoreParams = {\n  model: string;\n  response: string;\n  rubric: string;\n  criteria: LLMAsAJudgeCriterion[];\n\n  systemPrompt?: string;\n  maxExplanationLength?: number;\n};\n\ntype ScorerResultWithoutExtractedFields = BaseScorerResult & {\n  results: {\n    id: string;\n    score: number;\n    explanation: string;\n  }[];\n\n  provider: string;\n  inputTokensUsed?: number;\n  outputTokensUsed?: number;\n  inputCost?: string;\n  outputCost?: string;\n};\n\ntype ScorerResultWithExtractedFields<T extends z.ZodRawShape> =\n  ScorerResultWithoutExtractedFields & {\n    extractedFields: z.infer<z.ZodObject<T>>;\n  };\n\nfunction normalizeWeights(\n  criteria: LLMAsAJudgeCriterion[]\n): LLMAsAJudgeCriterion[] {\n  const sum = criteria.reduce((a, c) => a + (c.weight ?? 1), 0) || 1;\n  return criteria.map((c) => ({ ...c, weight: (c.weight ?? 1) / sum }));\n}\n\nfunction computeOverallScore(\n  results: ScorerResultWithoutExtractedFields[\"results\"],\n  criteria: LLMAsAJudgeCriterion[]\n): number {\n  let total = 0;\n  for (const pc of results) {\n    const criterion = criteria.find((c) => c.id === pc.id);\n    const min = criterion?.scale?.min ?? 0;\n    const max = criterion?.scale?.max ?? 5;\n    const weight = criterion?.weight ?? 0;\n\n    const score = Number(pc.score);\n    if (!Number.isFinite(score)) continue;\n\n    const clamped = Math.max(min, Math.min(max, score));\n    const normalized01 = max === min ? 0 : (clamped - min) / (max - min);\n    total += normalized01 * weight;\n  }\n  return Math.max(0, Math.min(1, total));\n}\n"],"mappings":";;;;;;;;AAAO,IAAe,iBAAf,MAA8B;AAIrC;;;ACoDO,IAAM,cAAN,cAA0B,eAAe;AAAA,EAC5B,OAAO,GAAG,mBAAmB;AAAA,EAE/C,MAAe,MAAM,QAA2B;AAE9C,UAAM,gBAAgB,oBAAI,IAAY;AACtC,eAAW,WAAW,OAAO,UAAU;AACrC,YAAM,cAAc,QAAQ,MAAM;AAClC,YAAM,kBAAkB;AAExB,aAAO,MAAM;AACX,cAAM,QAAQ,gBAAgB,KAAK,WAAW;AAC9C,YAAI,UAAU,MAAM;AAClB;AAAA,QACF;AAEA,YAAI,MAAM,CAAC,GAAG;AACZ,wBAAc,IAAI,MAAM,CAAC,CAAC;AAAA,QAC5B;AAAA,MACF;AAAA,IACF;AAGA,UAAM,kBAAiD,CAAC;AACxD,eAAW,aAAa,eAAe;AACrC,sBAAgB,SAAS,IAAI;AAAA,IAC/B;AAGA,UAAM,kBAAkB,OAAO,mBAAmB;AAClD,eAAW,WAAW,OAAO,UAAU;AACrC,YAAM,UAAU,MAAM,KAAK,OAAO,MAAM,SAAS,QAAQ,KAAK,CAAC;AAC/D,YAAM,QAAQ,oBAAoB,UAAU,QAAQ,CAAC,IAAI,QAAQ,GAAG,EAAE;AAEtE,UAAI,SAAS,MAAM,QAAQ;AAEzB,YAAI,oBAAoB;AACxB,mBAAW,CAAC,WAAW,UAAU,KAAK,OAAO,QAAQ,MAAM,MAAM,GAAG;AAClE,cAAI,eAAe,QAAW;AAC5B,gBAAI,QAAQ;AAGZ,gBAAI,QAAQ,WAAW;AACrB,oBAAM,cAAc,QAAQ,UAAU,KAAK;AAC3C,kBAAI,gBAAgB,QAAW;AAC7B;AAAA,cACF;AACA,sBAAQ;AAAA,YACV;AAEA,4BAAgB,SAAS,IAAI;AAC7B,gCAAoB;AAAA,UACtB;AAAA,QACF;AAEA,YAAI,mBAAmB;AACrB;AAAA,QACF;AAAA,MACF,WAAW,OAAO;AAIhB,cAAM,oBAAoB,QAAQ,qBAAqB;AACvD,cAAM,iBAAiB,MAAM,iBAAiB;AAE9C,YAAI,mBAAmB,QAAW;AAEhC,cAAI,QAAQ,WAAW;AACrB,kBAAM,cAAc,QAAQ,UAAU,cAAc;AACpD,gBAAI,gBAAgB,QAAW;AAC7B;AAAA,YACF;AAAA,UAGF;AAGA;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAGA,UAAM,eAAe,OAAO,uBAAuB;AACnD,QAAI,QAAQ;AAEZ,QAAI,OAAO,OAAO,kBAAkB,YAAY;AAE9C,YAAM,YAAY,OAAO;AACzB,YAAM,mBAAmB,OAAO,QAAQ,eAAe,EAAE;AAAA,QACvD,CAAC,CAAC,EAAE,KAAK,MAAM,UAAU;AAAA,MAC3B;AAGA,UAAI,iBAAiB,WAAW,GAAG;AACjC,gBAAQ;AAAA,MACV,OAAO;AACL,YAAI,cAAc;AAEhB,gBAAM,eAAe,iBAAiB;AAAA,YACpC,CAAC,CAAC,WAAW,cAAc,MACzB,UAAU,WAAW,cAAwB;AAAA,UACjD,EAAE;AACF,kBAAQ,eAAe,iBAAiB;AAAA,QAC1C,OAAO;AAEL,gBAAM,WAAW,iBAAiB;AAAA,YAChC,CAAC,CAAC,WAAW,cAAc,MACzB,UAAU,WAAW,cAAwB;AAAA,UACjD;AACA,kBAAQ,WAAW,IAAI;AAAA,QACzB;AAAA,MACF;AAAA,IACF,OAAO;AACL,YAAM,kBAAkB,OAAO,QAAQ,OAAO,aAAa;AAC3D,YAAM,gBAAgB,gBAAgB;AAEtC,UAAI,cAAc;AAEhB,cAAM,gBAAgB,gBAAgB,OAAO,CAAC,CAAC,KAAK,aAAa,MAAM;AACrE,gBAAM,iBAAiB,gBAAgB,GAAG;AAC1C,iBAAO,mBAAmB,QAAQ,mBAAmB;AAAA,QACvD,CAAC,EAAE;AACH,gBAAQ,gBAAgB,IAAI,gBAAgB,gBAAgB;AAAA,MAC9D,OAAO;AAEL,cAAM,WAAW,gBAAgB,MAAM,CAAC,CAAC,KAAK,aAAa,MAAM;AAC/D,gBAAM,iBAAiB,gBAAgB,GAAG;AAC1C,iBAAO,mBAAmB,QAAQ,mBAAmB;AAAA,QACvD,CAAC;AACD,gBAAQ,WAAW,IAAI;AAAA,MACzB;AAAA,IACF;AAEA,WAAO;AAAA,MACL,OAAO;AAAA,MACP,kBAAkB,OAAO;AAAA,QACvB,OAAO,QAAQ,eAAe,EAAE;AAAA,UAC9B,CAAC,UAAqC,MAAM,CAAC,MAAM;AAAA,QACrD;AAAA,MACF;AAAA,IACF;AAAA,EACF;AACF;;;AC5LO,IAAM,YAAN,cAAwB,eAAe;AAAA,EAC1B,OAAO,GAAG,mBAAmB;AAAA,EACvC,cAAc,IAAI,YAAY;AAAA,EAEtC,MAAM,MAAM,QAIV;AACA,UAAM,EAAE,UAAU,SAAS,eAAe,IAAI;AAC9C,UAAM,2BAA2B,eAAe;AAAA,MAAI,CAAC,OACnD,GAAG,YAAY;AAAA,IACjB;AAGA,UAAM,qBAAqB,SAAS,KAAK,EAAE,YAAY;AACvD,QAAI,yBAAyB,SAAS,kBAAkB,GAAG;AACzD,aAAO;AAAA,QACL,OAAO;AAAA,QACP,kBAAkB,CAAC,kBAAkB;AAAA,MACvC;AAAA,IACF;AAGA,UAAM,OAAO,oBAAwC,QAAQ;AAC7D,QAAI,SAAS,UAAa,OAAO,SAAS,UAAU;AAClD,YAAM,kBACJ,KAAK,WAAW,SAAY,eAAe,KAAK,MAAM,IAAI;AAE5D,UAAI,oBAAoB,QAAW;AACjC,cAAM,sBAAsB,gBAAgB,KAAK,EAAE,YAAY;AAC/D,YAAI,yBAAyB,SAAS,mBAAmB,GAAG;AAC1D,iBAAO;AAAA,YACL,OAAO;AAAA,YACP,kBAAkB,CAAC,eAAe;AAAA,UACpC;AAAA,QACF;AAGA,eAAO;AAAA,UACL,OAAO;AAAA,UACP,kBACE,KAAK,WAAW,SACZ,CAAC,IACD,CAAC,mBAAmB,OAAO,KAAK,MAAM,CAAC;AAAA,QAC/C;AAAA,MACF;AAAA,IACF;AAGA,UAAM,WAA2B,CAAC;AAClC,eAAW,UAAU,OAAO,OAAO,OAAO,OAAO,GAAG;AAClD,YAAM,iBAAiB,KAAK,uBAAuB,MAAM;AACzD,eAAS,KAAK,GAAG,cAAc;AAAA,IACjC;AAIA,UAAM,iBAAiB,CAAC,WAAmB,cAA+B;AACxE,YAAM,sBAAsB,UAAU,KAAK,EAAE,YAAY;AAGzD,UAAI,yBAAyB,SAAS,mBAAmB,GAAG;AAC1D,eAAO;AAAA,MACT;AAIA,YAAM,eAAe,OAAO,QAAQ,OAAO,EAAE;AAAA,QAC3C,CAAC,CAAC,EAAE,KAAK,MACP,MAAM,KAAK,EAAE,YAAY,MAAM,UAAU,KAAK,EAAE,YAAY;AAAA,MAChE;AAEA,UACE,gBACA,yBAAyB,SAAS,aAAa,CAAC,EAAE,YAAY,CAAC,GAC/D;AACA,eAAO;AAAA,MACT;AAEA,aAAO;AAAA,IACT;AAGA,UAAM,cAAiC;AAAA,MACrC,OAAO;AAAA,MACP;AAAA,MACA,eAAe;AAAA,MACf,iBAAiB;AAAA,IACnB;AAGA,UAAM,SAAS,MAAM,KAAK,YAAY,MAAM,WAAW;AAEvD,WAAO;AAAA,MACL,OAAO,OAAO;AAAA,MACd,kBAAkB,OAAO,QAAQ,OAAO,gBAAgB,EAAE;AAAA,QACxD,CAAC,CAAC,EAAE,KAAK,MAAM;AAAA,MACjB;AAAA,IACF;AAAA,EACF;AAAA,EAEQ,uBAAuB,YAAoC;AACjE,UAAM,gBAAgB,YAAY,UAAU;AAE5C,WAAO;AAAA,MACL;AAAA;AAAA,QAEE,OAAO;AAAA,MACT;AAAA;AAAA,MAEA;AAAA;AAAA,QAEE,OAAO,IAAI;AAAA,UACT,yCAAyC,aAAa;AAAA,UACtD;AAAA,QACF;AAAA,QACA,WAAW,CAAC,UAAkB,MAAM,YAAY;AAAA,MAClD;AAAA,MACA;AAAA;AAAA,QAEE,OAAO,IAAI,OAAO,6BAA6B,aAAa,KAAK,GAAG;AAAA,QACpE,WAAW,CAAC,UAAkB,MAAM,YAAY;AAAA,MAClD;AAAA,MACA;AAAA;AAAA,QAEE,OAAO,IAAI;AAAA,UACT,iCAAiC,aAAa;AAAA,UAC9C;AAAA,QACF;AAAA,QACA,WAAW,CAAC,UAAkB,MAAM,YAAY;AAAA,MAClD;AAAA;AAAA,MAEA;AAAA;AAAA,QAEE,OAAO;AAAA,MACT;AAAA,MACA;AAAA;AAAA;AAAA,QAGE,OAAO;AAAA,MACT;AAAA,MACA;AAAA;AAAA,QAEE,OAAO;AAAA,MACT;AAAA,MACA;AAAA;AAAA,QAEE,OAAO;AAAA,MACT;AAAA,MACA;AAAA;AAAA,QAEE,OAAO;AAAA,MACT;AAAA,MACA;AAAA;AAAA,QAEE,OAAO;AAAA,MACT;AAAA,IACF;AAAA,EACF;AACF;AAEA,SAAS,eAAe,MAAkC;AACxD,QAAM,QAAQ,KAAK,MAAM,UAAU;AACnC,SAAO,QAAQ,MAAM,CAAC,EAAE,YAAY,IAAI;AAC1C;AAEA,SAAS,YAAY,KAAqB;AACxC,SAAO,IAAI,QAAQ,uBAAuB,MAAM;AAClD;;;AC/KA,OAAO,OAAO;AAEP,IAAM,oBAAN,cAAgC,eAAe;AAAA,EAClC,OAAO,GAAG,mBAAmB;AAAA,EAEvC;AAAA,EAER,YAAY,QAGT;AACD,UAAM;AACN,SAAK,WAAW,OAAO;AAAA,EACzB;AAAA,EAQA,MAAe,MACb,QAKA;AACA,UAAM,WAAW,iBAAiB,OAAO,QAAQ;AACjD,UAAM,eAAe,CAAC;AACtB,UAAM,iBAAiB,EAAE,OAAO;AAAA,MAC9B,SAAS,EACN;AAAA,QACC,EAAE,OAAO;AAAA,UACP,IAAI,EAAE,OAAO,EAAE,SAAS,yBAAyB;AAAA,UACjD,OAAO,EAAE,OAAO,EAAE,SAAS,4BAA4B;AAAA,UACvD,aAAa,EACV,OAAO,EACP,SAAS,kCAAkC;AAAA,QAChD,CAAC;AAAA,MACH,EACC,SAAS,6CAA6C;AAAA,MAEzD,aAAa,EACV,OAAO,EACP;AAAA,QACC,oDAAoD,OAAO,wBAAwB,GAAG;AAAA,MACxF;AAAA,MAEF,GAAI,OAAO,mBAAmB,CAAC;AAAA,IACjC,CAAC;AAED,QAAI,OAAO,cAAc;AACvB,mBAAa,KAAK,OAAO,YAAY;AAAA,IACvC,OAAO;AACL,mBAAa,KAAK,0CAA0C;AAC5D,mBAAa,KAAK,sCAAsC;AAAA,IAC1D;AAEA,iBAAa;AAAA,MACX;AAAA,IACF;AACA,iBAAa;AAAA,MACX;AAAA,QACE,WAAW,OAAO,MAAM;AAAA,QACxB;AAAA,QACA,GAAG,SAAS;AAAA,UACV,CAAC,cACC,KAAK,UAAU,EAAE,KAAK,UAAU,WAAW,aAAa,UAAU,MAAM,YAAY,UAAU,OAAO,OAAO,CAAC,KAAK,UAAU,OAAO,OAAO,CAAC;AAAA,QAC/I;AAAA,MACF,EAAE,KAAK,IAAI;AAAA,IACb;AAEA,UAAM,qBAAqB,eAAe,aAAa;AACvD,iBAAa;AAAA,MACX;AAAA,EAAwD,KAAK,UAAU,oBAAoB,MAAM,CAAC,CAAC;AAAA;AAAA,IACrG;AAEA,UAAM,aAAa,CAAC,WAAW,OAAO,QAAQ,EAAE;AAChD,UAAM,mBAAmB,MAAM,KAAK,SAAS,QAAQ;AAAA,MACnD,UAAU;AAAA,QACR;AAAA,UACE,MAAM;AAAA,UACN,SAAS,aAAa,KAAK,IAAI;AAAA,QACjC;AAAA,QACA;AAAA,UACE,MAAM;AAAA,UACN,SAAS,WAAW,KAAK,IAAI;AAAA,QAC/B;AAAA,MACF;AAAA,MACA,OAAO,OAAO;AAAA,MACd,gBAAgB;AAAA,QACd,MAAM;AAAA,QACN,aAAa;AAAA,UACX,MAAM;AAAA,UACN,QAAQ;AAAA,QACV;AAAA,MACF;AAAA,IACF,CAAC;AAED,UAAM,SAAS,eAAe;AAAA,MAC5B,oBAAoB,iBAAiB,IAAI;AAAA,IAC3C;AAEA,UAAM,EAAE,aAAa,SAAS,GAAG,gBAAgB,IAAI;AAErD,WAAO;AAAA,MACL;AAAA,MACA;AAAA,MACA,OAAO,oBAAoB,SAAS,QAAQ;AAAA,MAC5C;AAAA,MAEA,UAAU,KAAK,SAAS;AAAA,MACxB,iBAAiB,iBAAiB;AAAA,MAClC,kBAAkB,iBAAiB;AAAA,MACnC,WAAW,iBAAiB;AAAA,MAC5B,YAAY,iBAAiB;AAAA,IAC/B;AAAA,EACF;AACF;AAyCA,SAAS,iBACP,UACwB;AACxB,QAAM,MAAM,SAAS,OAAO,CAAC,GAAG,MAAM,KAAK,EAAE,UAAU,IAAI,CAAC,KAAK;AACjE,SAAO,SAAS,IAAI,CAAC,OAAO,EAAE,GAAG,GAAG,SAAS,EAAE,UAAU,KAAK,IAAI,EAAE;AACtE;AAEA,SAAS,oBACP,SACA,UACQ;AACR,MAAI,QAAQ;AACZ,aAAW,MAAM,SAAS;AACxB,UAAM,YAAY,SAAS,KAAK,CAAC,MAAM,EAAE,OAAO,GAAG,EAAE;AACrD,UAAM,MAAM,WAAW,OAAO,OAAO;AACrC,UAAM,MAAM,WAAW,OAAO,OAAO;AACrC,UAAM,SAAS,WAAW,UAAU;AAEpC,UAAM,QAAQ,OAAO,GAAG,KAAK;AAC7B,QAAI,CAAC,OAAO,SAAS,KAAK,EAAG;AAE7B,UAAM,UAAU,KAAK,IAAI,KAAK,KAAK,IAAI,KAAK,KAAK,CAAC;AAClD,UAAM,eAAe,QAAQ,MAAM,KAAK,UAAU,QAAQ,MAAM;AAChE,aAAS,eAAe;AAAA,EAC1B;AACA,SAAO,KAAK,IAAI,GAAG,KAAK,IAAI,GAAG,KAAK,CAAC;AACvC;","names":[]}

package/dist/chunk-HPPCDSJ3.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../src/providers/abstract/provider.ts","../src/providers/abstract/llm.ts","../src/providers/mastra.ts","../src/providers/openai.ts","../src/providers/openrouter.ts"],"sourcesContent":["export abstract class AbstractProvider {\n  abstract readonly kind: string;\n}\n\nexport type ProviderResponse<TData = unknown> = {\n  startedAt: number;\n  completedAt: number;\n  data: TData;\n};\n","import { AbstractProvider, ProviderResponse } from \"./provider\";\nimport {\n  ResponseFormatJSONObject,\n  ResponseFormatJSONSchema,\n  ResponseFormatText,\n} from \"openai/resources/shared\";\nimport { ChatCompletionMessageParam } from \"openai/resources/chat/completions\";\n\nexport abstract class AbstractLLMProvider extends AbstractProvider {\n  abstract forward(args: LLMProviderForwardArgs): Promise<ChatResponse>;\n}\n\nexport type LLMProviderForwardArgs = {\n  messages: ChatCompletionMessageParam[];\n  model: string;\n  abortSignal?: AbortSignal;\n  temperature?: number;\n  responseFormat?:\n    | ResponseFormatText\n    | ResponseFormatJSONSchema\n    | ResponseFormatJSONObject;\n};\n\nexport type ChatResponse = ProviderResponse<string> & {\n  inputTokensUsed?: number;\n  outputTokensUsed?: number;\n  inputCost?: string;\n  outputCost?: string;\n\n  metadata?: Record<string, unknown>;\n};\n","import {\n  AbstractLLMProvider,\n  type ChatResponse,\n  type LLMProviderForwardArgs,\n} from \"./abstract/llm\";\nimport { MastraClient, type GetAgentResponse } from \"@mastra/client-js\";\n\nexport class MastraProvider extends AbstractLLMProvider {\n  override readonly kind = \"mastra\";\n\n  private readonly endpoint: string;\n  private readonly authToken?: string;\n  private client: MastraClient;\n  private memory?: AgentMemoryOption;\n\n  constructor(params: {\n    endpoint: string;\n    authToken?: string;\n    memory?: AgentMemoryOption;\n  }) {\n    super();\n    this.endpoint = params.endpoint;\n    this.authToken = params.authToken;\n    this.client = new MastraClient({\n      baseUrl: this.endpoint,\n      headers: this.authToken\n        ? {\n          Authorization: `Bearer ${this.authToken}`,\n        }\n        : undefined,\n    });\n    this.memory = params.memory;\n  }\n\n  override async forward(\n    args: LLMProviderForwardArgs & {\n      memory?: AgentMemoryOption;\n\n      /**\n       * The model that will be used as the brain for the agent.\n       */\n      modelName?: string\n    }\n  ): Promise<ChatResponse> {\n    const apiMessages = args.messages\n      .filter((m) => m.role === \"user\" || m.role === \"assistant\")\n      .map((m) => ({\n        role: m.role as \"user\" | \"assistant\",\n        content: String((m as any).content ?? \"\"),\n      }));\n\n    const agent = this.client.getAgent(args.model);\n\n    const startedAt = Date.now();\n    const response = await agent.generate(\n      {\n        messages: apiMessages,\n        runtimeContext: {\n          \"model-id\": args.model,\n        },\n      },\n      { memory: args.memory ?? this.memory }\n    );\n\n    return {\n      data: response.text,\n      startedAt,\n      completedAt: Date.now(),\n    };\n  }\n\n  async getAgentInfo(args: {\n    agentId: string;\n    runtimeContext?: MastraRuntimeContext;\n  }) {\n    return await this.client\n      .getAgent(args.agentId)\n      .details(args.runtimeContext);\n  }\n\n  async getAgents(args?: {\n    runtimeContext?: MastraRuntimeContext;\n    partial?: boolean;\n  }): Promise<Record<string, GetAgentResponse>> {\n    return this.client.getAgents(args?.runtimeContext, args?.partial);\n  }\n}\n\n// NOTE: Mastra client does not export these types\nexport type AgentMemoryOption = Parameters<\n  Parameters<MastraClient[\"getAgent\"]>[\"0\"] extends string\n  ? ReturnType<MastraClient[\"getAgent\"]>[\"generate\"]\n  : never\n>[0] extends { memory?: infer M }\n  ? M\n  : never;\n\ntype MastraRuntimeContext = Parameters<\n  Parameters<MastraClient[\"getAgent\"]>[\"0\"] extends string\n  ? ReturnType<MastraClient[\"getAgent\"]>[\"generate\"]\n  : never\n>[0] extends { runtimeContext?: infer R }\n  ? R\n  : never;\n","import { RateLimiter } from \"@/utils\";\nimport { ChatCompletionMessageParam } from \"openai/resources/chat/completions\";\nimport {\n  ResponseFormatJSONObject,\n  ResponseFormatJSONSchema,\n  ResponseFormatText,\n} from \"openai/resources/shared\";\nimport OpenAI, { APIError } from \"openai\";\nimport { AbstractLLMProvider, ChatResponse } from \"./abstract/llm\";\nimport { PEERBENCH_NAMESPACE } from \"@/constants\";\n\nexport class OpenAIProvider extends AbstractLLMProvider {\n  override readonly kind = `${PEERBENCH_NAMESPACE}/llm/openai` as const;\n\n  private client: OpenAI;\n  private rateLimiter: RateLimiter;\n  private maxRetries: number;\n\n  constructor(config: {\n    apiKey: string;\n    baseURL: string;\n    maxRetries?: number;\n    timeout?: number;\n    rateLimiter?: RateLimiter;\n  }) {\n    super();\n    this.maxRetries = config.maxRetries ?? 3;\n    this.rateLimiter =\n      config.rateLimiter ??\n      new RateLimiter({\n        maxWeight: 20,\n        timeWindow: 3_000,\n      });\n\n    this.client = new OpenAI({\n      baseURL: config.baseURL,\n      apiKey: config.apiKey,\n      timeout: config.timeout,\n      dangerouslyAllowBrowser: true,\n    });\n  }\n\n  async forward(args: {\n    messages: ChatCompletionMessageParam[];\n    model: string;\n    abortSignal?: AbortSignal;\n    temperature?: number;\n    responseFormat?:\n      | ResponseFormatText\n      | ResponseFormatJSONSchema\n      | ResponseFormatJSONObject;\n  }): Promise<ChatResponse> {\n    let retryCount = this.maxRetries;\n    while (retryCount > 0) {\n      let startedAt: Date = new Date();\n\n      try {\n        const response = await this.rateLimiter.execute(\n          async () => {\n            // Capture the start time of the request\n            startedAt = new Date();\n            return await this.client.chat.completions.create(\n              {\n                model: args.model,\n                messages: args.messages,\n                temperature: args.temperature,\n                response_format: args.responseFormat,\n              },\n              // Signal for request\n              { signal: args.abortSignal }\n            );\n          },\n          // Signal for rate limiting\n          { signal: args.abortSignal }\n        );\n\n        if (\"error\" in response) {\n          const err = response.error as any;\n          throw new Error(\n            `${err.message} - Code ${err.code} - ${JSON.stringify(err)}`\n          );\n        }\n\n        if (!response?.choices?.[0]?.message?.content) {\n          throw new Error(\"No content returned from the model\");\n        }\n\n        return {\n          data: response.choices[0].message.content,\n\n          inputTokensUsed: response?.usage?.prompt_tokens,\n          outputTokensUsed: response?.usage?.completion_tokens,\n\n          startedAt: startedAt.getTime(),\n          completedAt: Date.now(),\n        };\n      } catch (err) {\n        if (err instanceof APIError && err.status === 401) {\n          throw new Error(`Invalid credentials provided`, { cause: err });\n        }\n\n        retryCount--;\n\n        // More likely an empty HTTP response returned by the Provider\n        // and it couldn't be parsed as JSON by the OpenAI SDK. We need to retry the request\n        // More info can be found in the following links:\n        // https://www.reddit.com/r/SillyTavernAI/comments/1ik95vr/deepseek_r1_on_openrouter_returning_blank_messages/\n        // https://github.com/cline/cline/issues/60\n        if (err instanceof SyntaxError) {\n          console.debug(err);\n          continue;\n        }\n\n        // If it was another error, just continue until we run out of retries\n        if (retryCount !== 0) {\n          continue;\n        }\n\n        throw new Error(\n          `Failed to forward prompt to the model: ${err instanceof Error ? err.message : err}`,\n          { cause: err }\n        );\n      }\n    }\n\n    throw new Error(\n      `Failed to forward prompt to the model: Max retries reached`,\n      { cause: new Error(\"Max retries reached\") }\n    );\n  }\n}\n","import {\n  AbstractLLMProvider,\n  ChatResponse,\n  LLMProviderForwardArgs,\n} from \"./abstract/llm\";\nimport { RateLimiter } from \"@/utils\";\nimport { OpenAIProvider } from \"./openai\";\nimport { PEERBENCH_NAMESPACE } from \"@/constants\";\nimport Decimal from \"decimal.js\";\nimport axios from \"axios\";\n\nconst baseURL = \"https://openrouter.ai/api/v1\";\nconst MODELS_CACHE_TTL = 1000 * 60 * 60 * 24; // 24 hours\n\nexport class OpenRouterProvider extends AbstractLLMProvider {\n  override readonly kind = `${PEERBENCH_NAMESPACE}/llm/openrouter.ai` as const;\n\n  private models: ModelsResponse | undefined = undefined;\n  private modelsCachePromise: Promise<ModelsResponse | undefined> =\n    Promise.resolve(undefined);\n  private modelsUpdatedAt = 0;\n  private openAIProvider: OpenAIProvider;\n\n  constructor(config: {\n    apiKey: string;\n    maxRetries?: number;\n    timeout?: number;\n    rateLimiter?: RateLimiter;\n  }) {\n    super();\n    this.openAIProvider = new OpenAIProvider({\n      baseURL,\n      apiKey: config.apiKey,\n      maxRetries: config.maxRetries,\n      timeout: config.timeout,\n      rateLimiter: config.rateLimiter,\n    });\n  }\n\n  override async forward(args: LLMProviderForwardArgs): Promise<ChatResponse> {\n    // Update models cache concurrently (non-blocking)\n    const [response] = await Promise.all([\n      this.openAIProvider.forward(args),\n      this.updateModelsCache().catch(() => {\n        // Silently fail if cache update fails so we won't have cost info in the result\n      }),\n    ]);\n\n    // Get the model info from the cache\n    const modelInfo = this.models?.data.find((m) => m.id === args.model);\n    let inputCost: string | undefined = undefined;\n    let outputCost: string | undefined = undefined;\n\n    if (modelInfo !== undefined) {\n      // Use Decimal.js for more accurate calculation\n      if (response.inputTokensUsed !== undefined) {\n        inputCost = new Decimal(modelInfo.pricing.prompt)\n          .mul(response.inputTokensUsed)\n          .toFixed(10);\n      }\n      if (response.outputTokensUsed !== undefined) {\n        outputCost = new Decimal(modelInfo.pricing.completion)\n          .mul(response.outputTokensUsed)\n          .toFixed(10);\n      }\n    }\n\n    return {\n      ...response,\n      inputCost,\n      outputCost,\n    };\n  }\n\n  /**\n   * Updates the cache that holds information about OpenRouter models\n   * including pricing information. It will be valid for 24 hours as\n   * long as the instance of this Provider object is alive.\n   */\n  private async updateModelsCache() {\n    // Chain each update method call to the promise.\n    // This approach prevents race conditions between multiple calls.\n    // Since each call is chained to the end of the previous one,\n    // each promise makes a request only if the models cache is not updated\n    // in the last call. Otherwise it simply resolves to the cached value.\n    this.modelsCachePromise = this.modelsCachePromise\n      .then(async () => {\n        if (\n          // The data presented in the cache\n          this.models !== undefined &&\n          // The cache is still valid\n          Date.now() - this.modelsUpdatedAt < MODELS_CACHE_TTL\n        ) {\n          return this.models;\n        }\n\n        // If the cache is not valid, update it\n        return axios\n          .get<ModelsResponse>(`${baseURL}/models`)\n          .then((res) => res.data)\n          .then((data) => {\n            // Only get the models that supports text input and output\n            data = {\n              data: data.data.filter(\n                (m) =>\n                  m.architecture.input_modalities.includes(\"text\") &&\n                  m.architecture.output_modalities.includes(\"text\") &&\n                  // These models are \"fast apply model\" and don't support multi turn conversations so don't include them\n                  ![\n                    \"morph/morph-v3-large\",\n                    \"morph/morph-v3-fast\",\n                    \"relace/relace-apply-3\",\n                  ].includes(m.id)\n              ),\n            };\n\n            this.models = data;\n            this.modelsUpdatedAt = Date.now();\n\n            return data;\n          });\n      })\n      .catch(() => undefined);\n\n    // Wait for the promise chain to resolve\n    await this.modelsCachePromise;\n  }\n}\n\ntype PutModality = \"text\" | \"image\" | \"file\" | \"audio\";\ntype Modality = \"text->text\" | \"text+image->text\" | \"text+image->text+image\";\ntype ModelsResponse = {\n  data: {\n    readonly id: string;\n    readonly canonical_slug: string;\n    readonly hugging_face_id: null | string;\n    readonly name: string;\n    readonly created: number;\n    readonly description: string;\n    readonly context_length: number;\n    readonly architecture: {\n      readonly modality: Modality;\n      readonly input_modalities: PutModality[];\n      readonly output_modalities: PutModality[];\n      readonly instruct_type: null | string;\n    };\n    readonly pricing: {\n      readonly prompt: string;\n      readonly completion: string;\n      readonly request?: string;\n      readonly image?: string;\n      readonly web_search?: string;\n      readonly internal_reasoning?: string;\n      readonly input_cache_read?: string;\n      readonly input_cache_write?: string;\n      readonly audio?: string;\n    };\n  }[];\n};\n"],"mappings":";;;;;;;;AAAO,IAAe,mBAAf,MAAgC;AAEvC;;;ACMO,IAAe,sBAAf,cAA2C,iBAAiB;AAEnE;;;ACLA,SAAS,oBAA2C;AAE7C,IAAM,iBAAN,cAA6B,oBAAoB;AAAA,EACpC,OAAO;AAAA,EAER;AAAA,EACA;AAAA,EACT;AAAA,EACA;AAAA,EAER,YAAY,QAIT;AACD,UAAM;AACN,SAAK,WAAW,OAAO;AACvB,SAAK,YAAY,OAAO;AACxB,SAAK,SAAS,IAAI,aAAa;AAAA,MAC7B,SAAS,KAAK;AAAA,MACd,SAAS,KAAK,YACV;AAAA,QACA,eAAe,UAAU,KAAK,SAAS;AAAA,MACzC,IACE;AAAA,IACN,CAAC;AACD,SAAK,SAAS,OAAO;AAAA,EACvB;AAAA,EAEA,MAAe,QACb,MAQuB;AACvB,UAAM,cAAc,KAAK,SACtB,OAAO,CAAC,MAAM,EAAE,SAAS,UAAU,EAAE,SAAS,WAAW,EACzD,IAAI,CAAC,OAAO;AAAA,MACX,MAAM,EAAE;AAAA,MACR,SAAS,OAAQ,EAAU,WAAW,EAAE;AAAA,IAC1C,EAAE;AAEJ,UAAM,QAAQ,KAAK,OAAO,SAAS,KAAK,KAAK;AAE7C,UAAM,YAAY,KAAK,IAAI;AAC3B,UAAM,WAAW,MAAM,MAAM;AAAA,MAC3B;AAAA,QACE,UAAU;AAAA,QACV,gBAAgB;AAAA,UACd,YAAY,KAAK;AAAA,QACnB;AAAA,MACF;AAAA,MACA,EAAE,QAAQ,KAAK,UAAU,KAAK,OAAO;AAAA,IACvC;AAEA,WAAO;AAAA,MACL,MAAM,SAAS;AAAA,MACf;AAAA,MACA,aAAa,KAAK,IAAI;AAAA,IACxB;AAAA,EACF;AAAA,EAEA,MAAM,aAAa,MAGhB;AACD,WAAO,MAAM,KAAK,OACf,SAAS,KAAK,OAAO,EACrB,QAAQ,KAAK,cAAc;AAAA,EAChC;AAAA,EAEA,MAAM,UAAU,MAG8B;AAC5C,WAAO,KAAK,OAAO,UAAU,MAAM,gBAAgB,MAAM,OAAO;AAAA,EAClE;AACF;;;AC/EA,OAAO,UAAU,gBAAgB;AAI1B,IAAM,iBAAN,cAA6B,oBAAoB;AAAA,EACpC,OAAO,GAAG,mBAAmB;AAAA,EAEvC;AAAA,EACA;AAAA,EACA;AAAA,EAER,YAAY,QAMT;AACD,UAAM;AACN,SAAK,aAAa,OAAO,cAAc;AACvC,SAAK,cACH,OAAO,eACP,IAAI,YAAY;AAAA,MACd,WAAW;AAAA,MACX,YAAY;AAAA,IACd,CAAC;AAEH,SAAK,SAAS,IAAI,OAAO;AAAA,MACvB,SAAS,OAAO;AAAA,MAChB,QAAQ,OAAO;AAAA,MACf,SAAS,OAAO;AAAA,MAChB,yBAAyB;AAAA,IAC3B,CAAC;AAAA,EACH;AAAA,EAEA,MAAM,QAAQ,MASY;AACxB,QAAI,aAAa,KAAK;AACtB,WAAO,aAAa,GAAG;AACrB,UAAI,YAAkB,oBAAI,KAAK;AAE/B,UAAI;AACF,cAAM,WAAW,MAAM,KAAK,YAAY;AAAA,UACtC,YAAY;AAEV,wBAAY,oBAAI,KAAK;AACrB,mBAAO,MAAM,KAAK,OAAO,KAAK,YAAY;AAAA,cACxC;AAAA,gBACE,OAAO,KAAK;AAAA,gBACZ,UAAU,KAAK;AAAA,gBACf,aAAa,KAAK;AAAA,gBAClB,iBAAiB,KAAK;AAAA,cACxB;AAAA;AAAA,cAEA,EAAE,QAAQ,KAAK,YAAY;AAAA,YAC7B;AAAA,UACF;AAAA;AAAA,UAEA,EAAE,QAAQ,KAAK,YAAY;AAAA,QAC7B;AAEA,YAAI,WAAW,UAAU;AACvB,gBAAM,MAAM,SAAS;AACrB,gBAAM,IAAI;AAAA,YACR,GAAG,IAAI,OAAO,WAAW,IAAI,IAAI,MAAM,KAAK,UAAU,GAAG,CAAC;AAAA,UAC5D;AAAA,QACF;AAEA,YAAI,CAAC,UAAU,UAAU,CAAC,GAAG,SAAS,SAAS;AAC7C,gBAAM,IAAI,MAAM,oCAAoC;AAAA,QACtD;AAEA,eAAO;AAAA,UACL,MAAM,SAAS,QAAQ,CAAC,EAAE,QAAQ;AAAA,UAElC,iBAAiB,UAAU,OAAO;AAAA,UAClC,kBAAkB,UAAU,OAAO;AAAA,UAEnC,WAAW,UAAU,QAAQ;AAAA,UAC7B,aAAa,KAAK,IAAI;AAAA,QACxB;AAAA,MACF,SAAS,KAAK;AACZ,YAAI,eAAe,YAAY,IAAI,WAAW,KAAK;AACjD,gBAAM,IAAI,MAAM,gCAAgC,EAAE,OAAO,IAAI,CAAC;AAAA,QAChE;AAEA;AAOA,YAAI,eAAe,aAAa;AAC9B,kBAAQ,MAAM,GAAG;AACjB;AAAA,QACF;AAGA,YAAI,eAAe,GAAG;AACpB;AAAA,QACF;AAEA,cAAM,IAAI;AAAA,UACR,0CAA0C,eAAe,QAAQ,IAAI,UAAU,GAAG;AAAA,UAClF,EAAE,OAAO,IAAI;AAAA,QACf;AAAA,MACF;AAAA,IACF;AAEA,UAAM,IAAI;AAAA,MACR;AAAA,MACA,EAAE,OAAO,IAAI,MAAM,qBAAqB,EAAE;AAAA,IAC5C;AAAA,EACF;AACF;;;AC1HA,OAAO,aAAa;AACpB,OAAO,WAAW;AAElB,IAAM,UAAU;AAChB,IAAM,mBAAmB,MAAO,KAAK,KAAK;AAEnC,IAAM,qBAAN,cAAiC,oBAAoB;AAAA,EACxC,OAAO,GAAG,mBAAmB;AAAA,EAEvC,SAAqC;AAAA,EACrC,qBACN,QAAQ,QAAQ,MAAS;AAAA,EACnB,kBAAkB;AAAA,EAClB;AAAA,EAER,YAAY,QAKT;AACD,UAAM;AACN,SAAK,iBAAiB,IAAI,eAAe;AAAA,MACvC;AAAA,MACA,QAAQ,OAAO;AAAA,MACf,YAAY,OAAO;AAAA,MACnB,SAAS,OAAO;AAAA,MAChB,aAAa,OAAO;AAAA,IACtB,CAAC;AAAA,EACH;AAAA,EAEA,MAAe,QAAQ,MAAqD;AAE1E,UAAM,CAAC,QAAQ,IAAI,MAAM,QAAQ,IAAI;AAAA,MACnC,KAAK,eAAe,QAAQ,IAAI;AAAA,MAChC,KAAK,kBAAkB,EAAE,MAAM,MAAM;AAAA,MAErC,CAAC;AAAA,IACH,CAAC;AAGD,UAAM,YAAY,KAAK,QAAQ,KAAK,KAAK,CAAC,MAAM,EAAE,OAAO,KAAK,KAAK;AACnE,QAAI,YAAgC;AACpC,QAAI,aAAiC;AAErC,QAAI,cAAc,QAAW;AAE3B,UAAI,SAAS,oBAAoB,QAAW;AAC1C,oBAAY,IAAI,QAAQ,UAAU,QAAQ,MAAM,EAC7C,IAAI,SAAS,eAAe,EAC5B,QAAQ,EAAE;AAAA,MACf;AACA,UAAI,SAAS,qBAAqB,QAAW;AAC3C,qBAAa,IAAI,QAAQ,UAAU,QAAQ,UAAU,EAClD,IAAI,SAAS,gBAAgB,EAC7B,QAAQ,EAAE;AAAA,MACf;AAAA,IACF;AAEA,WAAO;AAAA,MACL,GAAG;AAAA,MACH;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAc,oBAAoB;AAMhC,SAAK,qBAAqB,KAAK,mBAC5B,KAAK,YAAY;AAChB;AAAA;AAAA,QAEE,KAAK,WAAW;AAAA,QAEhB,KAAK,IAAI,IAAI,KAAK,kBAAkB;AAAA,QACpC;AACA,eAAO,KAAK;AAAA,MACd;AAGA,aAAO,MACJ,IAAoB,GAAG,OAAO,SAAS,EACvC,KAAK,CAAC,QAAQ,IAAI,IAAI,EACtB,KAAK,CAAC,SAAS;AAEd,eAAO;AAAA,UACL,MAAM,KAAK,KAAK;AAAA,YACd,CAAC,MACC,EAAE,aAAa,iBAAiB,SAAS,MAAM,KAC/C,EAAE,aAAa,kBAAkB,SAAS,MAAM;AAAA,YAEhD,CAAC;AAAA,cACC;AAAA,cACA;AAAA,cACA;AAAA,YACF,EAAE,SAAS,EAAE,EAAE;AAAA,UACnB;AAAA,QACF;AAEA,aAAK,SAAS;AACd,aAAK,kBAAkB,KAAK,IAAI;AAEhC,eAAO;AAAA,MACT,CAAC;AAAA,IACL,CAAC,EACA,MAAM,MAAM,MAAS;AAGxB,UAAM,KAAK;AAAA,EACb;AACF;","names":[]}

package/dist/chunk-QY5MPNNB.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../src/helpers/define-runner.ts"],"sourcesContent":["import {\n  IdGenerator,\n  InferRunConfig,\n  ProviderCtor,\n  Runner,\n  ScorerCtor,\n} from \"@/types\";\nimport { idGeneratorUUIDv7 } from \"@/utils\";\nimport z from \"zod\";\n\nexport function defineRunner<\n  const TProviders extends ProviderCtor[],\n  const TScorers extends ScorerCtor[],\n  const TSchemaSets extends SchemaSetDefinition[],\n  const TRunConfigSchema extends z.ZodRawShape = {},\n>(\n  config: {\n    schemaSets: TSchemaSets;\n    providers: TProviders;\n    scorers: TScorers;\n    runConfigSchema?: TRunConfigSchema;\n\n    /**\n     * @default true\n     */\n    parseRunConfig?: boolean;\n    defaults?: {\n      scorer?: InstanceType<TScorers[number]>;\n      responseIdGenerator?: IdGenerator;\n      scoreIdGenerator?: IdGenerator;\n    };\n  },\n  fn: Runner<\n    TSchemaSets[number][\"testCase\"],\n    TSchemaSets[number][\"response\"],\n    TSchemaSets[number][\"score\"],\n    InstanceType<TProviders[number]>,\n    InstanceType<TScorers[number]>,\n    InferRunConfig<TRunConfigSchema>\n  >\n) {\n  return async (params: Parameters<typeof fn>[0]) => {\n    if (config.runConfigSchema && config.parseRunConfig !== false) {\n      z.object(config.runConfigSchema).parse(params.runConfig);\n    }\n\n    if (params.idGenerators && !params.idGenerators.response) {\n      params.idGenerators.response =\n        config.defaults?.responseIdGenerator ?? idGeneratorUUIDv7;\n    }\n\n    if (params.idGenerators && !params.idGenerators.score) {\n      params.idGenerators.score =\n        config.defaults?.scoreIdGenerator ?? idGeneratorUUIDv7;\n    }\n\n    if (params.scorer === undefined) {\n      params.scorer = config.defaults?.scorer ?? undefined;\n    }\n\n    return await fn(params);\n  };\n}\n\ntype SchemaSetDefinition<\n  TTestCase extends z.ZodObject = z.ZodObject,\n  TResponse extends z.ZodObject = z.ZodObject,\n  TScore extends z.ZodObject = z.ZodObject,\n> = {\n  testCase: TTestCase;\n  response: TResponse;\n  score: TScore;\n};\n"],"mappings":";;;;;AAQA,OAAO,OAAO;AAEP,SAAS,aAMd,QAgBA,IAQA;AACA,SAAO,OAAO,WAAqC;AACjD,QAAI,OAAO,mBAAmB,OAAO,mBAAmB,OAAO;AAC7D,QAAE,OAAO,OAAO,eAAe,EAAE,MAAM,OAAO,SAAS;AAAA,IACzD;AAEA,QAAI,OAAO,gBAAgB,CAAC,OAAO,aAAa,UAAU;AACxD,aAAO,aAAa,WAClB,OAAO,UAAU,uBAAuB;AAAA,IAC5C;AAEA,QAAI,OAAO,gBAAgB,CAAC,OAAO,aAAa,OAAO;AACrD,aAAO,aAAa,QAClB,OAAO,UAAU,oBAAoB;AAAA,IACzC;AAEA,QAAI,OAAO,WAAW,QAAW;AAC/B,aAAO,SAAS,OAAO,UAAU,UAAU;AAAA,IAC7C;AAEA,WAAO,MAAM,GAAG,MAAM;AAAA,EACxB;AACF;","names":[]}

package/dist/provider-BDjGp2y-.d.ts

@@ -1,10 +0,0 @@
-declare abstract class AbstractProvider {
-    abstract readonly kind: string;
-}
-type ProviderResponse<TData = unknown> = {
-    startedAt: number;
-    completedAt: number;
-    data: TData;
-};
-
-export { AbstractProvider as A, type ProviderResponse as P };