peerbench 0.0.10 → 0.0.12

package/dist/benchmarks/index.js

@@ -1,12 +1,6 @@
-import {
-  SimpleSystemPromptSchemaV1
-} from "../chunk-Q6GSOHOP.js";
 import {
   defineRunner
-} from "../chunk-RTEAK4II.js";
-import {
-  AbstractLLMProvider
-} from "../chunk-SMLNDQFX.js";
+} from "../chunk-6WDCU5BP.js";
 import {
   BaseResponseSchemaV1,
   BaseScoreSchemaV1,
@@ -14,28 +8,24 @@ import {
   defineResponseSchema,
   defineScoreSchema,
   defineTestCaseSchema
-} from "../chunk-YY33MNMV.js";
+} from "../chunk-7KMGLEYP.js";
 import "../chunk-OQE6TQXZ.js";
 import {
   ScoringMethod
-} from "../chunk-HMQYGCKI.js";
-import {
-  JSONFileStorage
-} from "../chunk-WBCMV445.js";
-import {
-  LLMAsAJudgeScorer,
-  MCQScorer
-} from "../chunk-3JHDJEY3.js";
+} from "../chunk-ZJWSK4VO.js";
 import {
   PEERBENCH_NAMESPACE
 } from "../chunk-UHHHSYVE.js";
+import {
+  JSONFileStorage
+} from "../chunk-WBCMV445.js";
 import {
   idGeneratorUUIDv7
 } from "../chunk-4UBK6452.js";
 import {
   ExtensionLLMAsAJudgeScoreFieldsV1,
   ExtensionLLMResponseFieldsV1
-} from "../chunk-TRNCF2BG.js";
+} from "../chunk-HBGC6BDW.js";
 import "../chunk-NUEOE3K5.js";
 import {
   __export
@@ -57,7 +47,8 @@ __export(peerbench_exports, {
   QAResponseSchemaV1: () => QAResponseSchemaV1,
   QAScoreSchemaV1: () => QAScoreSchemaV1,
   QATestCaseSchemaV1: () => QATestCaseSchemaV1,
-  peerbenchRunner: () => peerbenchRunner
+  mcqRunner: () => mcqRunner,
+  qaRunner: () => qaRunner
 });
 
 // src/benchmarks/peerbench/schema-sets/mcq.v1.ts
@@ -179,254 +170,112 @@ var QAScoreSchemaV1 = defineScoreSchema({
   }
 });
 
-// src/benchmarks/peerbench/runner.ts
+// src/benchmarks/peerbench/mcq-runner.ts
 import Handlebars from "handlebars";
 import z4 from "zod";
-var peerbenchRunner = defineRunner(
-  {
-    schemaSets: [
-      {
-        testCase: MCQTestCaseSchemaV1,
-        response: MCQResponseSchemaV1,
-        score: MCQScoreSchemaV1
-      },
-      {
-        testCase: QATestCaseSchemaV1,
-        response: QAResponseSchemaV1,
-        score: QAScoreSchemaV1
-      }
-    ],
-    providers: [AbstractLLMProvider],
-    scorers: [LLMAsAJudgeScorer, MCQScorer],
-    runConfigSchema: {
-      model: z4.string(),
-      llmJudgeModel: z4.string().optional(),
-      llmJudgeSystemPrompt: SimpleSystemPromptSchemaV1.optional(),
-      llmJudgeFieldsToExtract: z4.record(z4.string(), z4.custom()).optional(),
-      systemPrompt: SimpleSystemPromptSchemaV1.optional(),
-      templateVariables: z4.record(z4.string(), z4.string()).optional()
-    }
-  },
+var mcqRunner = defineRunner(
   async (params) => {
-    const { testCase, provider, scorer, runConfig } = params;
+    const { testCase, target, scorer } = params;
     const messages = [];
-    if (runConfig.systemPrompt) {
+    if (params.systemPrompt) {
       messages.push({
         role: "system",
-        content: runConfig.systemPrompt.content
+        content: params.systemPrompt.content
       });
     }
-    if (testCase.kind === "llm/mcq.tc") {
-      messages.push({
-        role: "user",
-        content: formatMCQ(testCase)
-      });
-      templateMessages(messages, runConfig.templateVariables ?? {});
-      return runMCQ({
-        testCase,
-        messages,
-        provider,
-        scorer,
-        runConfig,
-        idGenerators: {
-          response: params.idGenerators?.response ?? idGeneratorUUIDv7,
-          score: params.idGenerators?.score ?? idGeneratorUUIDv7
-        }
+    messages.push({
+      role: "user",
+      content: formatMCQ(testCase)
+    });
+    templateMessages(messages, params.templateVariables ?? {});
+    const providerResponse = await target.forward({ messages });
+    const response = await MCQResponseSchemaV1.newWithId(
+      {
+        data: providerResponse.data,
+        startedAt: providerResponse.startedAt,
+        completedAt: providerResponse.completedAt,
+        testCaseId: testCase.id,
+        modelSlug: target.slug,
+        provider: target.provider.kind,
+        systemPromptId: params.systemPrompt?.id,
+        inputTokensUsed: providerResponse.inputTokensUsed,
+        outputTokensUsed: providerResponse.outputTokensUsed,
+        inputCost: providerResponse.inputCost,
+        outputCost: providerResponse.outputCost
+      },
+      params.idGenerators?.response ?? idGeneratorUUIDv7
+    );
+    if (scorer?.kind === `${PEERBENCH_NAMESPACE}/mcq`) {
+      const scorerResult = await scorer.score({
+        response: response.data,
+        choices: testCase.options,
+        correctAnswers: testCase.correctAnswerKeys
       });
-    }
-    if (testCase.kind === "llm/qa.tc") {
-      if (scorer && scorer?.kind !== `${PEERBENCH_NAMESPACE}/llm-as-a-judge`) {
-        throw new Error(
-          `QA test cases can only be scored with an LLM as a judge scorer, but ${scorer?.kind} was provided`
+      if (scorerResult !== null) {
+        const score = await MCQScoreSchemaV1.newWithId(
+          {
+            scoringMethod: ScoringMethod.algo,
+            value: scorerResult.value,
+            responseId: response.id,
+            extractedAnswers: scorerResult.extractedAnswers,
+            explanation: scorerResult.explanation,
+            metadata: scorerResult.metadata
+          },
+          params.idGenerators?.score ?? idGeneratorUUIDv7
         );
+        return { response, score };
       }
-      messages.push({
-        role: "user",
-        content: testCase.question
-      });
-      templateMessages(messages, runConfig.templateVariables ?? {});
-      return runQA({
-        testCase,
-        messages,
-        provider,
-        scorer,
-        runConfig,
-        idGenerators: {
-          response: params.idGenerators?.response ?? idGeneratorUUIDv7,
-          score: params.idGenerators?.score ?? idGeneratorUUIDv7
-        }
-      });
-    }
-    throw new Error("Unsupported test case kind");
-  }
-);
-async function runQA(params) {
-  const { messages, testCase, provider, scorer, runConfig } = params;
-  const providerResponse = await provider.forward({
-    model: runConfig.model,
-    messages
-  });
-  const response = await QAResponseSchemaV1.newWithId(
-    {
-      data: providerResponse.data,
-      startedAt: providerResponse.startedAt,
-      completedAt: providerResponse.completedAt,
-      testCaseId: testCase.id,
-      modelSlug: runConfig.model,
-      provider: provider.kind,
-      systemPromptId: runConfig.systemPrompt?.id,
-      inputTokensUsed: providerResponse.inputTokensUsed,
-      outputTokensUsed: providerResponse.outputTokensUsed,
-      inputCost: providerResponse.inputCost,
-      outputCost: providerResponse.outputCost
-    },
-    params.idGenerators?.response ?? idGeneratorUUIDv7
-  );
-  if (scorer?.kind === `${PEERBENCH_NAMESPACE}/llm-as-a-judge`) {
-    if (!runConfig.llmJudgeModel) {
-      throw new Error(
-        "LLM judge model is required when using LLM as a judge scorer"
-      );
     }
-    const scorerResult = await scorer.score({
-      model: runConfig.llmJudgeModel,
-      response: response.data,
-      rubric: `Expected/Valid answers: ${testCase.goodAnswers.join("\n")}
-Invalid answers: ${testCase.badAnswers.join("\n")}`,
-      systemPrompt: runConfig.llmJudgeSystemPrompt?.content,
-      criteria: [
-        {
-          id: "correctness",
-          description: "Is the response matches with the expected/valid answers in terms of meaning?",
-          weight: 1
-        }
-      ],
-      fieldsToExtract: runConfig.llmJudgeFieldsToExtract ?? {}
-    });
-    if (scorerResult !== null) {
-      const score = await QAScoreSchemaV1.newWithId(
-        {
-          scoringMethod: ScoringMethod.ai,
-          value: scorerResult.value,
-          responseId: response.id,
-          explanation: scorerResult.explanation,
-          scorerAIInputCost: scorerResult.inputCost,
-          scorerAIOutputCost: scorerResult.outputCost,
-          scorerAIInputTokensUsed: scorerResult.inputTokensUsed,
-          scorerAIOutputTokensUsed: scorerResult.outputTokensUsed,
-          scorerAIProvider: scorerResult.provider,
-          scorerAIModelSlug: runConfig.llmJudgeModel,
-          scorerAISystemPromptId: runConfig.llmJudgeSystemPrompt?.id,
-          metadata: {
-            ...scorerResult.metadata,
-            extractedFields: scorerResult.extractedFields
+    if (scorer?.kind === `${PEERBENCH_NAMESPACE}/llm-as-a-judge`) {
+      const scorerResult = await scorer.score({
+        criteria: [
+          {
+            id: "correctness",
+            description: "Is the given answer key matches with one of the correct answer keys?",
+            weight: 1
           }
-        },
-        params.idGenerators?.score ?? idGeneratorUUIDv7
-      );
-      return { response, score };
-    }
-  }
-  return { response };
-}
-async function runMCQ(params) {
-  const { messages, testCase, provider, scorer, runConfig } = params;
-  const providerResponse = await provider.forward({
-    model: runConfig.model,
-    messages
-  });
-  const response = await MCQResponseSchemaV1.newWithId(
-    {
-      data: providerResponse.data,
-      startedAt: providerResponse.startedAt,
-      completedAt: providerResponse.completedAt,
-      testCaseId: testCase.id,
-      modelSlug: runConfig.model,
-      provider: provider.kind,
-      systemPromptId: runConfig.systemPrompt?.id,
-      inputTokensUsed: providerResponse.inputTokensUsed,
-      outputTokensUsed: providerResponse.outputTokensUsed,
-      inputCost: providerResponse.inputCost,
-      outputCost: providerResponse.outputCost
-    },
-    params.idGenerators?.response ?? idGeneratorUUIDv7
-  );
-  if (scorer?.kind === `${PEERBENCH_NAMESPACE}/mcq`) {
-    const scorerResult = await scorer.score({
-      response: response.data,
-      choices: testCase.options,
-      correctAnswers: testCase.correctAnswerKeys
-    });
-    if (scorerResult !== null) {
-      const score = await MCQScoreSchemaV1.newWithId(
-        {
-          scoringMethod: ScoringMethod.algo,
-          value: scorerResult.value,
-          responseId: response.id,
-          extractedAnswers: scorerResult.extractedAnswers,
-          explanation: scorerResult.explanation,
-          metadata: scorerResult.metadata
-        },
-        params.idGenerators?.score ?? idGeneratorUUIDv7
-      );
-      return { response, score };
-    }
-  }
-  if (scorer?.kind === `${PEERBENCH_NAMESPACE}/llm-as-a-judge`) {
-    if (!runConfig.llmJudgeModel) {
-      throw new Error(
-        "LLM judge model is required when using LLM as a judge scorer"
-      );
-    }
-    const scorerResult = await scorer.score({
-      model: runConfig.llmJudgeModel,
-      criteria: [
-        {
-          id: "correctness",
-          description: "Is the given answer key matches with one of the correct answer keys?",
-          weight: 1
-        }
-      ],
-      rubric: `Answer text itself or the key (A, B, C) is accepted
+        ],
+        rubric: `Answer text itself or the key (A, B, C) is accepted
 Valid answer keys: ${testCase.correctAnswerKeys.map((key) => `- ${key}`).join("\n")}
 Valid Answer texts: ${testCase.correctAnswerKeys.map((key) => `- ${testCase.options?.[key] ?? ""}`).join("\n")}`,
-      fieldsToExtract: {
-        extractedAnswers: z4.string().array().describe(
-          "The extracted answer keys, valid or invalid (even if the answer text is provided rather than the key)"
-        ),
-        ...runConfig.llmJudgeFieldsToExtract ?? {}
-      },
-      response: response.data,
-      systemPrompt: runConfig.llmJudgeSystemPrompt?.content
-    });
-    if (scorerResult !== null) {
-      const { extractedAnswers, ...extractedFields } = scorerResult.extractedFields;
-      const score = await MCQScoreSchemaV1.newWithId(
-        {
-          scoringMethod: ScoringMethod.ai,
-          value: scorerResult.value,
-          extractedAnswers,
-          responseId: response.id,
-          explanation: scorerResult.explanation,
-          scorerAIInputCost: scorerResult.inputCost,
-          scorerAIOutputCost: scorerResult.outputCost,
-          scorerAIInputTokensUsed: scorerResult.inputTokensUsed,
-          scorerAIOutputTokensUsed: scorerResult.outputTokensUsed,
-          scorerAIProvider: scorerResult.provider,
-          scorerAIModelSlug: runConfig.llmJudgeModel,
-          scorerAISystemPromptId: runConfig.llmJudgeSystemPrompt?.id,
-          metadata: {
-            ...scorerResult.metadata,
-            extractedFields
-          }
+        fieldsToExtract: {
+          extractedAnswers: z4.string().array().describe(
+            "The extracted answer keys, valid or invalid (even if the answer text is provided rather than the key)"
+          ),
+          ...params.llmJudgeFieldsToExtract ?? {}
         },
-        params.idGenerators?.score ?? idGeneratorUUIDv7
-      );
-      return { response, score };
+        response: response.data,
+        systemPrompt: params.llmJudgeSystemPrompt?.content
+      });
+      if (scorerResult !== null) {
+        const { extractedAnswers, ...extractedFields } = scorerResult.extractedFields;
+        const score = await MCQScoreSchemaV1.newWithId(
+          {
+            scoringMethod: ScoringMethod.ai,
+            value: scorerResult.value,
+            extractedAnswers,
+            responseId: response.id,
+            explanation: scorerResult.explanation,
+            scorerAIInputCost: scorerResult.inputCost,
+            scorerAIOutputCost: scorerResult.outputCost,
+            scorerAIInputTokensUsed: scorerResult.inputTokensUsed,
+            scorerAIOutputTokensUsed: scorerResult.outputTokensUsed,
+            scorerAIProvider: scorerResult.provider,
+            scorerAIModelSlug: scorerResult.modelSlug,
+            scorerAISystemPromptId: params.llmJudgeSystemPrompt?.id,
+            metadata: {
+              ...scorerResult.metadata,
+              extractedFields
+            }
+          },
+          params.idGenerators?.score ?? idGeneratorUUIDv7
+        );
+        return { response, score };
+      }
     }
+    return { response };
   }
-  return { response };
-}
+);
 function formatMCQ(testCase) {
   return `Question: ${testCase.question}
 Options:
@@ -441,6 +290,89 @@ function templateMessages(messages, templateVariables) {
   }
 }
 
+// src/benchmarks/peerbench/qa-runner.ts
+import Handlebars2 from "handlebars";
+var qaRunner = defineRunner(
+  async (params) => {
+    const { testCase, target, scorer } = params;
+    const messages = [];
+    if (params.systemPrompt) {
+      messages.push({
+        role: "system",
+        content: params.systemPrompt.content
+      });
+    }
+    messages.push({
+      role: "user",
+      content: testCase.question
+    });
+    templateMessages2(messages, params.templateVariables ?? {});
+    const providerResponse = await target.forward({ messages });
+    const response = await QAResponseSchemaV1.newWithId(
+      {
+        data: providerResponse.data,
+        startedAt: providerResponse.startedAt,
+        completedAt: providerResponse.completedAt,
+        testCaseId: testCase.id,
+        modelSlug: target.slug,
+        provider: target.provider.kind,
+        systemPromptId: params.systemPrompt?.id,
+        inputTokensUsed: providerResponse.inputTokensUsed,
+        outputTokensUsed: providerResponse.outputTokensUsed,
+        inputCost: providerResponse.inputCost,
+        outputCost: providerResponse.outputCost
+      },
+      params.idGenerators?.response ?? idGeneratorUUIDv7
+    );
+    if (scorer?.kind === `${PEERBENCH_NAMESPACE}/llm-as-a-judge`) {
+      const scorerResult = await scorer.score({
+        response: response.data,
+        rubric: `Expected/Valid answers: ${testCase.goodAnswers.join("\n")}
+Invalid answers: ${testCase.badAnswers.join("\n")}`,
+        systemPrompt: params.llmJudgeSystemPrompt?.content,
+        criteria: [
+          {
+            id: "correctness",
+            description: "Is the response matches with the expected/valid answers in terms of meaning?",
+            weight: 1
+          }
+        ],
+        fieldsToExtract: params.llmJudgeFieldsToExtract ?? {}
+      });
+      if (scorerResult !== null) {
+        const score = await QAScoreSchemaV1.newWithId(
+          {
+            scoringMethod: ScoringMethod.ai,
+            value: scorerResult.value,
+            responseId: response.id,
+            explanation: scorerResult.explanation,
+            scorerAIInputCost: scorerResult.inputCost,
+            scorerAIOutputCost: scorerResult.outputCost,
+            scorerAIInputTokensUsed: scorerResult.inputTokensUsed,
+            scorerAIOutputTokensUsed: scorerResult.outputTokensUsed,
+            scorerAIProvider: scorerResult.provider,
+            scorerAIModelSlug: scorerResult.modelSlug,
+            scorerAISystemPromptId: params.llmJudgeSystemPrompt?.id,
+            metadata: {
+              ...scorerResult.metadata,
+              extractedFields: scorerResult.extractedFields
+            }
+          },
+          params.idGenerators?.score ?? idGeneratorUUIDv7
+        );
+        return { response, score };
+      }
+    }
+    return { response };
+  }
+);
+function templateMessages2(messages, templateVariables) {
+  for (let i = 0; i < messages.length; i++) {
+    const template = Handlebars2.compile(messages[i].content);
+    messages[i].content = template(templateVariables);
+  }
+}
+
 // src/benchmarks/peerbench/storages/json.ts
 import z5 from "zod";
 var PeerbenchJSONStorage = class extends JSONFileStorage {

package/dist/benchmarks/index.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/benchmarks/peerbench/index.ts","../../src/benchmarks/peerbench/schema-sets/mcq.v1.ts","../../src/benchmarks/peerbench/schema-sets/multi-turn.v1.ts","../../src/benchmarks/peerbench/schema-sets/qa.v1.ts","../../src/benchmarks/peerbench/runner.ts","../../src/benchmarks/peerbench/storages/json.ts"],"sourcesContent":["export * from \"./schema-sets/mcq.v1\";\nexport * from \"./schema-sets/multi-turn.v1\";\nexport * from \"./schema-sets/qa.v1\";\n\nexport * from \"./runner\";\n\nexport * from \"./storages/json\";\n","import { PEERBENCH_NAMESPACE } from \"@/constants\";\nimport {\n  BaseResponseSchemaV1,\n  BaseScoreSchemaV1,\n  BaseTestCaseSchemaV1,\n  defineResponseSchema,\n  defineScoreSchema,\n  defineTestCaseSchema,\n} from \"@/schemas\";\nimport { ExtensionLLMResponseFieldsV1 } from \"@/schemas/extensions/response/llm\";\nimport { ExtensionLLMAsAJudgeScoreFieldsV1 } from \"@/schemas/extensions/score/llm-as-a-judge-scorer\";\nimport { z } from \"zod\";\n\nexport const MCQKind = `llm/mcq` as const;\n\nexport const MCQTestCaseSchemaV1 = defineTestCaseSchema({\n  baseSchema: BaseTestCaseSchemaV1,\n  namespace: PEERBENCH_NAMESPACE,\n  kind: MCQKind,\n  schemaVersion: 1,\n  fields: {\n    question: z.string(),\n    options: z.record(z.string(), z.string()),\n    correctAnswerKeys: z.string().array(),\n  },\n});\nexport type MCQTestCaseV1 = z.infer<typeof MCQTestCaseSchemaV1>;\n\nexport const MCQResponseSchemaV1 = defineResponseSchema({\n  baseSchema: BaseResponseSchemaV1,\n  namespace: PEERBENCH_NAMESPACE,\n  kind: MCQKind,\n  schemaVersion: 1,\n  fields: {\n    ...ExtensionLLMResponseFieldsV1,\n  },\n});\nexport type MCQResponseV1 = z.infer<typeof MCQResponseSchemaV1>;\n\nexport const MCQScoreSchemaV1 = defineScoreSchema({\n  baseSchema: BaseScoreSchemaV1,\n  namespace: PEERBENCH_NAMESPACE,\n  kind: MCQKind,\n  schemaVersion: 1,\n  fields: {\n    ...ExtensionLLMAsAJudgeScoreFieldsV1,\n    extractedAnswers: z.array(z.string()),\n  },\n});\nexport type MCQScoreV1 = z.infer<typeof MCQScoreSchemaV1>;\n","import { PEERBENCH_NAMESPACE } from \"@/constants\";\nimport {\n  BaseResponseSchemaV1,\n  BaseScoreSchemaV1,\n  BaseTestCaseSchemaV1,\n  defineResponseSchema,\n  defineScoreSchema,\n  defineTestCaseSchema,\n} from \"@/schemas\";\nimport { ExtensionLLMResponseFieldsV1 } from \"@/schemas/extensions/response/llm\";\nimport { ExtensionLLMAsAJudgeScoreFieldsV1 } from \"@/schemas/extensions/score/llm-as-a-judge-scorer\";\nimport { z } from \"zod\";\n\nexport const MultiTurnKind = `llm/multi-turn` as const;\n\nexport const MultiTurnTestCaseSchemaV1 = defineTestCaseSchema({\n  baseSchema: BaseTestCaseSchemaV1,\n  namespace: PEERBENCH_NAMESPACE,\n  kind: MultiTurnKind,\n  schemaVersion: 1,\n  fields: {\n    messages: z\n      .object({\n        role: z.string(),\n        content: z.string(),\n        goodAnswers: z.string().array().optional(),\n        badAnswers: z.string().array().optional(),\n      })\n      .array(),\n\n    maxTurns: z.number().optional(),\n    expectedOutcome: z.string().optional(),\n  },\n});\nexport type MultiTurnTestCaseV1 = z.infer<typeof MultiTurnTestCaseSchemaV1>;\n\nexport const MultiTurnResponseSchemaV1 = defineResponseSchema({\n  baseSchema: BaseResponseSchemaV1,\n  namespace: PEERBENCH_NAMESPACE,\n  kind: MultiTurnKind,\n  schemaVersion: 1,\n  fields: {\n    ...ExtensionLLMResponseFieldsV1,\n    replies: z\n      .object({\n        messageIndex: z.number(),\n        startedAt: z.number(),\n        completedAt: z.number(),\n        data: z.string(),\n\n        inputTokensUsed: z.number().optional(),\n        outputTokensUsed: z.number().optional(),\n        inputCost: z.string().optional(),\n        outputCost: z.string().optional(),\n      })\n      .array(),\n  },\n});\nexport type MultiTurnResponseV1 = z.infer<typeof MultiTurnResponseSchemaV1>;\n\nexport const MultiTurnScoreSchemaV1 = defineScoreSchema({\n  baseSchema: BaseScoreSchemaV1,\n  namespace: PEERBENCH_NAMESPACE,\n  kind: MultiTurnKind,\n  schemaVersion: 1,\n  fields: {\n    ...ExtensionLLMAsAJudgeScoreFieldsV1,\n    individualScores: z\n      .object({\n        replyIndex: z.number(),\n        value: z.number(),\n      })\n      .array(),\n  },\n});\nexport type MultiTurnScoreV1 = z.infer<typeof MultiTurnScoreSchemaV1>;\n","import { PEERBENCH_NAMESPACE } from \"@/constants\";\nimport {\n  BaseResponseSchemaV1,\n  BaseScoreSchemaV1,\n  BaseTestCaseSchemaV1,\n  defineResponseSchema,\n  defineScoreSchema,\n  defineTestCaseSchema,\n} from \"@/schemas\";\nimport { ExtensionLLMResponseFieldsV1 } from \"@/schemas/extensions/response/llm\";\nimport { ExtensionLLMAsAJudgeScoreFieldsV1 } from \"@/schemas/extensions/score/llm-as-a-judge-scorer\";\nimport { z } from \"zod\";\n\nexport const QAKind = `llm/qa` as const;\n\nexport const QATestCaseSchemaV1 = defineTestCaseSchema({\n  baseSchema: BaseTestCaseSchemaV1,\n  namespace: PEERBENCH_NAMESPACE,\n  kind: QAKind,\n  schemaVersion: 1,\n  fields: {\n    question: z.string(),\n    goodAnswers: z.string().array(),\n    badAnswers: z.string().array(),\n  },\n});\nexport type QATestCaseV1 = z.infer<typeof QATestCaseSchemaV1>;\n\nexport const QAResponseSchemaV1 = defineResponseSchema({\n  baseSchema: BaseResponseSchemaV1,\n  namespace: PEERBENCH_NAMESPACE,\n  kind: QAKind,\n  schemaVersion: 1,\n  fields: {\n    ...ExtensionLLMResponseFieldsV1,\n  },\n});\nexport type QAResponseV1 = z.infer<typeof QAResponseSchemaV1>;\n\nexport const QAScoreSchemaV1 = defineScoreSchema({\n  baseSchema: BaseScoreSchemaV1,\n  namespace: PEERBENCH_NAMESPACE,\n  kind: QAKind,\n  schemaVersion: 1,\n  fields: {\n    ...ExtensionLLMAsAJudgeScoreFieldsV1,\n  },\n});\nexport type QAScoreV1 = z.infer<typeof QAScoreSchemaV1>;\n","import { defineRunner } from \"@/helpers/define-runner\";\nimport { AbstractLLMProvider } from \"@/providers\";\nimport {\n  SimpleSystemPromptSchemaV1,\n  SimpleSystemPromptV1,\n} from \"@/schemas/llm\";\nimport { LLMAsAJudgeScorer, MCQScorer } from \"@/scorers\";\nimport { IdGenerator, ScoringMethod } from \"@/types\";\nimport { idGeneratorUUIDv7 } from \"@/utils\";\nimport { ChatCompletionMessageParam } from \"openai/resources/index\";\nimport Handlebars from \"handlebars\";\nimport z from \"zod\";\nimport {\n  MCQResponseSchemaV1,\n  MCQScoreSchemaV1,\n  MCQTestCaseSchemaV1,\n  MCQTestCaseV1,\n} from \"./schema-sets/mcq.v1\";\nimport {\n  QAResponseSchemaV1,\n  QAScoreSchemaV1,\n  QATestCaseSchemaV1,\n  QATestCaseV1,\n} from \"./schema-sets/qa.v1\";\nimport { PEERBENCH_NAMESPACE } from \"@/constants\";\n\nexport const peerbenchRunner = defineRunner(\n  {\n    schemaSets: [\n      {\n        testCase: MCQTestCaseSchemaV1,\n        response: MCQResponseSchemaV1,\n        score: MCQScoreSchemaV1,\n      },\n      {\n        testCase: QATestCaseSchemaV1,\n        response: QAResponseSchemaV1,\n        score: QAScoreSchemaV1,\n      },\n    ],\n    providers: [AbstractLLMProvider],\n    scorers: [LLMAsAJudgeScorer, MCQScorer],\n\n    runConfigSchema: {\n      model: z.string(),\n      llmJudgeModel: z.string().optional(),\n      llmJudgeSystemPrompt: SimpleSystemPromptSchemaV1.optional(),\n      llmJudgeFieldsToExtract: z\n        .record(z.string(), z.custom<z.ZodType>())\n        .optional(),\n      systemPrompt: SimpleSystemPromptSchemaV1.optional(),\n      templateVariables: z.record(z.string(), z.string()).optional(),\n    },\n  },\n  async (params) => {\n    const { testCase, provider, scorer, runConfig } = params;\n    const messages: ChatCompletionMessageParam[] = [];\n\n    if (runConfig.systemPrompt) {\n      messages.push({\n        role: \"system\",\n        content: runConfig.systemPrompt.content,\n      });\n    }\n\n    if (testCase.kind === \"llm/mcq.tc\") {\n      messages.push({\n        role: \"user\",\n        content: formatMCQ(testCase),\n      });\n      templateMessages(messages, runConfig.templateVariables ?? {});\n\n      return runMCQ({\n        testCase,\n        messages,\n        provider,\n        scorer,\n        runConfig,\n        idGenerators: {\n          response: params.idGenerators?.response ?? idGeneratorUUIDv7,\n          score: params.idGenerators?.score ?? idGeneratorUUIDv7,\n        },\n      });\n    }\n\n    if (testCase.kind === \"llm/qa.tc\") {\n      if (\n        scorer &&\n        scorer?.kind !== (`${PEERBENCH_NAMESPACE}/llm-as-a-judge` as const)\n      ) {\n        throw new Error(\n          `QA test cases can only be scored with an LLM as a judge scorer, but ${scorer?.kind} was provided`\n        );\n      }\n\n      messages.push({\n        role: \"user\",\n        content: testCase.question,\n      });\n      templateMessages(messages, runConfig.templateVariables ?? {});\n\n      return runQA({\n        testCase,\n        messages,\n        provider,\n        scorer,\n        runConfig,\n        idGenerators: {\n          response: params.idGenerators?.response ?? idGeneratorUUIDv7,\n          score: params.idGenerators?.score ?? idGeneratorUUIDv7,\n        },\n      });\n    }\n\n    throw new Error(\"Unsupported test case kind\");\n  }\n);\n\nasync function runQA(params: {\n  messages: ChatCompletionMessageParam[];\n  testCase: QATestCaseV1;\n  provider: AbstractLLMProvider;\n  scorer?: LLMAsAJudgeScorer;\n  runConfig: {\n    model: string;\n    llmJudgeModel?: string;\n    llmJudgeSystemPrompt?: SimpleSystemPromptV1;\n    llmJudgeFieldsToExtract?: Record<string, z.ZodType>;\n    systemPrompt?: SimpleSystemPromptV1;\n  };\n  idGenerators: {\n    response: IdGenerator;\n    score: IdGenerator;\n  };\n}) {\n  const { messages, testCase, provider, scorer, runConfig } = params;\n\n  const providerResponse = await provider.forward({\n    model: runConfig.model,\n    messages,\n  });\n\n  const response = await QAResponseSchemaV1.newWithId(\n    {\n      data: providerResponse.data,\n      startedAt: providerResponse.startedAt,\n      completedAt: providerResponse.completedAt,\n      testCaseId: testCase.id,\n      modelSlug: runConfig.model,\n      provider: provider.kind,\n      systemPromptId: runConfig.systemPrompt?.id,\n\n      inputTokensUsed: providerResponse.inputTokensUsed,\n      outputTokensUsed: providerResponse.outputTokensUsed,\n      inputCost: providerResponse.inputCost,\n      outputCost: providerResponse.outputCost,\n    },\n    params.idGenerators?.response ?? idGeneratorUUIDv7\n  );\n\n  if (scorer?.kind === (`${PEERBENCH_NAMESPACE}/llm-as-a-judge` as const)) {\n    if (!runConfig.llmJudgeModel) {\n      throw new Error(\n        \"LLM judge model is required when using LLM as a judge scorer\"\n      );\n    }\n\n    const scorerResult = await scorer.score({\n      model: runConfig.llmJudgeModel,\n      response: response.data,\n      rubric: `Expected/Valid answers: ${testCase.goodAnswers.join(\"\\n\")}\\nInvalid answers: ${testCase.badAnswers.join(\"\\n\")}`,\n      systemPrompt: runConfig.llmJudgeSystemPrompt?.content,\n      criteria: [\n        {\n          id: \"correctness\",\n          description:\n            \"Is the response matches with the expected/valid answers in terms of meaning?\",\n          weight: 1,\n        },\n      ],\n      fieldsToExtract: runConfig.llmJudgeFieldsToExtract ?? {},\n    });\n\n    if (scorerResult !== null) {\n      const score = await QAScoreSchemaV1.newWithId(\n        {\n          scoringMethod: ScoringMethod.ai,\n          value: scorerResult.value,\n          responseId: response.id,\n          explanation: scorerResult.explanation,\n          scorerAIInputCost: scorerResult.inputCost,\n          scorerAIOutputCost: scorerResult.outputCost,\n          scorerAIInputTokensUsed: scorerResult.inputTokensUsed,\n          scorerAIOutputTokensUsed: scorerResult.outputTokensUsed,\n          scorerAIProvider: scorerResult.provider,\n          scorerAIModelSlug: runConfig.llmJudgeModel,\n          scorerAISystemPromptId: runConfig.llmJudgeSystemPrompt?.id,\n          metadata: {\n            ...scorerResult.metadata,\n            extractedFields: scorerResult.extractedFields,\n          },\n        },\n        params.idGenerators?.score ?? idGeneratorUUIDv7\n      );\n\n      return { response, score };\n    }\n  }\n\n  return { response };\n}\n\nasync function runMCQ(params: {\n  messages: ChatCompletionMessageParam[];\n  testCase: MCQTestCaseV1;\n  provider: AbstractLLMProvider;\n  scorer?: MCQScorer | LLMAsAJudgeScorer;\n  runConfig: {\n    model: string;\n    llmJudgeModel?: string;\n    llmJudgeSystemPrompt?: SimpleSystemPromptV1;\n    llmJudgeFieldsToExtract?: Record<string, z.ZodType>;\n    systemPrompt?: SimpleSystemPromptV1;\n  };\n  idGenerators: {\n    response: IdGenerator;\n    score: IdGenerator;\n  };\n}) {\n  const { messages, testCase, provider, scorer, runConfig } = params;\n\n  const providerResponse = await provider.forward({\n    model: runConfig.model,\n    messages,\n  });\n\n  const response = await MCQResponseSchemaV1.newWithId(\n    {\n      data: providerResponse.data,\n      startedAt: providerResponse.startedAt,\n      completedAt: providerResponse.completedAt,\n      testCaseId: testCase.id,\n      modelSlug: runConfig.model,\n      provider: provider.kind,\n      systemPromptId: runConfig.systemPrompt?.id,\n\n      inputTokensUsed: providerResponse.inputTokensUsed,\n      outputTokensUsed: providerResponse.outputTokensUsed,\n      inputCost: providerResponse.inputCost,\n      outputCost: providerResponse.outputCost,\n    },\n    params.idGenerators?.response ?? idGeneratorUUIDv7\n  );\n\n  if (scorer?.kind === (`${PEERBENCH_NAMESPACE}/mcq` as const)) {\n    const scorerResult = await scorer.score({\n      response: response.data,\n      choices: testCase.options,\n      correctAnswers: testCase.correctAnswerKeys,\n    });\n\n    if (scorerResult !== null) {\n      const score = await MCQScoreSchemaV1.newWithId(\n        {\n          scoringMethod: ScoringMethod.algo,\n          value: scorerResult.value,\n          responseId: response.id,\n          extractedAnswers: scorerResult.extractedAnswers,\n          explanation: scorerResult.explanation,\n          metadata: scorerResult.metadata,\n        },\n        params.idGenerators?.score ?? idGeneratorUUIDv7\n      );\n\n      return { response, score };\n    }\n  }\n\n  if (scorer?.kind === (`${PEERBENCH_NAMESPACE}/llm-as-a-judge` as const)) {\n    if (!runConfig.llmJudgeModel) {\n      throw new Error(\n        \"LLM judge model is required when using LLM as a judge scorer\"\n      );\n    }\n\n    const scorerResult = await scorer.score({\n      model: runConfig.llmJudgeModel,\n      criteria: [\n        {\n          id: \"correctness\",\n          description:\n            \"Is the given answer key matches with one of the correct answer keys?\",\n          weight: 1,\n        },\n      ],\n      rubric: `Answer text itself or the key (A, B, C) is accepted\nValid answer keys: ${testCase.correctAnswerKeys.map((key) => `- ${key}`).join(\"\\n\")}\nValid Answer texts: ${testCase.correctAnswerKeys.map((key) => `- ${testCase.options?.[key] ?? \"\"}`).join(\"\\n\")}`,\n      fieldsToExtract: {\n        extractedAnswers: z\n          .string()\n          .array()\n          .describe(\n            \"The extracted answer keys, valid or invalid (even if the answer text is provided rather than the key)\"\n          ),\n        ...(runConfig.llmJudgeFieldsToExtract ?? {}),\n      },\n      response: response.data,\n      systemPrompt: runConfig.llmJudgeSystemPrompt?.content,\n    });\n\n    if (scorerResult !== null) {\n      const { extractedAnswers, ...extractedFields } =\n        scorerResult.extractedFields;\n      const score = await MCQScoreSchemaV1.newWithId(\n        {\n          scoringMethod: ScoringMethod.ai,\n          value: scorerResult.value,\n          extractedAnswers,\n          responseId: response.id,\n          explanation: scorerResult.explanation,\n          scorerAIInputCost: scorerResult.inputCost,\n          scorerAIOutputCost: scorerResult.outputCost,\n          scorerAIInputTokensUsed: scorerResult.inputTokensUsed,\n          scorerAIOutputTokensUsed: scorerResult.outputTokensUsed,\n          scorerAIProvider: scorerResult.provider,\n          scorerAIModelSlug: runConfig.llmJudgeModel,\n          scorerAISystemPromptId: runConfig.llmJudgeSystemPrompt?.id,\n          metadata: {\n            ...scorerResult.metadata,\n            extractedFields,\n          },\n        },\n        params.idGenerators?.score ?? idGeneratorUUIDv7\n      );\n\n      return { response, score };\n    }\n  }\n\n  return { response };\n}\n\nfunction formatMCQ(testCase: MCQTestCaseV1) {\n  return `Question: ${testCase.question}\\nOptions:\\n${Object.entries(\n    testCase.options ?? {}\n  )\n    .map(([key, value]) => `${key}: ${value}`)\n    .join(\"\\n\")}`;\n}\n\nfunction templateMessages(\n  messages: ChatCompletionMessageParam[],\n  templateVariables: Record<string, string>\n) {\n  for (let i = 0; i < messages.length; i++) {\n    const template = Handlebars.compile(messages[i]!.content);\n    messages[i]!.content = template(templateVariables);\n  }\n}\n","import { JSONFileStorage } from \"@/storages/json-file\";\nimport {\n  MCQResponseSchemaV1,\n  MCQResponseV1,\n  MCQScoreSchemaV1,\n  MCQScoreV1,\n  MCQTestCaseSchemaV1,\n  MCQTestCaseV1,\n} from \"../schema-sets/mcq.v1\";\nimport {\n  QAResponseSchemaV1,\n  QAResponseV1,\n  QAScoreSchemaV1,\n  QAScoreV1,\n  QATestCaseSchemaV1,\n  QATestCaseV1,\n} from \"../schema-sets/qa.v1\";\nimport {\n  MultiTurnResponseSchemaV1,\n  MultiTurnResponseV1,\n  MultiTurnScoreSchemaV1,\n  MultiTurnScoreV1,\n  MultiTurnTestCaseSchemaV1,\n  MultiTurnTestCaseV1,\n} from \"../schema-sets/multi-turn.v1\";\nimport z from \"zod\";\n\nexport class PeerbenchJSONStorage extends JSONFileStorage<\n  | MCQTestCaseV1\n  | MCQResponseV1\n  | MCQScoreV1\n  | QATestCaseV1\n  | QAResponseV1\n  | QAScoreV1\n  | MultiTurnTestCaseV1\n  | MultiTurnResponseV1\n  | MultiTurnScoreV1\n> {\n  constructor(config: { path: string; chunkSize?: number }) {\n    super({\n      path: config.path,\n      chunkSize: config.chunkSize,\n\n      schema: z.union([\n        MCQTestCaseSchemaV1,\n        MCQResponseSchemaV1,\n        MCQScoreSchemaV1,\n        QATestCaseSchemaV1,\n        QAResponseSchemaV1,\n        QAScoreSchemaV1,\n        MultiTurnTestCaseSchemaV1,\n        MultiTurnResponseSchemaV1,\n        MultiTurnScoreSchemaV1,\n      ]),\n    });\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;;;ACWA,SAAS,SAAS;AAEX,IAAM,UAAU;AAEhB,IAAM,sBAAsB,qBAAqB;AAAA,EACtD,YAAY;AAAA,EACZ,WAAW;AAAA,EACX,MAAM;AAAA,EACN,eAAe;AAAA,EACf,QAAQ;AAAA,IACN,UAAU,EAAE,OAAO;AAAA,IACnB,SAAS,EAAE,OAAO,EAAE,OAAO,GAAG,EAAE,OAAO,CAAC;AAAA,IACxC,mBAAmB,EAAE,OAAO,EAAE,MAAM;AAAA,EACtC;AACF,CAAC;AAGM,IAAM,sBAAsB,qBAAqB;AAAA,EACtD,YAAY;AAAA,EACZ,WAAW;AAAA,EACX,MAAM;AAAA,EACN,eAAe;AAAA,EACf,QAAQ;AAAA,IACN,GAAG;AAAA,EACL;AACF,CAAC;AAGM,IAAM,mBAAmB,kBAAkB;AAAA,EAChD,YAAY;AAAA,EACZ,WAAW;AAAA,EACX,MAAM;AAAA,EACN,eAAe;AAAA,EACf,QAAQ;AAAA,IACN,GAAG;AAAA,IACH,kBAAkB,EAAE,MAAM,EAAE,OAAO,CAAC;AAAA,EACtC;AACF,CAAC;;;ACrCD,SAAS,KAAAA,UAAS;AAEX,IAAM,gBAAgB;AAEtB,IAAM,4BAA4B,qBAAqB;AAAA,EAC5D,YAAY;AAAA,EACZ,WAAW;AAAA,EACX,MAAM;AAAA,EACN,eAAe;AAAA,EACf,QAAQ;AAAA,IACN,UAAUA,GACP,OAAO;AAAA,MACN,MAAMA,GAAE,OAAO;AAAA,MACf,SAASA,GAAE,OAAO;AAAA,MAClB,aAAaA,GAAE,OAAO,EAAE,MAAM,EAAE,SAAS;AAAA,MACzC,YAAYA,GAAE,OAAO,EAAE,MAAM,EAAE,SAAS;AAAA,IAC1C,CAAC,EACA,MAAM;AAAA,IAET,UAAUA,GAAE,OAAO,EAAE,SAAS;AAAA,IAC9B,iBAAiBA,GAAE,OAAO,EAAE,SAAS;AAAA,EACvC;AACF,CAAC;AAGM,IAAM,4BAA4B,qBAAqB;AAAA,EAC5D,YAAY;AAAA,EACZ,WAAW;AAAA,EACX,MAAM;AAAA,EACN,eAAe;AAAA,EACf,QAAQ;AAAA,IACN,GAAG;AAAA,IACH,SAASA,GACN,OAAO;AAAA,MACN,cAAcA,GAAE,OAAO;AAAA,MACvB,WAAWA,GAAE,OAAO;AAAA,MACpB,aAAaA,GAAE,OAAO;AAAA,MACtB,MAAMA,GAAE,OAAO;AAAA,MAEf,iBAAiBA,GAAE,OAAO,EAAE,SAAS;AAAA,MACrC,kBAAkBA,GAAE,OAAO,EAAE,SAAS;AAAA,MACtC,WAAWA,GAAE,OAAO,EAAE,SAAS;AAAA,MAC/B,YAAYA,GAAE,OAAO,EAAE,SAAS;AAAA,IAClC,CAAC,EACA,MAAM;AAAA,EACX;AACF,CAAC;AAGM,IAAM,yBAAyB,kBAAkB;AAAA,EACtD,YAAY;AAAA,EACZ,WAAW;AAAA,EACX,MAAM;AAAA,EACN,eAAe;AAAA,EACf,QAAQ;AAAA,IACN,GAAG;AAAA,IACH,kBAAkBA,GACf,OAAO;AAAA,MACN,YAAYA,GAAE,OAAO;AAAA,MACrB,OAAOA,GAAE,OAAO;AAAA,IAClB,CAAC,EACA,MAAM;AAAA,EACX;AACF,CAAC;;;AC/DD,SAAS,KAAAC,UAAS;AAEX,IAAM,SAAS;AAEf,IAAM,qBAAqB,qBAAqB;AAAA,EACrD,YAAY;AAAA,EACZ,WAAW;AAAA,EACX,MAAM;AAAA,EACN,eAAe;AAAA,EACf,QAAQ;AAAA,IACN,UAAUA,GAAE,OAAO;AAAA,IACnB,aAAaA,GAAE,OAAO,EAAE,MAAM;AAAA,IAC9B,YAAYA,GAAE,OAAO,EAAE,MAAM;AAAA,EAC/B;AACF,CAAC;AAGM,IAAM,qBAAqB,qBAAqB;AAAA,EACrD,YAAY;AAAA,EACZ,WAAW;AAAA,EACX,MAAM;AAAA,EACN,eAAe;AAAA,EACf,QAAQ;AAAA,IACN,GAAG;AAAA,EACL;AACF,CAAC;AAGM,IAAM,kBAAkB,kBAAkB;AAAA,EAC/C,YAAY;AAAA,EACZ,WAAW;AAAA,EACX,MAAM;AAAA,EACN,eAAe;AAAA,EACf,QAAQ;AAAA,IACN,GAAG;AAAA,EACL;AACF,CAAC;;;ACrCD,OAAO,gBAAgB;AACvB,OAAOC,QAAO;AAeP,IAAM,kBAAkB;AAAA,EAC7B;AAAA,IACE,YAAY;AAAA,MACV;AAAA,QACE,UAAU;AAAA,QACV,UAAU;AAAA,QACV,OAAO;AAAA,MACT;AAAA,MACA;AAAA,QACE,UAAU;AAAA,QACV,UAAU;AAAA,QACV,OAAO;AAAA,MACT;AAAA,IACF;AAAA,IACA,WAAW,CAAC,mBAAmB;AAAA,IAC/B,SAAS,CAAC,mBAAmB,SAAS;AAAA,IAEtC,iBAAiB;AAAA,MACf,OAAOC,GAAE,OAAO;AAAA,MAChB,eAAeA,GAAE,OAAO,EAAE,SAAS;AAAA,MACnC,sBAAsB,2BAA2B,SAAS;AAAA,MAC1D,yBAAyBA,GACtB,OAAOA,GAAE,OAAO,GAAGA,GAAE,OAAkB,CAAC,EACxC,SAAS;AAAA,MACZ,cAAc,2BAA2B,SAAS;AAAA,MAClD,mBAAmBA,GAAE,OAAOA,GAAE,OAAO,GAAGA,GAAE,OAAO,CAAC,EAAE,SAAS;AAAA,IAC/D;AAAA,EACF;AAAA,EACA,OAAO,WAAW;AAChB,UAAM,EAAE,UAAU,UAAU,QAAQ,UAAU,IAAI;AAClD,UAAM,WAAyC,CAAC;AAEhD,QAAI,UAAU,cAAc;AAC1B,eAAS,KAAK;AAAA,QACZ,MAAM;AAAA,QACN,SAAS,UAAU,aAAa;AAAA,MAClC,CAAC;AAAA,IACH;AAEA,QAAI,SAAS,SAAS,cAAc;AAClC,eAAS,KAAK;AAAA,QACZ,MAAM;AAAA,QACN,SAAS,UAAU,QAAQ;AAAA,MAC7B,CAAC;AACD,uBAAiB,UAAU,UAAU,qBAAqB,CAAC,CAAC;AAE5D,aAAO,OAAO;AAAA,QACZ;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA,cAAc;AAAA,UACZ,UAAU,OAAO,cAAc,YAAY;AAAA,UAC3C,OAAO,OAAO,cAAc,SAAS;AAAA,QACvC;AAAA,MACF,CAAC;AAAA,IACH;AAEA,QAAI,SAAS,SAAS,aAAa;AACjC,UACE,UACA,QAAQ,SAAU,GAAG,mBAAmB,mBACxC;AACA,cAAM,IAAI;AAAA,UACR,uEAAuE,QAAQ,IAAI;AAAA,QACrF;AAAA,MACF;AAEA,eAAS,KAAK;AAAA,QACZ,MAAM;AAAA,QACN,SAAS,SAAS;AAAA,MACpB,CAAC;AACD,uBAAiB,UAAU,UAAU,qBAAqB,CAAC,CAAC;AAE5D,aAAO,MAAM;AAAA,QACX;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA,cAAc;AAAA,UACZ,UAAU,OAAO,cAAc,YAAY;AAAA,UAC3C,OAAO,OAAO,cAAc,SAAS;AAAA,QACvC;AAAA,MACF,CAAC;AAAA,IACH;AAEA,UAAM,IAAI,MAAM,4BAA4B;AAAA,EAC9C;AACF;AAEA,eAAe,MAAM,QAgBlB;AACD,QAAM,EAAE,UAAU,UAAU,UAAU,QAAQ,UAAU,IAAI;AAE5D,QAAM,mBAAmB,MAAM,SAAS,QAAQ;AAAA,IAC9C,OAAO,UAAU;AAAA,IACjB;AAAA,EACF,CAAC;AAED,QAAM,WAAW,MAAM,mBAAmB;AAAA,IACxC;AAAA,MACE,MAAM,iBAAiB;AAAA,MACvB,WAAW,iBAAiB;AAAA,MAC5B,aAAa,iBAAiB;AAAA,MAC9B,YAAY,SAAS;AAAA,MACrB,WAAW,UAAU;AAAA,MACrB,UAAU,SAAS;AAAA,MACnB,gBAAgB,UAAU,cAAc;AAAA,MAExC,iBAAiB,iBAAiB;AAAA,MAClC,kBAAkB,iBAAiB;AAAA,MACnC,WAAW,iBAAiB;AAAA,MAC5B,YAAY,iBAAiB;AAAA,IAC/B;AAAA,IACA,OAAO,cAAc,YAAY;AAAA,EACnC;AAEA,MAAI,QAAQ,SAAU,GAAG,mBAAmB,mBAA6B;AACvE,QAAI,CAAC,UAAU,eAAe;AAC5B,YAAM,IAAI;AAAA,QACR;AAAA,MACF;AAAA,IACF;AAEA,UAAM,eAAe,MAAM,OAAO,MAAM;AAAA,MACtC,OAAO,UAAU;AAAA,MACjB,UAAU,SAAS;AAAA,MACnB,QAAQ,2BAA2B,SAAS,YAAY,KAAK,IAAI,CAAC;AAAA,mBAAsB,SAAS,WAAW,KAAK,IAAI,CAAC;AAAA,MACtH,cAAc,UAAU,sBAAsB;AAAA,MAC9C,UAAU;AAAA,QACR;AAAA,UACE,IAAI;AAAA,UACJ,aACE;AAAA,UACF,QAAQ;AAAA,QACV;AAAA,MACF;AAAA,MACA,iBAAiB,UAAU,2BAA2B,CAAC;AAAA,IACzD,CAAC;AAED,QAAI,iBAAiB,MAAM;AACzB,YAAM,QAAQ,MAAM,gBAAgB;AAAA,QAClC;AAAA,UACE,eAAe,cAAc;AAAA,UAC7B,OAAO,aAAa;AAAA,UACpB,YAAY,SAAS;AAAA,UACrB,aAAa,aAAa;AAAA,UAC1B,mBAAmB,aAAa;AAAA,UAChC,oBAAoB,aAAa;AAAA,UACjC,yBAAyB,aAAa;AAAA,UACtC,0BAA0B,aAAa;AAAA,UACvC,kBAAkB,aAAa;AAAA,UAC/B,mBAAmB,UAAU;AAAA,UAC7B,wBAAwB,UAAU,sBAAsB;AAAA,UACxD,UAAU;AAAA,YACR,GAAG,aAAa;AAAA,YAChB,iBAAiB,aAAa;AAAA,UAChC;AAAA,QACF;AAAA,QACA,OAAO,cAAc,SAAS;AAAA,MAChC;AAEA,aAAO,EAAE,UAAU,MAAM;AAAA,IAC3B;AAAA,EACF;AAEA,SAAO,EAAE,SAAS;AACpB;AAEA,eAAe,OAAO,QAgBnB;AACD,QAAM,EAAE,UAAU,UAAU,UAAU,QAAQ,UAAU,IAAI;AAE5D,QAAM,mBAAmB,MAAM,SAAS,QAAQ;AAAA,IAC9C,OAAO,UAAU;AAAA,IACjB;AAAA,EACF,CAAC;AAED,QAAM,WAAW,MAAM,oBAAoB;AAAA,IACzC;AAAA,MACE,MAAM,iBAAiB;AAAA,MACvB,WAAW,iBAAiB;AAAA,MAC5B,aAAa,iBAAiB;AAAA,MAC9B,YAAY,SAAS;AAAA,MACrB,WAAW,UAAU;AAAA,MACrB,UAAU,SAAS;AAAA,MACnB,gBAAgB,UAAU,cAAc;AAAA,MAExC,iBAAiB,iBAAiB;AAAA,MAClC,kBAAkB,iBAAiB;AAAA,MACnC,WAAW,iBAAiB;AAAA,MAC5B,YAAY,iBAAiB;AAAA,IAC/B;AAAA,IACA,OAAO,cAAc,YAAY;AAAA,EACnC;AAEA,MAAI,QAAQ,SAAU,GAAG,mBAAmB,QAAkB;AAC5D,UAAM,eAAe,MAAM,OAAO,MAAM;AAAA,MACtC,UAAU,SAAS;AAAA,MACnB,SAAS,SAAS;AAAA,MAClB,gBAAgB,SAAS;AAAA,IAC3B,CAAC;AAED,QAAI,iBAAiB,MAAM;AACzB,YAAM,QAAQ,MAAM,iBAAiB;AAAA,QACnC;AAAA,UACE,eAAe,cAAc;AAAA,UAC7B,OAAO,aAAa;AAAA,UACpB,YAAY,SAAS;AAAA,UACrB,kBAAkB,aAAa;AAAA,UAC/B,aAAa,aAAa;AAAA,UAC1B,UAAU,aAAa;AAAA,QACzB;AAAA,QACA,OAAO,cAAc,SAAS;AAAA,MAChC;AAEA,aAAO,EAAE,UAAU,MAAM;AAAA,IAC3B;AAAA,EACF;AAEA,MAAI,QAAQ,SAAU,GAAG,mBAAmB,mBAA6B;AACvE,QAAI,CAAC,UAAU,eAAe;AAC5B,YAAM,IAAI;AAAA,QACR;AAAA,MACF;AAAA,IACF;AAEA,UAAM,eAAe,MAAM,OAAO,MAAM;AAAA,MACtC,OAAO,UAAU;AAAA,MACjB,UAAU;AAAA,QACR;AAAA,UACE,IAAI;AAAA,UACJ,aACE;AAAA,UACF,QAAQ;AAAA,QACV;AAAA,MACF;AAAA,MACA,QAAQ;AAAA,qBACO,SAAS,kBAAkB,IAAI,CAAC,QAAQ,KAAK,GAAG,EAAE,EAAE,KAAK,IAAI,CAAC;AAAA,sBAC7D,SAAS,kBAAkB,IAAI,CAAC,QAAQ,KAAK,SAAS,UAAU,GAAG,KAAK,EAAE,EAAE,EAAE,KAAK,IAAI,CAAC;AAAA,MACxG,iBAAiB;AAAA,QACf,kBAAkBA,GACf,OAAO,EACP,MAAM,EACN;AAAA,UACC;AAAA,QACF;AAAA,QACF,GAAI,UAAU,2BAA2B,CAAC;AAAA,MAC5C;AAAA,MACA,UAAU,SAAS;AAAA,MACnB,cAAc,UAAU,sBAAsB;AAAA,IAChD,CAAC;AAED,QAAI,iBAAiB,MAAM;AACzB,YAAM,EAAE,kBAAkB,GAAG,gBAAgB,IAC3C,aAAa;AACf,YAAM,QAAQ,MAAM,iBAAiB;AAAA,QACnC;AAAA,UACE,eAAe,cAAc;AAAA,UAC7B,OAAO,aAAa;AAAA,UACpB;AAAA,UACA,YAAY,SAAS;AAAA,UACrB,aAAa,aAAa;AAAA,UAC1B,mBAAmB,aAAa;AAAA,UAChC,oBAAoB,aAAa;AAAA,UACjC,yBAAyB,aAAa;AAAA,UACtC,0BAA0B,aAAa;AAAA,UACvC,kBAAkB,aAAa;AAAA,UAC/B,mBAAmB,UAAU;AAAA,UAC7B,wBAAwB,UAAU,sBAAsB;AAAA,UACxD,UAAU;AAAA,YACR,GAAG,aAAa;AAAA,YAChB;AAAA,UACF;AAAA,QACF;AAAA,QACA,OAAO,cAAc,SAAS;AAAA,MAChC;AAEA,aAAO,EAAE,UAAU,MAAM;AAAA,IAC3B;AAAA,EACF;AAEA,SAAO,EAAE,SAAS;AACpB;AAEA,SAAS,UAAU,UAAyB;AAC1C,SAAO,aAAa,SAAS,QAAQ;AAAA;AAAA,EAAe,OAAO;AAAA,IACzD,SAAS,WAAW,CAAC;AAAA,EACvB,EACG,IAAI,CAAC,CAAC,KAAK,KAAK,MAAM,GAAG,GAAG,KAAK,KAAK,EAAE,EACxC,KAAK,IAAI,CAAC;AACf;AAEA,SAAS,iBACP,UACA,mBACA;AACA,WAAS,IAAI,GAAG,IAAI,SAAS,QAAQ,KAAK;AACxC,UAAM,WAAW,WAAW,QAAQ,SAAS,CAAC,EAAG,OAAO;AACxD,aAAS,CAAC,EAAG,UAAU,SAAS,iBAAiB;AAAA,EACnD;AACF;;;AC9UA,OAAOC,QAAO;AAEP,IAAM,uBAAN,cAAmC,gBAUxC;AAAA,EACA,YAAY,QAA8C;AACxD,UAAM;AAAA,MACJ,MAAM,OAAO;AAAA,MACb,WAAW,OAAO;AAAA,MAElB,QAAQA,GAAE,MAAM;AAAA,QACd;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,MACF,CAAC;AAAA,IACH,CAAC;AAAA,EACH;AACF;","names":["z","z","z","z","z"]}
+{"version":3,"sources":["../../src/benchmarks/peerbench/index.ts","../../src/benchmarks/peerbench/schema-sets/mcq.v1.ts","../../src/benchmarks/peerbench/schema-sets/multi-turn.v1.ts","../../src/benchmarks/peerbench/schema-sets/qa.v1.ts","../../src/benchmarks/peerbench/mcq-runner.ts","../../src/benchmarks/peerbench/qa-runner.ts","../../src/benchmarks/peerbench/storages/json.ts"],"sourcesContent":["export * from \"./schema-sets/mcq.v1\";\nexport * from \"./schema-sets/multi-turn.v1\";\nexport * from \"./schema-sets/qa.v1\";\n\nexport * from \"./mcq-runner\";\nexport * from \"./qa-runner\";\n\nexport * from \"./storages/json\";\n","import { PEERBENCH_NAMESPACE } from \"@/constants\";\nimport {\n  BaseResponseSchemaV1,\n  BaseScoreSchemaV1,\n  BaseTestCaseSchemaV1,\n  defineResponseSchema,\n  defineScoreSchema,\n  defineTestCaseSchema,\n} from \"@/schemas\";\nimport { ExtensionLLMResponseFieldsV1 } from \"@/schemas/extensions/response/llm\";\nimport { ExtensionLLMAsAJudgeScoreFieldsV1 } from \"@/schemas/extensions/score/llm-as-a-judge-scorer\";\nimport { z } from \"zod\";\n\nexport const MCQKind = `llm/mcq` as const;\n\nexport const MCQTestCaseSchemaV1 = defineTestCaseSchema({\n  baseSchema: BaseTestCaseSchemaV1,\n  namespace: PEERBENCH_NAMESPACE,\n  kind: MCQKind,\n  schemaVersion: 1,\n  fields: {\n    question: z.string(),\n    options: z.record(z.string(), z.string()),\n    correctAnswerKeys: z.string().array(),\n  },\n});\nexport type MCQTestCaseV1 = z.infer<typeof MCQTestCaseSchemaV1>;\n\nexport const MCQResponseSchemaV1 = defineResponseSchema({\n  baseSchema: BaseResponseSchemaV1,\n  namespace: PEERBENCH_NAMESPACE,\n  kind: MCQKind,\n  schemaVersion: 1,\n  fields: {\n    ...ExtensionLLMResponseFieldsV1,\n  },\n});\nexport type MCQResponseV1 = z.infer<typeof MCQResponseSchemaV1>;\n\nexport const MCQScoreSchemaV1 = defineScoreSchema({\n  baseSchema: BaseScoreSchemaV1,\n  namespace: PEERBENCH_NAMESPACE,\n  kind: MCQKind,\n  schemaVersion: 1,\n  fields: {\n    ...ExtensionLLMAsAJudgeScoreFieldsV1,\n    extractedAnswers: z.array(z.string()),\n  },\n});\nexport type MCQScoreV1 = z.infer<typeof MCQScoreSchemaV1>;\n","import { PEERBENCH_NAMESPACE } from \"@/constants\";\nimport {\n  BaseResponseSchemaV1,\n  BaseScoreSchemaV1,\n  BaseTestCaseSchemaV1,\n  defineResponseSchema,\n  defineScoreSchema,\n  defineTestCaseSchema,\n} from \"@/schemas\";\nimport { ExtensionLLMResponseFieldsV1 } from \"@/schemas/extensions/response/llm\";\nimport { ExtensionLLMAsAJudgeScoreFieldsV1 } from \"@/schemas/extensions/score/llm-as-a-judge-scorer\";\nimport { z } from \"zod\";\n\nexport const MultiTurnKind = `llm/multi-turn` as const;\n\nexport const MultiTurnTestCaseSchemaV1 = defineTestCaseSchema({\n  baseSchema: BaseTestCaseSchemaV1,\n  namespace: PEERBENCH_NAMESPACE,\n  kind: MultiTurnKind,\n  schemaVersion: 1,\n  fields: {\n    messages: z\n      .object({\n        role: z.string(),\n        content: z.string(),\n        goodAnswers: z.string().array().optional(),\n        badAnswers: z.string().array().optional(),\n      })\n      .array(),\n\n    maxTurns: z.number().optional(),\n    expectedOutcome: z.string().optional(),\n  },\n});\nexport type MultiTurnTestCaseV1 = z.infer<typeof MultiTurnTestCaseSchemaV1>;\n\nexport const MultiTurnResponseSchemaV1 = defineResponseSchema({\n  baseSchema: BaseResponseSchemaV1,\n  namespace: PEERBENCH_NAMESPACE,\n  kind: MultiTurnKind,\n  schemaVersion: 1,\n  fields: {\n    ...ExtensionLLMResponseFieldsV1,\n    replies: z\n      .object({\n        messageIndex: z.number(),\n        startedAt: z.number(),\n        completedAt: z.number(),\n        data: z.string(),\n\n        inputTokensUsed: z.number().optional(),\n        outputTokensUsed: z.number().optional(),\n        inputCost: z.string().optional(),\n        outputCost: z.string().optional(),\n      })\n      .array(),\n  },\n});\nexport type MultiTurnResponseV1 = z.infer<typeof MultiTurnResponseSchemaV1>;\n\nexport const MultiTurnScoreSchemaV1 = defineScoreSchema({\n  baseSchema: BaseScoreSchemaV1,\n  namespace: PEERBENCH_NAMESPACE,\n  kind: MultiTurnKind,\n  schemaVersion: 1,\n  fields: {\n    ...ExtensionLLMAsAJudgeScoreFieldsV1,\n    individualScores: z\n      .object({\n        replyIndex: z.number(),\n        value: z.number(),\n      })\n      .array(),\n  },\n});\nexport type MultiTurnScoreV1 = z.infer<typeof MultiTurnScoreSchemaV1>;\n","import { PEERBENCH_NAMESPACE } from \"@/constants\";\nimport {\n  BaseResponseSchemaV1,\n  BaseScoreSchemaV1,\n  BaseTestCaseSchemaV1,\n  defineResponseSchema,\n  defineScoreSchema,\n  defineTestCaseSchema,\n} from \"@/schemas\";\nimport { ExtensionLLMResponseFieldsV1 } from \"@/schemas/extensions/response/llm\";\nimport { ExtensionLLMAsAJudgeScoreFieldsV1 } from \"@/schemas/extensions/score/llm-as-a-judge-scorer\";\nimport { z } from \"zod\";\n\nexport const QAKind = `llm/qa` as const;\n\nexport const QATestCaseSchemaV1 = defineTestCaseSchema({\n  baseSchema: BaseTestCaseSchemaV1,\n  namespace: PEERBENCH_NAMESPACE,\n  kind: QAKind,\n  schemaVersion: 1,\n  fields: {\n    question: z.string(),\n    goodAnswers: z.string().array(),\n    badAnswers: z.string().array(),\n  },\n});\nexport type QATestCaseV1 = z.infer<typeof QATestCaseSchemaV1>;\n\nexport const QAResponseSchemaV1 = defineResponseSchema({\n  baseSchema: BaseResponseSchemaV1,\n  namespace: PEERBENCH_NAMESPACE,\n  kind: QAKind,\n  schemaVersion: 1,\n  fields: {\n    ...ExtensionLLMResponseFieldsV1,\n  },\n});\nexport type QAResponseV1 = z.infer<typeof QAResponseSchemaV1>;\n\nexport const QAScoreSchemaV1 = defineScoreSchema({\n  baseSchema: BaseScoreSchemaV1,\n  namespace: PEERBENCH_NAMESPACE,\n  kind: QAKind,\n  schemaVersion: 1,\n  fields: {\n    ...ExtensionLLMAsAJudgeScoreFieldsV1,\n  },\n});\nexport type QAScoreV1 = z.infer<typeof QAScoreSchemaV1>;\n","import { defineRunner } from \"@/helpers/define-runner\";\nimport { CallableLLM } from \"@/providers\";\nimport { SimpleSystemPromptV1 } from \"@/schemas/llm\";\nimport { LLMAsAJudgeScorer, MCQScorer } from \"@/scorers\";\nimport { IdGenerator, ScoringMethod } from \"@/types\";\nimport { idGeneratorUUIDv7 } from \"@/utils\";\nimport { ChatCompletionMessageParam } from \"openai/resources/index\";\nimport Handlebars from \"handlebars\";\nimport z from \"zod\";\nimport {\n  MCQResponseSchemaV1,\n  MCQScoreSchemaV1,\n  MCQTestCaseV1,\n} from \"./schema-sets/mcq.v1\";\nimport { PEERBENCH_NAMESPACE } from \"@/constants\";\n\nexport const mcqRunner = defineRunner(\n  async (params: {\n    testCase: MCQTestCaseV1;\n    target: CallableLLM;\n    scorer?: MCQScorer | LLMAsAJudgeScorer;\n    systemPrompt?: SimpleSystemPromptV1;\n    llmJudgeSystemPrompt?: SimpleSystemPromptV1;\n    llmJudgeFieldsToExtract?: Record<string, z.ZodType>;\n    templateVariables?: Record<string, string>;\n    idGenerators?: {\n      response?: IdGenerator;\n      score?: IdGenerator;\n    };\n  }) => {\n    const { testCase, target, scorer } = params;\n    const messages: ChatCompletionMessageParam[] = [];\n\n    if (params.systemPrompt) {\n      messages.push({\n        role: \"system\",\n        content: params.systemPrompt.content,\n      });\n    }\n\n    messages.push({\n      role: \"user\",\n      content: formatMCQ(testCase),\n    });\n    templateMessages(messages, params.templateVariables ?? {});\n\n    const providerResponse = await target.forward({ messages });\n\n    const response = await MCQResponseSchemaV1.newWithId(\n      {\n        data: providerResponse.data,\n        startedAt: providerResponse.startedAt,\n        completedAt: providerResponse.completedAt,\n        testCaseId: testCase.id,\n        modelSlug: target.slug,\n        provider: target.provider.kind,\n        systemPromptId: params.systemPrompt?.id,\n        inputTokensUsed: providerResponse.inputTokensUsed,\n        outputTokensUsed: providerResponse.outputTokensUsed,\n        inputCost: providerResponse.inputCost,\n        outputCost: providerResponse.outputCost,\n      },\n      params.idGenerators?.response ?? idGeneratorUUIDv7\n    );\n\n    if (scorer?.kind === (`${PEERBENCH_NAMESPACE}/mcq` as const)) {\n      const scorerResult = await scorer.score({\n        response: response.data,\n        choices: testCase.options,\n        correctAnswers: testCase.correctAnswerKeys,\n      });\n\n      if (scorerResult !== null) {\n        const score = await MCQScoreSchemaV1.newWithId(\n          {\n            scoringMethod: ScoringMethod.algo,\n            value: scorerResult.value,\n            responseId: response.id,\n            extractedAnswers: scorerResult.extractedAnswers,\n            explanation: scorerResult.explanation,\n            metadata: scorerResult.metadata,\n          },\n          params.idGenerators?.score ?? idGeneratorUUIDv7\n        );\n\n        return { response, score };\n      }\n    }\n\n    if (scorer?.kind === (`${PEERBENCH_NAMESPACE}/llm-as-a-judge` as const)) {\n      const scorerResult = await scorer.score({\n        criteria: [\n          {\n            id: \"correctness\",\n            description:\n              \"Is the given answer key matches with one of the correct answer keys?\",\n            weight: 1,\n          },\n        ],\n        rubric: `Answer text itself or the key (A, B, C) is accepted\nValid answer keys: ${testCase.correctAnswerKeys.map((key) => `- ${key}`).join(\"\\n\")}\nValid Answer texts: ${testCase.correctAnswerKeys.map((key) => `- ${testCase.options?.[key] ?? \"\"}`).join(\"\\n\")}`,\n        fieldsToExtract: {\n          extractedAnswers: z\n            .string()\n            .array()\n            .describe(\n              \"The extracted answer keys, valid or invalid (even if the answer text is provided rather than the key)\"\n            ),\n          ...(params.llmJudgeFieldsToExtract ?? {}),\n        },\n        response: response.data,\n        systemPrompt: params.llmJudgeSystemPrompt?.content,\n      });\n\n      if (scorerResult !== null) {\n        const { extractedAnswers, ...extractedFields } =\n          scorerResult.extractedFields;\n        const score = await MCQScoreSchemaV1.newWithId(\n          {\n            scoringMethod: ScoringMethod.ai,\n            value: scorerResult.value,\n            extractedAnswers,\n            responseId: response.id,\n            explanation: scorerResult.explanation,\n            scorerAIInputCost: scorerResult.inputCost,\n            scorerAIOutputCost: scorerResult.outputCost,\n            scorerAIInputTokensUsed: scorerResult.inputTokensUsed,\n            scorerAIOutputTokensUsed: scorerResult.outputTokensUsed,\n            scorerAIProvider: scorerResult.provider,\n            scorerAIModelSlug: scorerResult.modelSlug,\n            scorerAISystemPromptId: params.llmJudgeSystemPrompt?.id,\n            metadata: {\n              ...scorerResult.metadata,\n              extractedFields,\n            },\n          },\n          params.idGenerators?.score ?? idGeneratorUUIDv7\n        );\n\n        return { response, score };\n      }\n    }\n\n    return { response };\n  }\n);\n\nfunction formatMCQ(testCase: MCQTestCaseV1) {\n  return `Question: ${testCase.question}\\nOptions:\\n${Object.entries(\n    testCase.options ?? {}\n  )\n    .map(([key, value]) => `${key}: ${value}`)\n    .join(\"\\n\")}`;\n}\n\nfunction templateMessages(\n  messages: ChatCompletionMessageParam[],\n  templateVariables: Record<string, string>\n) {\n  for (let i = 0; i < messages.length; i++) {\n    const template = Handlebars.compile(messages[i]!.content);\n    messages[i]!.content = template(templateVariables);\n  }\n}\n","import { defineRunner } from \"@/helpers/define-runner\";\nimport { CallableLLM } from \"@/providers\";\nimport { SimpleSystemPromptV1 } from \"@/schemas/llm\";\nimport { LLMAsAJudgeScorer } from \"@/scorers\";\nimport { IdGenerator, ScoringMethod } from \"@/types\";\nimport { idGeneratorUUIDv7 } from \"@/utils\";\nimport { ChatCompletionMessageParam } from \"openai/resources/index\";\nimport Handlebars from \"handlebars\";\nimport z from \"zod\";\nimport {\n  QAResponseSchemaV1,\n  QAScoreSchemaV1,\n  QATestCaseV1,\n} from \"./schema-sets/qa.v1\";\nimport { PEERBENCH_NAMESPACE } from \"@/constants\";\n\nexport const qaRunner = defineRunner(\n  async (params: {\n    testCase: QATestCaseV1;\n    target: CallableLLM;\n    scorer?: LLMAsAJudgeScorer;\n    systemPrompt?: SimpleSystemPromptV1;\n    llmJudgeSystemPrompt?: SimpleSystemPromptV1;\n    llmJudgeFieldsToExtract?: Record<string, z.ZodType>;\n    templateVariables?: Record<string, string>;\n    idGenerators?: {\n      response?: IdGenerator;\n      score?: IdGenerator;\n    };\n  }) => {\n    const { testCase, target, scorer } = params;\n    const messages: ChatCompletionMessageParam[] = [];\n\n    if (params.systemPrompt) {\n      messages.push({\n        role: \"system\",\n        content: params.systemPrompt.content,\n      });\n    }\n\n    messages.push({\n      role: \"user\",\n      content: testCase.question,\n    });\n    templateMessages(messages, params.templateVariables ?? {});\n\n    const providerResponse = await target.forward({ messages });\n\n    const response = await QAResponseSchemaV1.newWithId(\n      {\n        data: providerResponse.data,\n        startedAt: providerResponse.startedAt,\n        completedAt: providerResponse.completedAt,\n        testCaseId: testCase.id,\n        modelSlug: target.slug,\n        provider: target.provider.kind,\n        systemPromptId: params.systemPrompt?.id,\n        inputTokensUsed: providerResponse.inputTokensUsed,\n        outputTokensUsed: providerResponse.outputTokensUsed,\n        inputCost: providerResponse.inputCost,\n        outputCost: providerResponse.outputCost,\n      },\n      params.idGenerators?.response ?? idGeneratorUUIDv7\n    );\n\n    if (scorer?.kind === (`${PEERBENCH_NAMESPACE}/llm-as-a-judge` as const)) {\n      const scorerResult = await scorer.score({\n        response: response.data,\n        rubric: `Expected/Valid answers: ${testCase.goodAnswers.join(\"\\n\")}\\nInvalid answers: ${testCase.badAnswers.join(\"\\n\")}`,\n        systemPrompt: params.llmJudgeSystemPrompt?.content,\n        criteria: [\n          {\n            id: \"correctness\",\n            description:\n              \"Is the response matches with the expected/valid answers in terms of meaning?\",\n            weight: 1,\n          },\n        ],\n        fieldsToExtract: params.llmJudgeFieldsToExtract ?? {},\n      });\n\n      if (scorerResult !== null) {\n        const score = await QAScoreSchemaV1.newWithId(\n          {\n            scoringMethod: ScoringMethod.ai,\n            value: scorerResult.value,\n            responseId: response.id,\n            explanation: scorerResult.explanation,\n            scorerAIInputCost: scorerResult.inputCost,\n            scorerAIOutputCost: scorerResult.outputCost,\n            scorerAIInputTokensUsed: scorerResult.inputTokensUsed,\n            scorerAIOutputTokensUsed: scorerResult.outputTokensUsed,\n            scorerAIProvider: scorerResult.provider,\n            scorerAIModelSlug: scorerResult.modelSlug,\n            scorerAISystemPromptId: params.llmJudgeSystemPrompt?.id,\n            metadata: {\n              ...scorerResult.metadata,\n              extractedFields: scorerResult.extractedFields,\n            },\n          },\n          params.idGenerators?.score ?? idGeneratorUUIDv7\n        );\n\n        return { response, score };\n      }\n    }\n\n    return { response };\n  }\n);\n\nfunction templateMessages(\n  messages: ChatCompletionMessageParam[],\n  templateVariables: Record<string, string>\n) {\n  for (let i = 0; i < messages.length; i++) {\n    const template = Handlebars.compile(messages[i]!.content);\n    messages[i]!.content = template(templateVariables);\n  }\n}\n","import { JSONFileStorage } from \"@/storages/json-file\";\nimport {\n  MCQResponseSchemaV1,\n  MCQResponseV1,\n  MCQScoreSchemaV1,\n  MCQScoreV1,\n  MCQTestCaseSchemaV1,\n  MCQTestCaseV1,\n} from \"../schema-sets/mcq.v1\";\nimport {\n  QAResponseSchemaV1,\n  QAResponseV1,\n  QAScoreSchemaV1,\n  QAScoreV1,\n  QATestCaseSchemaV1,\n  QATestCaseV1,\n} from \"../schema-sets/qa.v1\";\nimport {\n  MultiTurnResponseSchemaV1,\n  MultiTurnResponseV1,\n  MultiTurnScoreSchemaV1,\n  MultiTurnScoreV1,\n  MultiTurnTestCaseSchemaV1,\n  MultiTurnTestCaseV1,\n} from \"../schema-sets/multi-turn.v1\";\nimport z from \"zod\";\n\nexport class PeerbenchJSONStorage extends JSONFileStorage<\n  | MCQTestCaseV1\n  | MCQResponseV1\n  | MCQScoreV1\n  | QATestCaseV1\n  | QAResponseV1\n  | QAScoreV1\n  | MultiTurnTestCaseV1\n  | MultiTurnResponseV1\n  | MultiTurnScoreV1\n> {\n  constructor(config: { path: string; chunkSize?: number }) {\n    super({\n      path: config.path,\n      chunkSize: config.chunkSize,\n\n      schema: z.union([\n        MCQTestCaseSchemaV1,\n        MCQResponseSchemaV1,\n        MCQScoreSchemaV1,\n        QATestCaseSchemaV1,\n        QAResponseSchemaV1,\n        QAScoreSchemaV1,\n        MultiTurnTestCaseSchemaV1,\n        MultiTurnResponseSchemaV1,\n        MultiTurnScoreSchemaV1,\n      ]),\n    });\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;;;ACWA,SAAS,SAAS;AAEX,IAAM,UAAU;AAEhB,IAAM,sBAAsB,qBAAqB;AAAA,EACtD,YAAY;AAAA,EACZ,WAAW;AAAA,EACX,MAAM;AAAA,EACN,eAAe;AAAA,EACf,QAAQ;AAAA,IACN,UAAU,EAAE,OAAO;AAAA,IACnB,SAAS,EAAE,OAAO,EAAE,OAAO,GAAG,EAAE,OAAO,CAAC;AAAA,IACxC,mBAAmB,EAAE,OAAO,EAAE,MAAM;AAAA,EACtC;AACF,CAAC;AAGM,IAAM,sBAAsB,qBAAqB;AAAA,EACtD,YAAY;AAAA,EACZ,WAAW;AAAA,EACX,MAAM;AAAA,EACN,eAAe;AAAA,EACf,QAAQ;AAAA,IACN,GAAG;AAAA,EACL;AACF,CAAC;AAGM,IAAM,mBAAmB,kBAAkB;AAAA,EAChD,YAAY;AAAA,EACZ,WAAW;AAAA,EACX,MAAM;AAAA,EACN,eAAe;AAAA,EACf,QAAQ;AAAA,IACN,GAAG;AAAA,IACH,kBAAkB,EAAE,MAAM,EAAE,OAAO,CAAC;AAAA,EACtC;AACF,CAAC;;;ACrCD,SAAS,KAAAA,UAAS;AAEX,IAAM,gBAAgB;AAEtB,IAAM,4BAA4B,qBAAqB;AAAA,EAC5D,YAAY;AAAA,EACZ,WAAW;AAAA,EACX,MAAM;AAAA,EACN,eAAe;AAAA,EACf,QAAQ;AAAA,IACN,UAAUA,GACP,OAAO;AAAA,MACN,MAAMA,GAAE,OAAO;AAAA,MACf,SAASA,GAAE,OAAO;AAAA,MAClB,aAAaA,GAAE,OAAO,EAAE,MAAM,EAAE,SAAS;AAAA,MACzC,YAAYA,GAAE,OAAO,EAAE,MAAM,EAAE,SAAS;AAAA,IAC1C,CAAC,EACA,MAAM;AAAA,IAET,UAAUA,GAAE,OAAO,EAAE,SAAS;AAAA,IAC9B,iBAAiBA,GAAE,OAAO,EAAE,SAAS;AAAA,EACvC;AACF,CAAC;AAGM,IAAM,4BAA4B,qBAAqB;AAAA,EAC5D,YAAY;AAAA,EACZ,WAAW;AAAA,EACX,MAAM;AAAA,EACN,eAAe;AAAA,EACf,QAAQ;AAAA,IACN,GAAG;AAAA,IACH,SAASA,GACN,OAAO;AAAA,MACN,cAAcA,GAAE,OAAO;AAAA,MACvB,WAAWA,GAAE,OAAO;AAAA,MACpB,aAAaA,GAAE,OAAO;AAAA,MACtB,MAAMA,GAAE,OAAO;AAAA,MAEf,iBAAiBA,GAAE,OAAO,EAAE,SAAS;AAAA,MACrC,kBAAkBA,GAAE,OAAO,EAAE,SAAS;AAAA,MACtC,WAAWA,GAAE,OAAO,EAAE,SAAS;AAAA,MAC/B,YAAYA,GAAE,OAAO,EAAE,SAAS;AAAA,IAClC,CAAC,EACA,MAAM;AAAA,EACX;AACF,CAAC;AAGM,IAAM,yBAAyB,kBAAkB;AAAA,EACtD,YAAY;AAAA,EACZ,WAAW;AAAA,EACX,MAAM;AAAA,EACN,eAAe;AAAA,EACf,QAAQ;AAAA,IACN,GAAG;AAAA,IACH,kBAAkBA,GACf,OAAO;AAAA,MACN,YAAYA,GAAE,OAAO;AAAA,MACrB,OAAOA,GAAE,OAAO;AAAA,IAClB,CAAC,EACA,MAAM;AAAA,EACX;AACF,CAAC;;;AC/DD,SAAS,KAAAC,UAAS;AAEX,IAAM,SAAS;AAEf,IAAM,qBAAqB,qBAAqB;AAAA,EACrD,YAAY;AAAA,EACZ,WAAW;AAAA,EACX,MAAM;AAAA,EACN,eAAe;AAAA,EACf,QAAQ;AAAA,IACN,UAAUA,GAAE,OAAO;AAAA,IACnB,aAAaA,GAAE,OAAO,EAAE,MAAM;AAAA,IAC9B,YAAYA,GAAE,OAAO,EAAE,MAAM;AAAA,EAC/B;AACF,CAAC;AAGM,IAAM,qBAAqB,qBAAqB;AAAA,EACrD,YAAY;AAAA,EACZ,WAAW;AAAA,EACX,MAAM;AAAA,EACN,eAAe;AAAA,EACf,QAAQ;AAAA,IACN,GAAG;AAAA,EACL;AACF,CAAC;AAGM,IAAM,kBAAkB,kBAAkB;AAAA,EAC/C,YAAY;AAAA,EACZ,WAAW;AAAA,EACX,MAAM;AAAA,EACN,eAAe;AAAA,EACf,QAAQ;AAAA,IACN,GAAG;AAAA,EACL;AACF,CAAC;;;ACxCD,OAAO,gBAAgB;AACvB,OAAOC,QAAO;AAQP,IAAM,YAAY;AAAA,EACvB,OAAO,WAYD;AACJ,UAAM,EAAE,UAAU,QAAQ,OAAO,IAAI;AACrC,UAAM,WAAyC,CAAC;AAEhD,QAAI,OAAO,cAAc;AACvB,eAAS,KAAK;AAAA,QACZ,MAAM;AAAA,QACN,SAAS,OAAO,aAAa;AAAA,MAC/B,CAAC;AAAA,IACH;AAEA,aAAS,KAAK;AAAA,MACZ,MAAM;AAAA,MACN,SAAS,UAAU,QAAQ;AAAA,IAC7B,CAAC;AACD,qBAAiB,UAAU,OAAO,qBAAqB,CAAC,CAAC;AAEzD,UAAM,mBAAmB,MAAM,OAAO,QAAQ,EAAE,SAAS,CAAC;AAE1D,UAAM,WAAW,MAAM,oBAAoB;AAAA,MACzC;AAAA,QACE,MAAM,iBAAiB;AAAA,QACvB,WAAW,iBAAiB;AAAA,QAC5B,aAAa,iBAAiB;AAAA,QAC9B,YAAY,SAAS;AAAA,QACrB,WAAW,OAAO;AAAA,QAClB,UAAU,OAAO,SAAS;AAAA,QAC1B,gBAAgB,OAAO,cAAc;AAAA,QACrC,iBAAiB,iBAAiB;AAAA,QAClC,kBAAkB,iBAAiB;AAAA,QACnC,WAAW,iBAAiB;AAAA,QAC5B,YAAY,iBAAiB;AAAA,MAC/B;AAAA,MACA,OAAO,cAAc,YAAY;AAAA,IACnC;AAEA,QAAI,QAAQ,SAAU,GAAG,mBAAmB,QAAkB;AAC5D,YAAM,eAAe,MAAM,OAAO,MAAM;AAAA,QACtC,UAAU,SAAS;AAAA,QACnB,SAAS,SAAS;AAAA,QAClB,gBAAgB,SAAS;AAAA,MAC3B,CAAC;AAED,UAAI,iBAAiB,MAAM;AACzB,cAAM,QAAQ,MAAM,iBAAiB;AAAA,UACnC;AAAA,YACE,eAAe,cAAc;AAAA,YAC7B,OAAO,aAAa;AAAA,YACpB,YAAY,SAAS;AAAA,YACrB,kBAAkB,aAAa;AAAA,YAC/B,aAAa,aAAa;AAAA,YAC1B,UAAU,aAAa;AAAA,UACzB;AAAA,UACA,OAAO,cAAc,SAAS;AAAA,QAChC;AAEA,eAAO,EAAE,UAAU,MAAM;AAAA,MAC3B;AAAA,IACF;AAEA,QAAI,QAAQ,SAAU,GAAG,mBAAmB,mBAA6B;AACvE,YAAM,eAAe,MAAM,OAAO,MAAM;AAAA,QACtC,UAAU;AAAA,UACR;AAAA,YACE,IAAI;AAAA,YACJ,aACE;AAAA,YACF,QAAQ;AAAA,UACV;AAAA,QACF;AAAA,QACA,QAAQ;AAAA,qBACK,SAAS,kBAAkB,IAAI,CAAC,QAAQ,KAAK,GAAG,EAAE,EAAE,KAAK,IAAI,CAAC;AAAA,sBAC7D,SAAS,kBAAkB,IAAI,CAAC,QAAQ,KAAK,SAAS,UAAU,GAAG,KAAK,EAAE,EAAE,EAAE,KAAK,IAAI,CAAC;AAAA,QACtG,iBAAiB;AAAA,UACf,kBAAkBC,GACf,OAAO,EACP,MAAM,EACN;AAAA,YACC;AAAA,UACF;AAAA,UACF,GAAI,OAAO,2BAA2B,CAAC;AAAA,QACzC;AAAA,QACA,UAAU,SAAS;AAAA,QACnB,cAAc,OAAO,sBAAsB;AAAA,MAC7C,CAAC;AAED,UAAI,iBAAiB,MAAM;AACzB,cAAM,EAAE,kBAAkB,GAAG,gBAAgB,IAC3C,aAAa;AACf,cAAM,QAAQ,MAAM,iBAAiB;AAAA,UACnC;AAAA,YACE,eAAe,cAAc;AAAA,YAC7B,OAAO,aAAa;AAAA,YACpB;AAAA,YACA,YAAY,SAAS;AAAA,YACrB,aAAa,aAAa;AAAA,YAC1B,mBAAmB,aAAa;AAAA,YAChC,oBAAoB,aAAa;AAAA,YACjC,yBAAyB,aAAa;AAAA,YACtC,0BAA0B,aAAa;AAAA,YACvC,kBAAkB,aAAa;AAAA,YAC/B,mBAAmB,aAAa;AAAA,YAChC,wBAAwB,OAAO,sBAAsB;AAAA,YACrD,UAAU;AAAA,cACR,GAAG,aAAa;AAAA,cAChB;AAAA,YACF;AAAA,UACF;AAAA,UACA,OAAO,cAAc,SAAS;AAAA,QAChC;AAEA,eAAO,EAAE,UAAU,MAAM;AAAA,MAC3B;AAAA,IACF;AAEA,WAAO,EAAE,SAAS;AAAA,EACpB;AACF;AAEA,SAAS,UAAU,UAAyB;AAC1C,SAAO,aAAa,SAAS,QAAQ;AAAA;AAAA,EAAe,OAAO;AAAA,IACzD,SAAS,WAAW,CAAC;AAAA,EACvB,EACG,IAAI,CAAC,CAAC,KAAK,KAAK,MAAM,GAAG,GAAG,KAAK,KAAK,EAAE,EACxC,KAAK,IAAI,CAAC;AACf;AAEA,SAAS,iBACP,UACA,mBACA;AACA,WAAS,IAAI,GAAG,IAAI,SAAS,QAAQ,KAAK;AACxC,UAAM,WAAW,WAAW,QAAQ,SAAS,CAAC,EAAG,OAAO;AACxD,aAAS,CAAC,EAAG,UAAU,SAAS,iBAAiB;AAAA,EACnD;AACF;;;AC7JA,OAAOC,iBAAgB;AAShB,IAAM,WAAW;AAAA,EACtB,OAAO,WAYD;AACJ,UAAM,EAAE,UAAU,QAAQ,OAAO,IAAI;AACrC,UAAM,WAAyC,CAAC;AAEhD,QAAI,OAAO,cAAc;AACvB,eAAS,KAAK;AAAA,QACZ,MAAM;AAAA,QACN,SAAS,OAAO,aAAa;AAAA,MAC/B,CAAC;AAAA,IACH;AAEA,aAAS,KAAK;AAAA,MACZ,MAAM;AAAA,MACN,SAAS,SAAS;AAAA,IACpB,CAAC;AACD,IAAAC,kBAAiB,UAAU,OAAO,qBAAqB,CAAC,CAAC;AAEzD,UAAM,mBAAmB,MAAM,OAAO,QAAQ,EAAE,SAAS,CAAC;AAE1D,UAAM,WAAW,MAAM,mBAAmB;AAAA,MACxC;AAAA,QACE,MAAM,iBAAiB;AAAA,QACvB,WAAW,iBAAiB;AAAA,QAC5B,aAAa,iBAAiB;AAAA,QAC9B,YAAY,SAAS;AAAA,QACrB,WAAW,OAAO;AAAA,QAClB,UAAU,OAAO,SAAS;AAAA,QAC1B,gBAAgB,OAAO,cAAc;AAAA,QACrC,iBAAiB,iBAAiB;AAAA,QAClC,kBAAkB,iBAAiB;AAAA,QACnC,WAAW,iBAAiB;AAAA,QAC5B,YAAY,iBAAiB;AAAA,MAC/B;AAAA,MACA,OAAO,cAAc,YAAY;AAAA,IACnC;AAEA,QAAI,QAAQ,SAAU,GAAG,mBAAmB,mBAA6B;AACvE,YAAM,eAAe,MAAM,OAAO,MAAM;AAAA,QACtC,UAAU,SAAS;AAAA,QACnB,QAAQ,2BAA2B,SAAS,YAAY,KAAK,IAAI,CAAC;AAAA,mBAAsB,SAAS,WAAW,KAAK,IAAI,CAAC;AAAA,QACtH,cAAc,OAAO,sBAAsB;AAAA,QAC3C,UAAU;AAAA,UACR;AAAA,YACE,IAAI;AAAA,YACJ,aACE;AAAA,YACF,QAAQ;AAAA,UACV;AAAA,QACF;AAAA,QACA,iBAAiB,OAAO,2BAA2B,CAAC;AAAA,MACtD,CAAC;AAED,UAAI,iBAAiB,MAAM;AACzB,cAAM,QAAQ,MAAM,gBAAgB;AAAA,UAClC;AAAA,YACE,eAAe,cAAc;AAAA,YAC7B,OAAO,aAAa;AAAA,YACpB,YAAY,SAAS;AAAA,YACrB,aAAa,aAAa;AAAA,YAC1B,mBAAmB,aAAa;AAAA,YAChC,oBAAoB,aAAa;AAAA,YACjC,yBAAyB,aAAa;AAAA,YACtC,0BAA0B,aAAa;AAAA,YACvC,kBAAkB,aAAa;AAAA,YAC/B,mBAAmB,aAAa;AAAA,YAChC,wBAAwB,OAAO,sBAAsB;AAAA,YACrD,UAAU;AAAA,cACR,GAAG,aAAa;AAAA,cAChB,iBAAiB,aAAa;AAAA,YAChC;AAAA,UACF;AAAA,UACA,OAAO,cAAc,SAAS;AAAA,QAChC;AAEA,eAAO,EAAE,UAAU,MAAM;AAAA,MAC3B;AAAA,IACF;AAEA,WAAO,EAAE,SAAS;AAAA,EACpB;AACF;AAEA,SAASA,kBACP,UACA,mBACA;AACA,WAAS,IAAI,GAAG,IAAI,SAAS,QAAQ,KAAK;AACxC,UAAM,WAAWC,YAAW,QAAQ,SAAS,CAAC,EAAG,OAAO;AACxD,aAAS,CAAC,EAAG,UAAU,SAAS,iBAAiB;AAAA,EACnD;AACF;;;AC9FA,OAAOC,QAAO;AAEP,IAAM,uBAAN,cAAmC,gBAUxC;AAAA,EACA,YAAY,QAA8C;AACxD,UAAM;AAAA,MACJ,MAAM,OAAO;AAAA,MACb,WAAW,OAAO;AAAA,MAElB,QAAQA,GAAE,MAAM;AAAA,QACd;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,MACF,CAAC;AAAA,IACH,CAAC;AAAA,EACH;AACF;","names":["z","z","z","z","Handlebars","templateMessages","Handlebars","z"]}

package/dist/benchmarks/peerbench/index.d.ts

@@ -1,5 +1,6 @@
 export * from "./schema-sets/mcq.v1";
 export * from "./schema-sets/multi-turn.v1";
 export * from "./schema-sets/qa.v1";
-export * from "./runner";
+export * from "./mcq-runner";
+export * from "./qa-runner";
 export * from "./storages/json";