peerbench 0.0.2-alpha.0 → 0.0.2-alpha.1

package/dist/benchmarks/index.js

@@ -1,36 +1,42 @@
 import {
-  AbstractDataLoader,
-  GenericJSONArrayDataLoader,
-  stableStringify
-} from "../chunk-HJH3SW3L.js";
+  SimpleSystemPromptSchemaV1
+} from "../chunk-Q6GSOHOP.js";
 import {
-  AbstractScorer
-} from "../chunk-22HU24QF.js";
+  defineRunner
+} from "../chunk-QY5MPNNB.js";
 import {
-  bufferToString,
-  idGeneratorUUIDv7,
-  parseResponseAsJSON
-} from "../chunk-DUBKY73H.js";
-import {
-  BaseBenchmarkSpecSchemaV1,
+  BaseResponseSchemaV1,
   BaseScoreSchemaV1,
   BaseTestCaseSchemaV1,
-  defineBenchmarkSpecSchema,
+  defineResponseSchema,
   defineScoreSchema,
   defineTestCaseSchema
-} from "../chunk-232PY7K3.js";
+} from "../chunk-YY33MNMV.js";
+import "../chunk-OQE6TQXZ.js";
 import {
   ScoringMethod
-} from "../chunk-ZJWSK4VO.js";
+} from "../chunk-HMQYGCKI.js";
+import {
+  JSONFileStorage
+} from "../chunk-ERALDEZY.js";
+import {
+  LLMAsAJudgeScorer,
+  MCQScorer
+} from "../chunk-ZEWI24CV.js";
+import {
+  AbstractLLMProvider
+} from "../chunk-R76XA2K6.js";
 import {
-  ExtensionLLMAsAJudgeScorerFieldsV1
-} from "../chunk-GVF4YZF3.js";
+  PEERBENCH_NAMESPACE
+} from "../chunk-UHHHSYVE.js";
 import {
-  BaseLLMChatResponseSchemaV1
-} from "../chunk-7TREBPSJ.js";
+  idGeneratorUUIDv7
+} from "../chunk-4UBK6452.js";
 import {
-  defineResponseSchema
-} from "../chunk-IUN2IUCS.js";
+  ExtensionLLMAsAJudgeScoreFieldsV1,
+  ExtensionLLMResponseFieldsV1
+} from "../chunk-TRNCF2BG.js";
+import "../chunk-NUEOE3K5.js";
 import {
   __export
 } from "../chunk-PZ5AY32C.js";
@@ -38,878 +44,415 @@ import {
 // src/benchmarks/peerbench/index.ts
 var peerbench_exports = {};
 __export(peerbench_exports, {
-  PeerbenchJSONDataLoader: () => PeerbenchJSONDataLoader,
-  PeerbenchMultipleChoiceResponseSchemaV1: () => PeerbenchMultipleChoiceResponseSchemaV1,
-  PeerbenchMultipleChoiceScoreSchemaV1: () => PeerbenchMultipleChoiceScoreSchemaV1,
-  PeerbenchMultipleChoiceTestCaseSchemaV1: () => PeerbenchMultipleChoiceTestCaseSchemaV1,
-  PeerbenchOpenEndedResponseSchemaV1: () => PeerbenchOpenEndedResponseSchemaV1,
-  PeerbenchOpenEndedScoreSchemaV1: () => PeerbenchOpenEndedScoreSchemaV1,
-  PeerbenchOpenEndedTestCaseSchemaV1: () => PeerbenchOpenEndedTestCaseSchemaV1,
-  runTestCase: () => runTestCase
+  MCQKind: () => MCQKind,
+  MCQResponseSchemaV1: () => MCQResponseSchemaV1,
+  MCQScoreSchemaV1: () => MCQScoreSchemaV1,
+  MCQTestCaseSchemaV1: () => MCQTestCaseSchemaV1,
+  MultiTurnKind: () => MultiTurnKind,
+  MultiTurnResponseSchemaV1: () => MultiTurnResponseSchemaV1,
+  MultiTurnScoreSchemaV1: () => MultiTurnScoreSchemaV1,
+  MultiTurnTestCaseSchemaV1: () => MultiTurnTestCaseSchemaV1,
+  PeerbenchJSONStorage: () => PeerbenchJSONStorage,
+  QAKind: () => QAKind,
+  QAResponseSchemaV1: () => QAResponseSchemaV1,
+  QAScoreSchemaV1: () => QAScoreSchemaV1,
+  QATestCaseSchemaV1: () => QATestCaseSchemaV1,
+  peerbenchRunner: () => peerbenchRunner
 });
 
-// src/benchmarks/peerbench/test-cases/mcq.v1.ts
+// src/benchmarks/peerbench/schema-sets/mcq.v1.ts
 import { z } from "zod";
-
-// src/benchmarks/peerbench/score.ts
-var PeerbenchBaseScoreSchemaV1 = defineScoreSchema({
-  baseSchema: BaseScoreSchemaV1,
-  fields: {
-    ...ExtensionLLMAsAJudgeScorerFieldsV1
-  }
-});
-
-// src/benchmarks/peerbench/test-cases/mcq.v1.ts
-var PeerbenchMultipleChoiceTestCaseSchemaV1 = defineTestCaseSchema({
+var MCQKind = `llm/mcq`;
+var MCQTestCaseSchemaV1 = defineTestCaseSchema({
   baseSchema: BaseTestCaseSchemaV1,
-  kind: "pb.ts.mcq",
+  namespace: PEERBENCH_NAMESPACE,
+  kind: MCQKind,
   schemaVersion: 1,
   fields: {
     question: z.string(),
     options: z.record(z.string(), z.string()),
-    answer: z.string(),
-    answerKey: z.string()
+    correctAnswerKeys: z.string().array()
   }
 });
-var PeerbenchMultipleChoiceResponseSchemaV1 = defineResponseSchema({
-  baseSchema: BaseLLMChatResponseSchemaV1,
-  kind: "pb.rs.mcq",
+var MCQResponseSchemaV1 = defineResponseSchema({
+  baseSchema: BaseResponseSchemaV1,
+  namespace: PEERBENCH_NAMESPACE,
+  kind: MCQKind,
   schemaVersion: 1,
-  fields: {}
+  fields: {
+    ...ExtensionLLMResponseFieldsV1
+  }
 });
-var PeerbenchMultipleChoiceScoreSchemaV1 = defineScoreSchema({
-  baseSchema: PeerbenchBaseScoreSchemaV1,
-  kind: "pb.sc.mcq",
+var MCQScoreSchemaV1 = defineScoreSchema({
+  baseSchema: BaseScoreSchemaV1,
+  namespace: PEERBENCH_NAMESPACE,
+  kind: MCQKind,
   schemaVersion: 1,
   fields: {
+    ...ExtensionLLMAsAJudgeScoreFieldsV1,
     extractedAnswers: z.array(z.string())
   }
 });
 
-// src/benchmarks/peerbench/test-cases/open-ended.v1.ts
+// src/benchmarks/peerbench/schema-sets/multi-turn.v1.ts
 import { z as z2 } from "zod";
-var PeerbenchOpenEndedTestCaseSchemaV1 = defineTestCaseSchema({
-  kind: "pb.ts.open-ended",
-  schemaVersion: 1,
+var MultiTurnKind = `llm/multi-turn`;
+var MultiTurnTestCaseSchemaV1 = defineTestCaseSchema({
   baseSchema: BaseTestCaseSchemaV1,
+  namespace: PEERBENCH_NAMESPACE,
+  kind: MultiTurnKind,
+  schemaVersion: 1,
   fields: {
-    question: z2.string(),
-    answer: z2.string().optional()
+    messages: z2.object({
+      role: z2.string(),
+      content: z2.string(),
+      goodAnswers: z2.string().array().optional(),
+      badAnswers: z2.string().array().optional()
+    }).array(),
+    maxTurns: z2.number().optional(),
+    expectedOutcome: z2.string().optional()
   }
 });
-var PeerbenchOpenEndedResponseSchemaV1 = defineResponseSchema({
-  baseSchema: BaseLLMChatResponseSchemaV1,
-  kind: "pb.rs.open-ended",
-  schemaVersion: 1
-});
-var PeerbenchOpenEndedScoreSchemaV1 = defineScoreSchema({
-  baseSchema: PeerbenchBaseScoreSchemaV1,
-  kind: "pb.sc.open-ended",
-  schemaVersion: 1,
-  fields: {}
-});
-
-// src/benchmarks/peerbench/spec.ts
-import z3 from "zod";
-var PeerbenchBenchmarkSpecSchemaV1 = defineBenchmarkSpecSchema({
-  baseSchema: BaseBenchmarkSpecSchemaV1,
-  kind: "pb.benchmark.spec",
+var MultiTurnResponseSchemaV1 = defineResponseSchema({
+  baseSchema: BaseResponseSchemaV1,
+  namespace: PEERBENCH_NAMESPACE,
+  kind: MultiTurnKind,
   schemaVersion: 1,
   fields: {
-    /**
-     * Big text contents that can be referred as <text>{key}</text> in a prompt or system prompt.
-     */
-    blobTexts: z3.record(z3.string(), z3.string()).optional()
+    ...ExtensionLLMResponseFieldsV1,
+    replies: z2.object({
+      messageIndex: z2.number(),
+      startedAt: z2.number(),
+      completedAt: z2.number(),
+      data: z2.string(),
+      inputTokensUsed: z2.number().optional(),
+      outputTokensUsed: z2.number().optional(),
+      inputCost: z2.string().optional(),
+      outputCost: z2.string().optional()
+    }).array()
   }
 });
-
-// src/benchmarks/peerbench/loader.ts
-import z4 from "zod";
-var PeerbenchJSONDataLoader = class extends GenericJSONArrayDataLoader {
-  kind = "pb.load.json.data";
-  async loadBenchmarkSpec(params) {
-    const content = bufferToString(params.content);
-    const parsed = PeerbenchBenchmarkSpecSchemaV1.parse(content);
-    return parsed;
-  }
-  testCaseBuilder(data) {
-    const testCaseValidation = z4.union([
-      PeerbenchMultipleChoiceTestCaseSchemaV1,
-      PeerbenchOpenEndedTestCaseSchemaV1
-    ]).safeParse(data);
-    return testCaseValidation.success ? testCaseValidation.data : void 0;
-  }
-  async responseBuilder(data) {
-    const responseValidation = z4.union([
-      PeerbenchMultipleChoiceResponseSchemaV1,
-      PeerbenchOpenEndedResponseSchemaV1
-    ]).safeParse(data);
-    return responseValidation.success ? responseValidation.data : void 0;
-  }
-  async scoreBuilder(data) {
-    const scoreValidation = z4.union([
-      PeerbenchMultipleChoiceScoreSchemaV1,
-      PeerbenchOpenEndedScoreSchemaV1
-    ]).safeParse(data);
-    return scoreValidation.success ? scoreValidation.data : void 0;
-  }
-};
-
-// src/benchmarks/peerbench/runner.ts
-async function runTestCase(params) {
-  const { testCase } = params;
-  const responseIdGenerator = params.idGenerators?.response ?? idGeneratorUUIDv7;
-  const scoreIdGenerator = params.idGenerators?.score ?? idGeneratorUUIDv7;
-  const messages = [];
-  if (params.systemPrompt) {
-    messages.push({
-      role: "system",
-      content: params.systemPrompt.content
-    });
-  }
-  if (testCase.kind === "pb.ts.mcq") {
-    const formattedPrompt = formatMCQPrompt(testCase);
-    messages.push({
-      role: "user",
-      content: formattedPrompt
-    });
-    const providerResponse = await params.provider.forward({
-      model: params.runConfig.model,
-      messages
-    });
-    const response = await PeerbenchMultipleChoiceResponseSchemaV1.newWithId(
-      {
-        data: providerResponse.data,
-        startedAt: providerResponse.startedAt,
-        completedAt: providerResponse.completedAt,
-        testCaseId: testCase.id,
-        modelSlug: params.runConfig.model,
-        provider: params.provider.kind,
-        inputTokensUsed: providerResponse.inputTokensUsed,
-        outputTokensUsed: providerResponse.outputTokensUsed,
-        inputCost: providerResponse.inputCost,
-        outputCost: providerResponse.outputCost
-      },
-      responseIdGenerator
-    );
-    if (params.scorer?.kind === "mcq") {
-      const scorerResult = await params.scorer.score({
-        response: response.data,
-        choices: testCase.options ?? {},
-        correctAnswers: [testCase.answerKey]
-      });
-      if (scorerResult !== null) {
-        const score = await PeerbenchMultipleChoiceScoreSchemaV1.newWithId(
-          {
-            scoringMethod: ScoringMethod.algo,
-            value: scorerResult.value,
-            responseId: response.id,
-            extractedAnswers: scorerResult.extractedAnswers,
-            metadata: response.metadata
-          },
-          scoreIdGenerator
-        );
-        return { response, score };
-      }
-    }
-    return { response };
-  } else if (testCase.kind === "pb.ts.open-ended") {
-    const messages2 = [];
-    if (params.systemPrompt) {
-      messages2.push({
-        role: "system",
-        content: params.systemPrompt.content
-      });
-    }
-    messages2.push({
-      role: "user",
-      content: testCase.question
-    });
-    const providerResponse = await params.provider.forward({
-      model: params.runConfig.model,
-      messages: messages2
-    });
-    const response = await PeerbenchOpenEndedResponseSchemaV1.newWithId(
-      {
-        data: providerResponse.data,
-        startedAt: providerResponse.startedAt,
-        completedAt: providerResponse.completedAt,
-        testCaseId: testCase.id,
-        modelSlug: params.runConfig.model,
-        provider: params.provider.kind,
-        inputTokensUsed: providerResponse.inputTokensUsed,
-        outputTokensUsed: providerResponse.outputTokensUsed,
-        inputCost: providerResponse.inputCost,
-        outputCost: providerResponse.outputCost
-      },
-      responseIdGenerator
-    );
-    if (params.scorer?.kind === "llmJudge" && params.runConfig.llmJudgeModel) {
-      const scorerResult = await params.scorer.score({
-        task: testCase.question,
-        candidateAnswer: response.data,
-        referenceAnswer: testCase.answer,
-        model: params.runConfig.llmJudgeModel
-      });
-      if (scorerResult !== null) {
-        const score = await PeerbenchOpenEndedScoreSchemaV1.newWithId(
-          {
-            scoringMethod: ScoringMethod.ai,
-            value: scorerResult.value,
-            responseId: response.id,
-            explanation: scorerResult.explanation,
-            metadata: scorerResult.metadata,
-            scorerAIProvider: scorerResult.provider,
-            scorerAIModelSlug: params.runConfig.llmJudgeModel,
-            scorerAIInputTokensUsed: scorerResult.inputTokensUsed,
-            scorerAIOutputTokensUsed: scorerResult.outputTokensUsed,
-            scorerAIInputCost: scorerResult.inputCost,
-            scorerAIOutputCost: scorerResult.outputCost
-          },
-          scoreIdGenerator
-        );
-        return { response, score };
-      }
-    }
-    return { response };
-  }
-  throw new Error("Unsupported test case kind");
-}
-function formatMCQPrompt(testCase) {
-  return `Question: ${testCase.question}
-Options:
-${Object.entries(
-    testCase.options ?? {}
-  ).map(([key, value]) => `${key}: ${value}`).join("\n")}`;
-}
-
-// src/benchmarks/mmlu-pro/index.ts
-var mmlu_pro_exports = {};
-__export(mmlu_pro_exports, {
-  BaseMMLUProScoreSchemaV1: () => BaseMMLUProScoreSchemaV1,
-  MMLUProBenchmarkSpecSchemaV1: () => MMLUProBenchmarkSpecSchemaV1,
-  MMLUProJSONDataLoader: () => MMLUProJSONDataLoader,
-  MMLUProMainResponseSchemaV1: () => MMLUProMainResponseSchemaV1,
-  MMLUProMainScoreSchemaV1: () => MMLUProMainScoreSchemaV1,
-  MMLUProMainTestCaseSchemaV1: () => MMLUProMainTestCaseSchemaV1,
-  MMLUProParquetDataLoader: () => MMLUProParquetDataLoader,
-  runTestCase: () => runTestCase2
-});
-
-// src/benchmarks/mmlu-pro/score.ts
-var BaseMMLUProScoreSchemaV1 = defineScoreSchema({
+var MultiTurnScoreSchemaV1 = defineScoreSchema({
   baseSchema: BaseScoreSchemaV1,
+  namespace: PEERBENCH_NAMESPACE,
+  kind: MultiTurnKind,
+  schemaVersion: 1,
   fields: {
-    ...ExtensionLLMAsAJudgeScorerFieldsV1
+    ...ExtensionLLMAsAJudgeScoreFieldsV1,
+    individualScores: z2.object({
+      replyIndex: z2.number(),
+      value: z2.number()
+    }).array()
   }
 });
 
-// src/benchmarks/mmlu-pro/test-cases/main.v1.ts
-import { z as z5 } from "zod";
-var MMLUProMainTestCaseSchemaV1 = defineTestCaseSchema({
+// src/benchmarks/peerbench/schema-sets/qa.v1.ts
+import { z as z3 } from "zod";
+var QAKind = `llm/qa`;
+var QATestCaseSchemaV1 = defineTestCaseSchema({
   baseSchema: BaseTestCaseSchemaV1,
-  kind: "mmlu-pro.ts.main",
+  namespace: PEERBENCH_NAMESPACE,
+  kind: QAKind,
   schemaVersion: 1,
   fields: {
-    question: z5.string(),
-    options: z5.record(z5.string(), z5.string()),
-    answer: z5.string(),
-    answerKey: z5.string()
+    question: z3.string(),
+    goodAnswers: z3.string().array(),
+    badAnswers: z3.string().array()
   }
 });
-var MMLUProMainResponseSchemaV1 = defineResponseSchema({
-  baseSchema: BaseLLMChatResponseSchemaV1,
-  kind: "mmlu-pro.rs.main",
-  schemaVersion: 1
-});
-var MMLUProMainScoreSchemaV1 = defineScoreSchema({
-  baseSchema: BaseMMLUProScoreSchemaV1,
-  kind: "mmlu-pro.sc.main",
+var QAResponseSchemaV1 = defineResponseSchema({
+  baseSchema: BaseResponseSchemaV1,
+  namespace: PEERBENCH_NAMESPACE,
+  kind: QAKind,
   schemaVersion: 1,
   fields: {
-    extractedAnswers: z5.array(z5.string())
+    ...ExtensionLLMResponseFieldsV1
   }
 });
-
-// src/benchmarks/mmlu-pro/loader.ts
-import z6 from "zod";
-import { parquetReadObjects } from "hyparquet";
-var jsonSchema = z6.object({
-  question_id: z6.coerce.number(),
-  question: z6.string(),
-  options: z6.array(z6.string()),
-  answer: z6.string(),
-  answer_index: z6.coerce.number(),
-  cot_content: z6.string(),
-  category: z6.string(),
-  src: z6.string()
-}).array();
-function mapData(data) {
-  return {
-    responses: [],
-    scores: [],
-    testCases: data.map(
-      (item) => MMLUProMainTestCaseSchemaV1.new({
-        id: `${item.src}-${item.category}-${item.question_id}`,
-        question: item.question,
-        answerKey: item.answer,
-        options: item.options.reduce(
-          (acc, option, index) => {
-            acc[String.fromCharCode(65 + index)] = option;
-            return acc;
-          },
-          {}
-        ),
-        answer: item.options[item.answer_index],
-        metadata: {
-          category: item.category,
-          src: item.src,
-          answer_index: item.answer_index
-        }
-      })
-    )
-  };
-}
-var MMLUProJSONDataLoader = class extends AbstractDataLoader {
-  kind = "mmlu-pro.load.json.data";
-  loadData(params) {
-    const content = typeof params.content === "string" ? params.content : bufferToString(params.content);
-    const parsed = jsonSchema.parse(JSON.parse(content));
-    return mapData(parsed);
-  }
-  // eslint-disable-next-line @typescript-eslint/no-unused-vars
-  loadBenchmarkSpec(params) {
-    throw new Error("Not implemented");
-  }
-};
-var MMLUProParquetDataLoader = class extends AbstractDataLoader {
-  kind = "mmlu-pro.load.parquet.data";
-  async loadData(params) {
-    const data = await parquetReadObjects({
-      file: params.content.buffer
-    });
-    if (!data) {
-      throw new Error("Invalid Parquet file");
-    }
-    return mapData(jsonSchema.parse(data));
-  }
-  // eslint-disable-next-line @typescript-eslint/no-unused-vars
-  loadBenchmarkSpec(params) {
-    throw new Error("Not implemented");
+var QAScoreSchemaV1 = defineScoreSchema({
+  baseSchema: BaseScoreSchemaV1,
+  namespace: PEERBENCH_NAMESPACE,
+  kind: QAKind,
+  schemaVersion: 1,
+  fields: {
+    ...ExtensionLLMAsAJudgeScoreFieldsV1
   }
-};
+});
 
-// src/benchmarks/mmlu-pro/runner.ts
-async function runTestCase2(params) {
-  const { testCase } = params;
-  const responseIdGenerator = params.idGenerators?.response ?? idGeneratorUUIDv7;
-  const scoreIdGenerator = params.idGenerators?.score ?? idGeneratorUUIDv7;
-  const messages = [];
-  if (params.systemPrompt) {
-    messages.push({
-      role: "system",
-      content: params.systemPrompt.content
-    });
-  }
-  if (testCase.kind === "mmlu-pro.ts.main") {
-    const formattedPrompt = formatMCQPrompt2(testCase);
-    messages.push({
-      role: "user",
-      content: formattedPrompt
-    });
-    const providerResponse = await params.provider.forward({
-      model: params.runConfig.model,
-      messages
-    });
-    const response = await MMLUProMainResponseSchemaV1.newWithId(
+// src/benchmarks/peerbench/runner.ts
+import Handlebars from "handlebars";
+import z4 from "zod";
+var peerbenchRunner = defineRunner(
+  {
+    schemaSets: [
       {
-        data: providerResponse.data,
-        startedAt: providerResponse.startedAt,
-        completedAt: providerResponse.completedAt,
-        testCaseId: testCase.id,
-        modelSlug: params.runConfig.model,
-        provider: params.provider.kind,
-        inputTokensUsed: providerResponse.inputTokensUsed,
-        outputTokensUsed: providerResponse.outputTokensUsed,
-        inputCost: providerResponse.inputCost,
-        outputCost: providerResponse.outputCost
+        testCase: MCQTestCaseSchemaV1,
+        response: MCQResponseSchemaV1,
+        score: MCQScoreSchemaV1
       },
-      responseIdGenerator
-    );
-    if (params.scorer?.kind === "mcq") {
-      const scorerResult = await params.scorer.score({
-        response: response.data,
-        choices: testCase.options ?? {},
-        correctAnswers: [testCase.answerKey]
-      });
-      if (scorerResult !== null) {
-        const score = await MMLUProMainScoreSchemaV1.newWithId(
-          {
-            scoringMethod: ScoringMethod.algo,
-            value: scorerResult.value,
-            responseId: response.id,
-            extractedAnswers: scorerResult.extractedAnswers,
-            metadata: response.metadata
-          },
-          scoreIdGenerator
-        );
-        return { response, score };
+      {
+        testCase: QATestCaseSchemaV1,
+        response: QAResponseSchemaV1,
+        score: QAScoreSchemaV1
       }
+    ],
+    providers: [AbstractLLMProvider],
+    scorers: [LLMAsAJudgeScorer, MCQScorer],
+    runConfigSchema: {
+      model: z4.string(),
+      llmJudgeModel: z4.string().optional(),
+      llmJudgeSystemPrompt: SimpleSystemPromptSchemaV1.optional(),
+      systemPrompt: SimpleSystemPromptSchemaV1.optional(),
+      templateVariables: z4.record(z4.string(), z4.string()).optional()
     }
-    return { response };
-  } else if (testCase.kind === "pb.ts.open-ended") {
-    const messages2 = [];
-    if (params.systemPrompt) {
-      messages2.push({
+  },
+  async (params) => {
+    const { testCase, provider, scorer, runConfig } = params;
+    const messages = [];
+    if (runConfig.systemPrompt) {
+      messages.push({
         role: "system",
-        content: params.systemPrompt.content
+        content: runConfig.systemPrompt.content
       });
     }
-    messages2.push({
-      role: "user",
-      content: testCase.question
-    });
-    const providerResponse = await params.provider.forward({
-      model: params.runConfig.model,
-      messages: messages2
-    });
-    const response = await MMLUProMainResponseSchemaV1.newWithId(
-      {
-        data: providerResponse.data,
-        startedAt: providerResponse.startedAt,
-        completedAt: providerResponse.completedAt,
-        testCaseId: testCase.id,
-        modelSlug: params.runConfig.model,
-        provider: params.provider.kind,
-        inputTokensUsed: providerResponse.inputTokensUsed,
-        outputTokensUsed: providerResponse.outputTokensUsed,
-        inputCost: providerResponse.inputCost,
-        outputCost: providerResponse.outputCost
-      },
-      responseIdGenerator
-    );
-    if (params.scorer?.kind === "llmJudge" && params.runConfig.llmJudgeModel) {
-      const scorerResult = await params.scorer.score({
-        task: testCase.question,
-        candidateAnswer: response.data,
-        referenceAnswer: testCase.answer,
-        model: params.runConfig.llmJudgeModel
+    if (testCase.kind === "llm/mcq.tc") {
+      messages.push({
+        role: "user",
+        content: formatMCQ(testCase)
+      });
+      templateMessages(messages, runConfig.templateVariables ?? {});
+      return runMCQ({
+        testCase,
+        messages,
+        provider,
+        scorer,
+        runConfig,
+        idGenerators: {
+          response: params.idGenerators?.response ?? idGeneratorUUIDv7,
+          score: params.idGenerators?.score ?? idGeneratorUUIDv7
+        }
       });
-      if (scorerResult !== null) {
-        const score = await MMLUProMainScoreSchemaV1.newWithId(
-          {
-            scoringMethod: ScoringMethod.ai,
-            value: scorerResult.value,
-            responseId: response.id,
-            explanation: scorerResult.explanation,
-            metadata: scorerResult.metadata,
-            extractedAnswers: [],
-            scorerAIProvider: scorerResult.provider,
-            scorerAIModelSlug: params.runConfig.llmJudgeModel,
-            scorerAIInputTokensUsed: scorerResult.inputTokensUsed,
-            scorerAIOutputTokensUsed: scorerResult.outputTokensUsed,
-            scorerAIInputCost: scorerResult.inputCost,
-            scorerAIOutputCost: scorerResult.outputCost
-          },
-          scoreIdGenerator
+    }
+    if (testCase.kind === "llm/qa.tc") {
+      if (scorer && scorer?.kind !== `${PEERBENCH_NAMESPACE}/llm-as-a-judge`) {
+        throw new Error(
+          `QA test cases can only be scored with an LLM as a judge scorer, but ${scorer?.kind} was provided`
         );
-        return { response, score };
       }
+      messages.push({
+        role: "user",
+        content: testCase.question
+      });
+      templateMessages(messages, runConfig.templateVariables ?? {});
+      return runQA({
+        testCase,
+        messages,
+        provider,
+        scorer,
+        runConfig,
+        idGenerators: {
+          response: params.idGenerators?.response ?? idGeneratorUUIDv7,
+          score: params.idGenerators?.score ?? idGeneratorUUIDv7
+        }
+      });
     }
-    return { response };
-  }
-  throw new Error("Unsupported test case kind");
-}
-function formatMCQPrompt2(testCase) {
-  return `Question: ${testCase.question}
-Options:
-${Object.entries(
-    testCase.options ?? {}
-  ).map(([key, value]) => `${key}: ${value}`).join("\n")}`;
-}
-
-// src/benchmarks/mmlu-pro/spec.ts
-var MMLUProBenchmarkSpecSchemaV1 = defineBenchmarkSpecSchema({
-  baseSchema: BaseBenchmarkSpecSchemaV1,
-  kind: "mmlu-pro.benchmark.spec",
-  schemaVersion: 1,
-  fields: {}
-});
-
-// src/benchmarks/fnol/index.ts
-var fnol_exports = {};
-__export(fnol_exports, {
-  FNOLBaseScoreSchemaV1: () => FNOLBaseScoreSchemaV1,
-  FNOLConversationMessageSchemaV1: () => FNOLConversationMessageSchemaV1,
-  FNOLDoneReason: () => FNOLDoneReason,
-  FNOLFieldSchemaV1: () => FNOLFieldSchemaV1,
-  FNOLFieldValueType: () => FNOLFieldValueType,
-  FNOLFieldsScoreSchemaV1: () => FNOLFieldsScoreSchemaV1,
-  FNOLFieldsScorer: () => FNOLFieldsScorer,
-  FNOLLLMJudgeScoreSchemaV1: () => FNOLLLMJudgeScoreSchemaV1,
-  FNOLResponseSchemaV1: () => FNOLResponseSchemaV1,
-  FNOLTestCaseSchemaV1: () => FNOLTestCaseSchemaV1,
-  runTestCase: () => runTestCase3
-});
-
-// src/benchmarks/fnol/test-cases/fnol.v1.ts
-import { z as z7 } from "zod";
-
-// src/benchmarks/fnol/score.ts
-var FNOLBaseScoreSchemaV1 = defineScoreSchema({
-  baseSchema: BaseScoreSchemaV1,
-  fields: {
-    ...ExtensionLLMAsAJudgeScorerFieldsV1
-  }
-});
-
-// src/benchmarks/fnol/types.ts
-var FNOLFieldValueType = {
-  string: "string",
-  number: "number",
-  boolean: "boolean",
-  object: "object"
-};
-var FNOLDoneReason = {
-  modelProvidedJson: "modelProvidedJson",
-  reachedMaxTurns: "reachedMaxTurns",
-  forcedFinalJson: "forcedFinalJson"
-};
-
-// src/benchmarks/fnol/test-cases/fnol.v1.ts
-var FNOLFieldSchemaV1 = z7.object({
-  description: z7.string(),
-  required: z7.boolean().optional(),
-  /**
-   * Optional expected value used by the deterministic scorer.
-   * If omitted, the scorer will only check presence.
-   */
-  expected: z7.unknown().optional(),
-  /**
-   * Optional type hint for the model/user simulation.
-   */
-  valueType: z7.enum(FNOLFieldValueType).optional()
-});
-var FNOLTestCaseSchemaV1 = defineTestCaseSchema({
-  baseSchema: BaseTestCaseSchemaV1,
-  kind: "fnol.ts.v1",
-  schemaVersion: 1,
-  fields: {
-    /**
-     * Scenario starter message. This is what the "user" would say initially.
-     */
-    initialUserMessage: z7.string(),
-    /**
-     * Private/structured information about the user and the incident.
-     * This is used by the user simulator LLM to answer the target model questions.
-     */
-    userProfile: z7.record(z7.string(), z7.unknown()),
-    /**
-     * The fields the target model must collect.
-     * Keys are canonical identifiers (e.g. "policyNumber", "dateOfLoss").
-     */
-    fieldsToCollect: z7.record(z7.string(), FNOLFieldSchemaV1),
-    /**
-     * Maximum number of back-and-forth turns (target question + user answer).
-     */
-    maxTurns: z7.number().int().min(1).max(100).default(10)
-  }
-});
-var FNOLConversationMessageSchemaV1 = z7.object({
-  role: z7.enum(["system", "user", "assistant"]),
-  content: z7.string()
-});
-var FNOLResponseSchemaV1 = defineResponseSchema({
-  baseSchema: BaseLLMChatResponseSchemaV1,
-  kind: "fnol.rs.v1",
-  schemaVersion: 1,
-  fields: {
-    /**
-     * Full conversation between the target model and simulated user.
-     */
-    conversation: z7.array(FNOLConversationMessageSchemaV1),
-    turnsUsed: z7.number().int(),
-    doneReason: z7.enum(FNOLDoneReason),
-    /**
-     * Parsed JSON object from the target model's final answer, if available.
-     */
-    extracted: z7.record(z7.string(), z7.unknown()).optional()
-  }
-});
-var FNOLFieldsScoreSchemaV1 = defineScoreSchema({
-  baseSchema: FNOLBaseScoreSchemaV1,
-  kind: "fnol.sc.fields.v1",
-  schemaVersion: 1,
-  fields: {
-    requiredKeys: z7.array(z7.string()),
-    presentKeys: z7.array(z7.string()),
-    missingKeys: z7.array(z7.string()),
-    mismatchedKeys: z7.array(z7.string())
-  }
-});
-var FNOLLLMJudgeScoreSchemaV1 = defineScoreSchema({
-  baseSchema: FNOLBaseScoreSchemaV1,
-  kind: "fnol.sc.llm-judge.v1",
-  schemaVersion: 1,
-  fields: {
-    verdict: z7.enum(["pass", "borderline", "fail"]).optional()
-  }
-});
-
-// src/benchmarks/fnol/runner.ts
-function formatFieldsToCollect(fieldsToCollect) {
-  return Object.entries(fieldsToCollect).map(([key, field]) => {
-    const required = field.required === false ? "optional" : "required";
-    return `- ${key} (${required}): ${field.description}`;
-  }).join("\n");
-}
-function hasAllRequiredFields(params) {
-  const extracted = params.extracted ?? {};
-  for (const [key, field] of Object.entries(params.fieldsToCollect)) {
-    if (field.required === false) continue;
-    const value = extracted[key];
-    if (value === void 0 || value === null || value === "") return false;
-  }
-  return true;
-}
-async function runTestCase3(params) {
-  const responseIdGenerator = params.idGenerators?.response ?? idGeneratorUUIDv7;
-  const scoreIdGenerator = params.idGenerators?.score ?? idGeneratorUUIDv7;
-  const userSimulatorProvider = params.userSimulatorProvider ?? params.provider;
-  const userSimulatorModel = params.runConfig.userSimulatorModel ?? params.runConfig.model;
-  const fieldsToCollectText = formatFieldsToCollect(
-    params.testCase.fieldsToCollect
-  );
-  const conversation = [];
-  if (params.systemPrompt) {
-    conversation.push({
-      role: "system",
-      content: params.systemPrompt.content
-    });
-  }
-  conversation.push({
-    role: "system",
-    content: [
-      "You are an insurance FNOL intake assistant.",
-      "Your job is to ask the user questions to collect the required fields listed below.",
-      "Ask concise questions, one or a few at a time.",
-      "When you have enough information OR when you are told to finish, output ONLY a single JSON object with the collected fields.",
-      "Do not include markdown fences. Do not include additional text outside the JSON.",
-      "",
-      "Fields to collect:",
-      fieldsToCollectText
-    ].join("\n")
-  });
-  conversation.push({
-    role: "user",
-    content: params.testCase.initialUserMessage
+    throw new Error("Unsupported test case kind");
+  }
+);
+async function runQA(params) {
+  const { messages, testCase, provider, scorer, runConfig } = params;
+  const providerResponse = await provider.forward({
+    model: runConfig.model,
+    messages
   });
-  let doneReason;
-  let extracted;
-  const startedAt = Date.now();
-  for (let turn = 0; turn < params.testCase.maxTurns; turn++) {
-    const targetReply = await params.provider.forward({
-      model: params.runConfig.model,
-      temperature: params.runConfig.temperature,
-      messages: conversation
-    });
-    conversation.push({
-      role: "assistant",
-      content: targetReply.data
-    });
-    extracted = parseResponseAsJSON(targetReply.data);
-    if (extracted && hasAllRequiredFields({
-      extracted,
-      fieldsToCollect: params.testCase.fieldsToCollect
-    })) {
-      doneReason = FNOLDoneReason.modelProvidedJson;
-      break;
+  const response = await QAResponseSchemaV1.newWithId(
+    {
+      data: providerResponse.data,
+      startedAt: providerResponse.startedAt,
+      completedAt: providerResponse.completedAt,
+      testCaseId: testCase.id,
+      modelSlug: runConfig.model,
+      provider: provider.kind,
+      systemPromptId: runConfig.systemPrompt?.id,
+      inputTokensUsed: providerResponse.inputTokensUsed,
+      outputTokensUsed: providerResponse.outputTokensUsed,
+      inputCost: providerResponse.inputCost,
+      outputCost: providerResponse.outputCost
+    },
+    params.idGenerators?.response ?? idGeneratorUUIDv7
+  );
+  if (scorer?.kind === `${PEERBENCH_NAMESPACE}/llm-as-a-judge`) {
+    if (!runConfig.llmJudgeModel) {
+      throw new Error(
+        "LLM judge model is required when using LLM as a judge scorer"
+      );
     }
-    const lastAssistantMessage = targetReply.data;
-    const simulatedUser = await userSimulatorProvider.forward({
-      model: userSimulatorModel,
-      temperature: params.runConfig.userSimulatorTemperature,
-      messages: [
-        {
-          role: "system",
-          content: [
-            "You are simulating a real insurance customer (the user).",
-            "Answer the assistant's questions truthfully using ONLY the provided user profile and incident details.",
-            "If asked about something not present in the profile, say you don't know.",
-            "Be concise and natural. Do not invent new facts.",
-            "",
-            "User profile (JSON):",
-            JSON.stringify(params.testCase.userProfile)
-          ].join("\n")
-        },
+    const scorerResult = await scorer.score({
+      model: runConfig.llmJudgeModel,
+      response: response.data,
+      rubric: `Expected/Valid answers: ${testCase.goodAnswers.join("\n")}
+Invalid answers: ${testCase.badAnswers.join("\n")}`,
+      systemPrompt: runConfig.llmJudgeSystemPrompt?.content,
+      criteria: [
         {
-          role: "user",
-          content: lastAssistantMessage
+          id: "correctness",
+          description: "Is the response matches with the expected/valid answers in terms of meaning?",
+          weight: 1
         }
       ]
     });
-    conversation.push({
-      role: "user",
-      content: simulatedUser.data
-    });
-  }
-  if (!doneReason) {
-    doneReason = FNOLDoneReason.reachedMaxTurns;
-    const forced = await params.provider.forward({
-      model: params.runConfig.model,
-      temperature: params.runConfig.temperature,
-      messages: [
-        ...conversation,
+    if (scorerResult !== null) {
+      const score = await QAScoreSchemaV1.newWithId(
         {
-          role: "user",
-          content: "Stop the interview now and output ONLY the final JSON object with the collected fields. No extra text."
-        }
-      ]
-    });
-    conversation.push({ role: "assistant", content: forced.data });
-    extracted = parseResponseAsJSON(forced.data);
-    if (extracted) {
-      doneReason = FNOLDoneReason.forcedFinalJson;
+          scoringMethod: ScoringMethod.ai,
+          value: scorerResult.value,
+          responseId: response.id,
+          explanation: scorerResult.explanation,
+          metadata: scorerResult.metadata,
+          scorerAIInputCost: scorerResult.inputCost,
+          scorerAIOutputCost: scorerResult.outputCost,
+          scorerAIInputTokensUsed: scorerResult.inputTokensUsed,
+          scorerAIOutputTokensUsed: scorerResult.outputTokensUsed,
+          scorerAIProvider: scorerResult.provider,
+          scorerAIModelSlug: runConfig.llmJudgeModel,
+          scorerAISystemPromptId: runConfig.llmJudgeSystemPrompt?.id
+        },
+        params.idGenerators?.score ?? idGeneratorUUIDv7
+      );
+      return { response, score };
     }
   }
-  const completedAt = Date.now();
-  const lastAssistant = [...conversation].reverse().find((m) => m.role === "assistant");
-  const response = await FNOLResponseSchemaV1.newWithId(
+  return { response };
+}
+async function runMCQ(params) {
+  const { messages, testCase, provider, scorer, runConfig } = params;
+  const providerResponse = await provider.forward({
+    model: runConfig.model,
+    messages
+  });
+  const response = await MCQResponseSchemaV1.newWithId(
     {
-      data: typeof lastAssistant?.content === "string" ? lastAssistant.content : "",
-      startedAt,
-      completedAt,
-      testCaseId: params.testCase.id,
-      modelSlug: params.runConfig.model,
-      provider: params.provider.kind,
-      conversation: conversation.map((m) => ({
-        role: m.role === "user" ? "user" : "assistant",
-        content: String(m.content)
-      })),
-      turnsUsed: conversation.filter((m) => m.role === "assistant").length,
-      doneReason,
-      extracted
+      data: providerResponse.data,
+      startedAt: providerResponse.startedAt,
+      completedAt: providerResponse.completedAt,
+      testCaseId: testCase.id,
+      modelSlug: runConfig.model,
+      provider: provider.kind,
+      systemPromptId: runConfig.systemPrompt?.id,
+      inputTokensUsed: providerResponse.inputTokensUsed,
+      outputTokensUsed: providerResponse.outputTokensUsed,
+      inputCost: providerResponse.inputCost,
+      outputCost: providerResponse.outputCost
     },
-    responseIdGenerator
+    params.idGenerators?.response ?? idGeneratorUUIDv7
   );
-  if (params.scorer?.kind === "fnol.fields") {
-    const scorerResult = await params.scorer.score({
-      fieldsToCollect: params.testCase.fieldsToCollect,
-      extracted
+  if (scorer?.kind === `${PEERBENCH_NAMESPACE}/mcq`) {
+    const scorerResult = await scorer.score({
+      response: response.data,
+      choices: testCase.options,
+      correctAnswers: testCase.correctAnswerKeys
     });
-    const score = await FNOLFieldsScoreSchemaV1.newWithId(
-      {
-        responseId: response.id,
-        value: scorerResult.value,
-        explanation: scorerResult.explanation,
-        metadata: scorerResult.metadata,
-        scoringMethod: ScoringMethod.algo,
-        requiredKeys: scorerResult.requiredKeys,
-        presentKeys: scorerResult.presentKeys,
-        missingKeys: scorerResult.missingKeys,
-        mismatchedKeys: scorerResult.mismatchedKeys
-      },
-      scoreIdGenerator
-    );
-    return { response, score };
+    if (scorerResult !== null) {
+      const score = await MCQScoreSchemaV1.newWithId(
+        {
+          scoringMethod: ScoringMethod.algo,
+          value: scorerResult.value,
+          responseId: response.id,
+          extractedAnswers: scorerResult.extractedAnswers,
+          explanation: scorerResult.explanation,
+          metadata: scorerResult.metadata
+        },
+        params.idGenerators?.score ?? idGeneratorUUIDv7
+      );
+      return { response, score };
+    }
   }
-  if (params.scorer?.kind === "llmJudge" && params.runConfig.llmJudgeModel) {
-    const scorerResult = await params.scorer.score({
-      task: "Evaluate whether the FNOL JSON contains the required fields and correct values.",
-      candidateAnswer: response.data,
-      referenceAnswer: JSON.stringify(
-        Object.fromEntries(
-          Object.entries(params.testCase.fieldsToCollect).map(([k, v]) => [
-            k,
-            v.expected
-          ])
+  if (scorer?.kind === `${PEERBENCH_NAMESPACE}/llm-as-a-judge`) {
+    if (!runConfig.llmJudgeModel) {
+      throw new Error(
+        "LLM judge model is required when using LLM as a judge scorer"
+      );
+    }
+    const scorerResult = await scorer.score({
+      model: runConfig.llmJudgeModel,
+      criteria: [
+        {
+          id: "correctness",
+          description: "Is the given answer key matches with one of the correct answer keys?",
+          weight: 1
+        }
+      ],
+      rubric: `Answer text itself or the key (A, B, C) is accepted
+Valid answer keys: ${testCase.correctAnswerKeys.map((key) => `- ${key}`).join("\n")}
+Valid Answer texts: ${testCase.correctAnswerKeys.map((key) => `- ${testCase.options?.[key] ?? ""}`).join("\n")}`,
+      fieldsToExtract: {
+        extractedAnswers: z4.string().array().describe(
+          "The extracted answer keys, valid or invalid (even if the answer text is provided rather than the key)"
         )
-      ),
-      model: params.runConfig.llmJudgeModel,
-      meta: {
-        fieldsToCollect: params.testCase.fieldsToCollect,
-        doneReason
-      }
+      },
+      response: response.data,
+      systemPrompt: runConfig.llmJudgeSystemPrompt?.content
     });
     if (scorerResult !== null) {
-      const score = await FNOLLLMJudgeScoreSchemaV1.newWithId(
+      const score = await MCQScoreSchemaV1.newWithId(
         {
-          responseId: response.id,
+          scoringMethod: ScoringMethod.ai,
           value: scorerResult.value,
+          extractedAnswers: scorerResult.extractedFields.extractedAnswers,
+          responseId: response.id,
           explanation: scorerResult.explanation,
           metadata: scorerResult.metadata,
-          scoringMethod: ScoringMethod.ai,
-          verdict: scorerResult.verdict,
-          scorerAIProvider: scorerResult.provider,
-          scorerAIModelSlug: params.runConfig.llmJudgeModel,
+          scorerAIInputCost: scorerResult.inputCost,
+          scorerAIOutputCost: scorerResult.outputCost,
           scorerAIInputTokensUsed: scorerResult.inputTokensUsed,
           scorerAIOutputTokensUsed: scorerResult.outputTokensUsed,
-          scorerAIInputCost: scorerResult.inputCost,
-          scorerAIOutputCost: scorerResult.outputCost
+          scorerAIProvider: scorerResult.provider,
+          scorerAIModelSlug: runConfig.llmJudgeModel,
+          scorerAISystemPromptId: runConfig.llmJudgeSystemPrompt?.id
         },
-        scoreIdGenerator
+        params.idGenerators?.score ?? idGeneratorUUIDv7
       );
       return { response, score };
     }
   }
   return { response };
 }
-
-// src/benchmarks/fnol/scorer.ts
-function isMissing(value) {
-  return value === void 0 || value === null || value === "";
-}
-function normalizeString(value) {
-  return value.trim();
+function formatMCQ(testCase) {
+  return `Question: ${testCase.question}
+Options:
+${Object.entries(
+    testCase.options ?? {}
+  ).map(([key, value]) => `${key}: ${value}`).join("\n")}`;
 }
-function valuesEqual(expected, actual) {
-  if (typeof expected === "string" && typeof actual === "string") {
-    return normalizeString(expected) === normalizeString(actual);
+function templateMessages(messages, templateVariables) {
+  for (let i = 0; i < messages.length; i++) {
+    const template = Handlebars.compile(messages[i].content);
+    messages[i].content = template(templateVariables);
   }
-  return stableStringify(expected) === stableStringify(actual);
 }
-var FNOLFieldsScorer = class extends AbstractScorer {
-  kind = "fnol.fields";
-  async score(params) {
-    const extracted = params.extracted ?? {};
-    const requiredKeys = Object.entries(params.fieldsToCollect).filter(([, field]) => field.required !== false).map(([key]) => key);
-    const presentKeys = [];
-    const missingKeys = [];
-    const mismatchedKeys = [];
-    for (const key of requiredKeys) {
-      const value = extracted[key];
-      if (isMissing(value)) {
-        missingKeys.push(key);
-        continue;
-      }
-      presentKeys.push(key);
-      const expected = params.fieldsToCollect[key]?.expected;
-      if (expected !== void 0 && !valuesEqual(expected, value)) {
-        mismatchedKeys.push(key);
-      }
-    }
-    const requiredCount = requiredKeys.length;
-    const correctCount = requiredCount - missingKeys.length - mismatchedKeys.length;
-    const score = requiredCount === 0 ? 1 : correctCount / requiredCount;
-    return {
-      value: Math.max(0, Math.min(1, score)),
-      explanation: missingKeys.length === 0 && mismatchedKeys.length === 0 ? "All required fields collected" : "Missing or mismatched fields",
-      requiredKeys,
-      presentKeys,
-      missingKeys,
-      mismatchedKeys,
-      metadata: {
-        requiredCount,
-        presentCount: presentKeys.length,
-        missingCount: missingKeys.length,
-        mismatchedCount: mismatchedKeys.length
-      }
-    };
+
+// src/benchmarks/peerbench/storages/json.ts
+import z5 from "zod";
+var PeerbenchJSONStorage = class extends JSONFileStorage {
+  constructor(config) {
+    super({
+      path: config.path,
+      chunkSize: config.chunkSize,
+      schema: z5.union([
+        MCQTestCaseSchemaV1,
+        MCQResponseSchemaV1,
+        MCQScoreSchemaV1,
+        QATestCaseSchemaV1,
+        QAResponseSchemaV1,
+        QAScoreSchemaV1,
+        MultiTurnTestCaseSchemaV1,
+        MultiTurnResponseSchemaV1,
+        MultiTurnScoreSchemaV1
+      ])
+    });
   }
 };
 export {
-  fnol_exports as fnol,
-  mmlu_pro_exports as mmluPro,
   peerbench_exports as peerbench
 };
 //# sourceMappingURL=index.js.map