peerbench 0.0.1 → 0.0.2-alpha.0

package/dist/benchmarks/index.js

@@ -0,0 +1,915 @@
+import {
+  AbstractDataLoader,
+  GenericJSONArrayDataLoader,
+  stableStringify
+} from "../chunk-HJH3SW3L.js";
+import {
+  AbstractScorer
+} from "../chunk-22HU24QF.js";
+import {
+  bufferToString,
+  idGeneratorUUIDv7,
+  parseResponseAsJSON
+} from "../chunk-DUBKY73H.js";
+import {
+  BaseBenchmarkSpecSchemaV1,
+  BaseScoreSchemaV1,
+  BaseTestCaseSchemaV1,
+  defineBenchmarkSpecSchema,
+  defineScoreSchema,
+  defineTestCaseSchema
+} from "../chunk-232PY7K3.js";
+import {
+  ScoringMethod
+} from "../chunk-ZJWSK4VO.js";
+import {
+  ExtensionLLMAsAJudgeScorerFieldsV1
+} from "../chunk-GVF4YZF3.js";
+import {
+  BaseLLMChatResponseSchemaV1
+} from "../chunk-7TREBPSJ.js";
+import {
+  defineResponseSchema
+} from "../chunk-IUN2IUCS.js";
+import {
+  __export
+} from "../chunk-PZ5AY32C.js";
+
+// src/benchmarks/peerbench/index.ts
+var peerbench_exports = {};
+__export(peerbench_exports, {
+  PeerbenchJSONDataLoader: () => PeerbenchJSONDataLoader,
+  PeerbenchMultipleChoiceResponseSchemaV1: () => PeerbenchMultipleChoiceResponseSchemaV1,
+  PeerbenchMultipleChoiceScoreSchemaV1: () => PeerbenchMultipleChoiceScoreSchemaV1,
+  PeerbenchMultipleChoiceTestCaseSchemaV1: () => PeerbenchMultipleChoiceTestCaseSchemaV1,
+  PeerbenchOpenEndedResponseSchemaV1: () => PeerbenchOpenEndedResponseSchemaV1,
+  PeerbenchOpenEndedScoreSchemaV1: () => PeerbenchOpenEndedScoreSchemaV1,
+  PeerbenchOpenEndedTestCaseSchemaV1: () => PeerbenchOpenEndedTestCaseSchemaV1,
+  runTestCase: () => runTestCase
+});
+
+// src/benchmarks/peerbench/test-cases/mcq.v1.ts
+import { z } from "zod";
+
+// src/benchmarks/peerbench/score.ts
+var PeerbenchBaseScoreSchemaV1 = defineScoreSchema({
+  baseSchema: BaseScoreSchemaV1,
+  fields: {
+    ...ExtensionLLMAsAJudgeScorerFieldsV1
+  }
+});
+
+// src/benchmarks/peerbench/test-cases/mcq.v1.ts
+var PeerbenchMultipleChoiceTestCaseSchemaV1 = defineTestCaseSchema({
+  baseSchema: BaseTestCaseSchemaV1,
+  kind: "pb.ts.mcq",
+  schemaVersion: 1,
+  fields: {
+    question: z.string(),
+    options: z.record(z.string(), z.string()),
+    answer: z.string(),
+    answerKey: z.string()
+  }
+});
+var PeerbenchMultipleChoiceResponseSchemaV1 = defineResponseSchema({
+  baseSchema: BaseLLMChatResponseSchemaV1,
+  kind: "pb.rs.mcq",
+  schemaVersion: 1,
+  fields: {}
+});
+var PeerbenchMultipleChoiceScoreSchemaV1 = defineScoreSchema({
+  baseSchema: PeerbenchBaseScoreSchemaV1,
+  kind: "pb.sc.mcq",
+  schemaVersion: 1,
+  fields: {
+    extractedAnswers: z.array(z.string())
+  }
+});
+
+// src/benchmarks/peerbench/test-cases/open-ended.v1.ts
+import { z as z2 } from "zod";
+var PeerbenchOpenEndedTestCaseSchemaV1 = defineTestCaseSchema({
+  kind: "pb.ts.open-ended",
+  schemaVersion: 1,
+  baseSchema: BaseTestCaseSchemaV1,
+  fields: {
+    question: z2.string(),
+    answer: z2.string().optional()
+  }
+});
+var PeerbenchOpenEndedResponseSchemaV1 = defineResponseSchema({
+  baseSchema: BaseLLMChatResponseSchemaV1,
+  kind: "pb.rs.open-ended",
+  schemaVersion: 1
+});
+var PeerbenchOpenEndedScoreSchemaV1 = defineScoreSchema({
+  baseSchema: PeerbenchBaseScoreSchemaV1,
+  kind: "pb.sc.open-ended",
+  schemaVersion: 1,
+  fields: {}
+});
+
+// src/benchmarks/peerbench/spec.ts
+import z3 from "zod";
+var PeerbenchBenchmarkSpecSchemaV1 = defineBenchmarkSpecSchema({
+  baseSchema: BaseBenchmarkSpecSchemaV1,
+  kind: "pb.benchmark.spec",
+  schemaVersion: 1,
+  fields: {
+    /**
+     * Big text contents that can be referred as <text>{key}</text> in a prompt or system prompt.
+     */
+    blobTexts: z3.record(z3.string(), z3.string()).optional()
+  }
+});
+
+// src/benchmarks/peerbench/loader.ts
+import z4 from "zod";
+var PeerbenchJSONDataLoader = class extends GenericJSONArrayDataLoader {
+  kind = "pb.load.json.data";
+  async loadBenchmarkSpec(params) {
+    const content = bufferToString(params.content);
+    const parsed = PeerbenchBenchmarkSpecSchemaV1.parse(content);
+    return parsed;
+  }
+  testCaseBuilder(data) {
+    const testCaseValidation = z4.union([
+      PeerbenchMultipleChoiceTestCaseSchemaV1,
+      PeerbenchOpenEndedTestCaseSchemaV1
+    ]).safeParse(data);
+    return testCaseValidation.success ? testCaseValidation.data : void 0;
+  }
+  async responseBuilder(data) {
+    const responseValidation = z4.union([
+      PeerbenchMultipleChoiceResponseSchemaV1,
+      PeerbenchOpenEndedResponseSchemaV1
+    ]).safeParse(data);
+    return responseValidation.success ? responseValidation.data : void 0;
+  }
+  async scoreBuilder(data) {
+    const scoreValidation = z4.union([
+      PeerbenchMultipleChoiceScoreSchemaV1,
+      PeerbenchOpenEndedScoreSchemaV1
+    ]).safeParse(data);
+    return scoreValidation.success ? scoreValidation.data : void 0;
+  }
+};
+
+// src/benchmarks/peerbench/runner.ts
+async function runTestCase(params) {
+  const { testCase } = params;
+  const responseIdGenerator = params.idGenerators?.response ?? idGeneratorUUIDv7;
+  const scoreIdGenerator = params.idGenerators?.score ?? idGeneratorUUIDv7;
+  const messages = [];
+  if (params.systemPrompt) {
+    messages.push({
+      role: "system",
+      content: params.systemPrompt.content
+    });
+  }
+  if (testCase.kind === "pb.ts.mcq") {
+    const formattedPrompt = formatMCQPrompt(testCase);
+    messages.push({
+      role: "user",
+      content: formattedPrompt
+    });
+    const providerResponse = await params.provider.forward({
+      model: params.runConfig.model,
+      messages
+    });
+    const response = await PeerbenchMultipleChoiceResponseSchemaV1.newWithId(
+      {
+        data: providerResponse.data,
+        startedAt: providerResponse.startedAt,
+        completedAt: providerResponse.completedAt,
+        testCaseId: testCase.id,
+        modelSlug: params.runConfig.model,
+        provider: params.provider.kind,
+        inputTokensUsed: providerResponse.inputTokensUsed,
+        outputTokensUsed: providerResponse.outputTokensUsed,
+        inputCost: providerResponse.inputCost,
+        outputCost: providerResponse.outputCost
+      },
+      responseIdGenerator
+    );
+    if (params.scorer?.kind === "mcq") {
+      const scorerResult = await params.scorer.score({
+        response: response.data,
+        choices: testCase.options ?? {},
+        correctAnswers: [testCase.answerKey]
+      });
+      if (scorerResult !== null) {
+        const score = await PeerbenchMultipleChoiceScoreSchemaV1.newWithId(
+          {
+            scoringMethod: ScoringMethod.algo,
+            value: scorerResult.value,
+            responseId: response.id,
+            extractedAnswers: scorerResult.extractedAnswers,
+            metadata: response.metadata
+          },
+          scoreIdGenerator
+        );
+        return { response, score };
+      }
+    }
+    return { response };
+  } else if (testCase.kind === "pb.ts.open-ended") {
+    const messages2 = [];
+    if (params.systemPrompt) {
+      messages2.push({
+        role: "system",
+        content: params.systemPrompt.content
+      });
+    }
+    messages2.push({
+      role: "user",
+      content: testCase.question
+    });
+    const providerResponse = await params.provider.forward({
+      model: params.runConfig.model,
+      messages: messages2
+    });
+    const response = await PeerbenchOpenEndedResponseSchemaV1.newWithId(
+      {
+        data: providerResponse.data,
+        startedAt: providerResponse.startedAt,
+        completedAt: providerResponse.completedAt,
+        testCaseId: testCase.id,
+        modelSlug: params.runConfig.model,
+        provider: params.provider.kind,
+        inputTokensUsed: providerResponse.inputTokensUsed,
+        outputTokensUsed: providerResponse.outputTokensUsed,
+        inputCost: providerResponse.inputCost,
+        outputCost: providerResponse.outputCost
+      },
+      responseIdGenerator
+    );
+    if (params.scorer?.kind === "llmJudge" && params.runConfig.llmJudgeModel) {
+      const scorerResult = await params.scorer.score({
+        task: testCase.question,
+        candidateAnswer: response.data,
+        referenceAnswer: testCase.answer,
+        model: params.runConfig.llmJudgeModel
+      });
+      if (scorerResult !== null) {
+        const score = await PeerbenchOpenEndedScoreSchemaV1.newWithId(
+          {
+            scoringMethod: ScoringMethod.ai,
+            value: scorerResult.value,
+            responseId: response.id,
+            explanation: scorerResult.explanation,
+            metadata: scorerResult.metadata,
+            scorerAIProvider: scorerResult.provider,
+            scorerAIModelSlug: params.runConfig.llmJudgeModel,
+            scorerAIInputTokensUsed: scorerResult.inputTokensUsed,
+            scorerAIOutputTokensUsed: scorerResult.outputTokensUsed,
+            scorerAIInputCost: scorerResult.inputCost,
+            scorerAIOutputCost: scorerResult.outputCost
+          },
+          scoreIdGenerator
+        );
+        return { response, score };
+      }
+    }
+    return { response };
+  }
+  throw new Error("Unsupported test case kind");
+}
+function formatMCQPrompt(testCase) {
+  return `Question: ${testCase.question}
+Options:
+${Object.entries(
+    testCase.options ?? {}
+  ).map(([key, value]) => `${key}: ${value}`).join("\n")}`;
+}
+
+// src/benchmarks/mmlu-pro/index.ts
+var mmlu_pro_exports = {};
+__export(mmlu_pro_exports, {
+  BaseMMLUProScoreSchemaV1: () => BaseMMLUProScoreSchemaV1,
+  MMLUProBenchmarkSpecSchemaV1: () => MMLUProBenchmarkSpecSchemaV1,
+  MMLUProJSONDataLoader: () => MMLUProJSONDataLoader,
+  MMLUProMainResponseSchemaV1: () => MMLUProMainResponseSchemaV1,
+  MMLUProMainScoreSchemaV1: () => MMLUProMainScoreSchemaV1,
+  MMLUProMainTestCaseSchemaV1: () => MMLUProMainTestCaseSchemaV1,
+  MMLUProParquetDataLoader: () => MMLUProParquetDataLoader,
+  runTestCase: () => runTestCase2
+});
+
+// src/benchmarks/mmlu-pro/score.ts
+var BaseMMLUProScoreSchemaV1 = defineScoreSchema({
+  baseSchema: BaseScoreSchemaV1,
+  fields: {
+    ...ExtensionLLMAsAJudgeScorerFieldsV1
+  }
+});
+
+// src/benchmarks/mmlu-pro/test-cases/main.v1.ts
+import { z as z5 } from "zod";
+var MMLUProMainTestCaseSchemaV1 = defineTestCaseSchema({
+  baseSchema: BaseTestCaseSchemaV1,
+  kind: "mmlu-pro.ts.main",
+  schemaVersion: 1,
+  fields: {
+    question: z5.string(),
+    options: z5.record(z5.string(), z5.string()),
+    answer: z5.string(),
+    answerKey: z5.string()
+  }
+});
+var MMLUProMainResponseSchemaV1 = defineResponseSchema({
+  baseSchema: BaseLLMChatResponseSchemaV1,
+  kind: "mmlu-pro.rs.main",
+  schemaVersion: 1
+});
+var MMLUProMainScoreSchemaV1 = defineScoreSchema({
+  baseSchema: BaseMMLUProScoreSchemaV1,
+  kind: "mmlu-pro.sc.main",
+  schemaVersion: 1,
+  fields: {
+    extractedAnswers: z5.array(z5.string())
+  }
+});
+
+// src/benchmarks/mmlu-pro/loader.ts
+import z6 from "zod";
+import { parquetReadObjects } from "hyparquet";
+var jsonSchema = z6.object({
+  question_id: z6.coerce.number(),
+  question: z6.string(),
+  options: z6.array(z6.string()),
+  answer: z6.string(),
+  answer_index: z6.coerce.number(),
+  cot_content: z6.string(),
+  category: z6.string(),
+  src: z6.string()
+}).array();
+function mapData(data) {
+  return {
+    responses: [],
+    scores: [],
+    testCases: data.map(
+      (item) => MMLUProMainTestCaseSchemaV1.new({
+        id: `${item.src}-${item.category}-${item.question_id}`,
+        question: item.question,
+        answerKey: item.answer,
+        options: item.options.reduce(
+          (acc, option, index) => {
+            acc[String.fromCharCode(65 + index)] = option;
+            return acc;
+          },
+          {}
+        ),
+        answer: item.options[item.answer_index],
+        metadata: {
+          category: item.category,
+          src: item.src,
+          answer_index: item.answer_index
+        }
+      })
+    )
+  };
+}
+var MMLUProJSONDataLoader = class extends AbstractDataLoader {
+  kind = "mmlu-pro.load.json.data";
+  loadData(params) {
+    const content = typeof params.content === "string" ? params.content : bufferToString(params.content);
+    const parsed = jsonSchema.parse(JSON.parse(content));
+    return mapData(parsed);
+  }
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  loadBenchmarkSpec(params) {
+    throw new Error("Not implemented");
+  }
+};
+var MMLUProParquetDataLoader = class extends AbstractDataLoader {
+  kind = "mmlu-pro.load.parquet.data";
+  async loadData(params) {
+    const data = await parquetReadObjects({
+      file: params.content.buffer
+    });
+    if (!data) {
+      throw new Error("Invalid Parquet file");
+    }
+    return mapData(jsonSchema.parse(data));
+  }
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  loadBenchmarkSpec(params) {
+    throw new Error("Not implemented");
+  }
+};
+
+// src/benchmarks/mmlu-pro/runner.ts
+async function runTestCase2(params) {
+  const { testCase } = params;
+  const responseIdGenerator = params.idGenerators?.response ?? idGeneratorUUIDv7;
+  const scoreIdGenerator = params.idGenerators?.score ?? idGeneratorUUIDv7;
+  const messages = [];
+  if (params.systemPrompt) {
+    messages.push({
+      role: "system",
+      content: params.systemPrompt.content
+    });
+  }
+  if (testCase.kind === "mmlu-pro.ts.main") {
+    const formattedPrompt = formatMCQPrompt2(testCase);
+    messages.push({
+      role: "user",
+      content: formattedPrompt
+    });
+    const providerResponse = await params.provider.forward({
+      model: params.runConfig.model,
+      messages
+    });
+    const response = await MMLUProMainResponseSchemaV1.newWithId(
+      {
+        data: providerResponse.data,
+        startedAt: providerResponse.startedAt,
+        completedAt: providerResponse.completedAt,
+        testCaseId: testCase.id,
+        modelSlug: params.runConfig.model,
+        provider: params.provider.kind,
+        inputTokensUsed: providerResponse.inputTokensUsed,
+        outputTokensUsed: providerResponse.outputTokensUsed,
+        inputCost: providerResponse.inputCost,
+        outputCost: providerResponse.outputCost
+      },
+      responseIdGenerator
+    );
+    if (params.scorer?.kind === "mcq") {
+      const scorerResult = await params.scorer.score({
+        response: response.data,
+        choices: testCase.options ?? {},
+        correctAnswers: [testCase.answerKey]
+      });
+      if (scorerResult !== null) {
+        const score = await MMLUProMainScoreSchemaV1.newWithId(
+          {
+            scoringMethod: ScoringMethod.algo,
+            value: scorerResult.value,
+            responseId: response.id,
+            extractedAnswers: scorerResult.extractedAnswers,
+            metadata: response.metadata
+          },
+          scoreIdGenerator
+        );
+        return { response, score };
+      }
+    }
+    return { response };
+  } else if (testCase.kind === "pb.ts.open-ended") {
+    const messages2 = [];
+    if (params.systemPrompt) {
+      messages2.push({
+        role: "system",
+        content: params.systemPrompt.content
+      });
+    }
+    messages2.push({
+      role: "user",
+      content: testCase.question
+    });
+    const providerResponse = await params.provider.forward({
+      model: params.runConfig.model,
+      messages: messages2
+    });
+    const response = await MMLUProMainResponseSchemaV1.newWithId(
+      {
+        data: providerResponse.data,
+        startedAt: providerResponse.startedAt,
+        completedAt: providerResponse.completedAt,
+        testCaseId: testCase.id,
+        modelSlug: params.runConfig.model,
+        provider: params.provider.kind,
+        inputTokensUsed: providerResponse.inputTokensUsed,
+        outputTokensUsed: providerResponse.outputTokensUsed,
+        inputCost: providerResponse.inputCost,
+        outputCost: providerResponse.outputCost
+      },
+      responseIdGenerator
+    );
+    if (params.scorer?.kind === "llmJudge" && params.runConfig.llmJudgeModel) {
+      const scorerResult = await params.scorer.score({
+        task: testCase.question,
+        candidateAnswer: response.data,
+        referenceAnswer: testCase.answer,
+        model: params.runConfig.llmJudgeModel
+      });
+      if (scorerResult !== null) {
+        const score = await MMLUProMainScoreSchemaV1.newWithId(
+          {
+            scoringMethod: ScoringMethod.ai,
+            value: scorerResult.value,
+            responseId: response.id,
+            explanation: scorerResult.explanation,
+            metadata: scorerResult.metadata,
+            extractedAnswers: [],
+            scorerAIProvider: scorerResult.provider,
+            scorerAIModelSlug: params.runConfig.llmJudgeModel,
+            scorerAIInputTokensUsed: scorerResult.inputTokensUsed,
+            scorerAIOutputTokensUsed: scorerResult.outputTokensUsed,
+            scorerAIInputCost: scorerResult.inputCost,
+            scorerAIOutputCost: scorerResult.outputCost
+          },
+          scoreIdGenerator
+        );
+        return { response, score };
+      }
+    }
+    return { response };
+  }
+  throw new Error("Unsupported test case kind");
+}
+function formatMCQPrompt2(testCase) {
+  return `Question: ${testCase.question}
+Options:
+${Object.entries(
+    testCase.options ?? {}
+  ).map(([key, value]) => `${key}: ${value}`).join("\n")}`;
+}
+
+// src/benchmarks/mmlu-pro/spec.ts
+var MMLUProBenchmarkSpecSchemaV1 = defineBenchmarkSpecSchema({
+  baseSchema: BaseBenchmarkSpecSchemaV1,
+  kind: "mmlu-pro.benchmark.spec",
+  schemaVersion: 1,
+  fields: {}
+});
+
+// src/benchmarks/fnol/index.ts
+var fnol_exports = {};
+__export(fnol_exports, {
+  FNOLBaseScoreSchemaV1: () => FNOLBaseScoreSchemaV1,
+  FNOLConversationMessageSchemaV1: () => FNOLConversationMessageSchemaV1,
+  FNOLDoneReason: () => FNOLDoneReason,
+  FNOLFieldSchemaV1: () => FNOLFieldSchemaV1,
+  FNOLFieldValueType: () => FNOLFieldValueType,
+  FNOLFieldsScoreSchemaV1: () => FNOLFieldsScoreSchemaV1,
+  FNOLFieldsScorer: () => FNOLFieldsScorer,
+  FNOLLLMJudgeScoreSchemaV1: () => FNOLLLMJudgeScoreSchemaV1,
+  FNOLResponseSchemaV1: () => FNOLResponseSchemaV1,
+  FNOLTestCaseSchemaV1: () => FNOLTestCaseSchemaV1,
+  runTestCase: () => runTestCase3
+});
+
+// src/benchmarks/fnol/test-cases/fnol.v1.ts
+import { z as z7 } from "zod";
+
+// src/benchmarks/fnol/score.ts
+var FNOLBaseScoreSchemaV1 = defineScoreSchema({
+  baseSchema: BaseScoreSchemaV1,
+  fields: {
+    ...ExtensionLLMAsAJudgeScorerFieldsV1
+  }
+});
+
+// src/benchmarks/fnol/types.ts
+var FNOLFieldValueType = {
+  string: "string",
+  number: "number",
+  boolean: "boolean",
+  object: "object"
+};
+var FNOLDoneReason = {
+  modelProvidedJson: "modelProvidedJson",
+  reachedMaxTurns: "reachedMaxTurns",
+  forcedFinalJson: "forcedFinalJson"
+};
+
+// src/benchmarks/fnol/test-cases/fnol.v1.ts
+var FNOLFieldSchemaV1 = z7.object({
+  description: z7.string(),
+  required: z7.boolean().optional(),
+  /**
+   * Optional expected value used by the deterministic scorer.
+   * If omitted, the scorer will only check presence.
+   */
+  expected: z7.unknown().optional(),
+  /**
+   * Optional type hint for the model/user simulation.
+   */
+  valueType: z7.enum(FNOLFieldValueType).optional()
+});
+var FNOLTestCaseSchemaV1 = defineTestCaseSchema({
+  baseSchema: BaseTestCaseSchemaV1,
+  kind: "fnol.ts.v1",
+  schemaVersion: 1,
+  fields: {
+    /**
+     * Scenario starter message. This is what the "user" would say initially.
+     */
+    initialUserMessage: z7.string(),
+    /**
+     * Private/structured information about the user and the incident.
+     * This is used by the user simulator LLM to answer the target model questions.
+     */
+    userProfile: z7.record(z7.string(), z7.unknown()),
+    /**
+     * The fields the target model must collect.
+     * Keys are canonical identifiers (e.g. "policyNumber", "dateOfLoss").
+     */
+    fieldsToCollect: z7.record(z7.string(), FNOLFieldSchemaV1),
+    /**
+     * Maximum number of back-and-forth turns (target question + user answer).
+     */
+    maxTurns: z7.number().int().min(1).max(100).default(10)
+  }
+});
+var FNOLConversationMessageSchemaV1 = z7.object({
+  role: z7.enum(["system", "user", "assistant"]),
+  content: z7.string()
+});
+var FNOLResponseSchemaV1 = defineResponseSchema({
+  baseSchema: BaseLLMChatResponseSchemaV1,
+  kind: "fnol.rs.v1",
+  schemaVersion: 1,
+  fields: {
+    /**
+     * Full conversation between the target model and simulated user.
+     */
+    conversation: z7.array(FNOLConversationMessageSchemaV1),
+    turnsUsed: z7.number().int(),
+    doneReason: z7.enum(FNOLDoneReason),
+    /**
+     * Parsed JSON object from the target model's final answer, if available.
+     */
+    extracted: z7.record(z7.string(), z7.unknown()).optional()
+  }
+});
+var FNOLFieldsScoreSchemaV1 = defineScoreSchema({
+  baseSchema: FNOLBaseScoreSchemaV1,
+  kind: "fnol.sc.fields.v1",
+  schemaVersion: 1,
+  fields: {
+    requiredKeys: z7.array(z7.string()),
+    presentKeys: z7.array(z7.string()),
+    missingKeys: z7.array(z7.string()),
+    mismatchedKeys: z7.array(z7.string())
+  }
+});
+var FNOLLLMJudgeScoreSchemaV1 = defineScoreSchema({
+  baseSchema: FNOLBaseScoreSchemaV1,
+  kind: "fnol.sc.llm-judge.v1",
+  schemaVersion: 1,
+  fields: {
+    verdict: z7.enum(["pass", "borderline", "fail"]).optional()
+  }
+});
+
+// src/benchmarks/fnol/runner.ts
+function formatFieldsToCollect(fieldsToCollect) {
+  return Object.entries(fieldsToCollect).map(([key, field]) => {
+    const required = field.required === false ? "optional" : "required";
+    return `- ${key} (${required}): ${field.description}`;
+  }).join("\n");
+}
+function hasAllRequiredFields(params) {
+  const extracted = params.extracted ?? {};
+  for (const [key, field] of Object.entries(params.fieldsToCollect)) {
+    if (field.required === false) continue;
+    const value = extracted[key];
+    if (value === void 0 || value === null || value === "") return false;
+  }
+  return true;
+}
+async function runTestCase3(params) {
+  const responseIdGenerator = params.idGenerators?.response ?? idGeneratorUUIDv7;
+  const scoreIdGenerator = params.idGenerators?.score ?? idGeneratorUUIDv7;
+  const userSimulatorProvider = params.userSimulatorProvider ?? params.provider;
+  const userSimulatorModel = params.runConfig.userSimulatorModel ?? params.runConfig.model;
+  const fieldsToCollectText = formatFieldsToCollect(
+    params.testCase.fieldsToCollect
+  );
+  const conversation = [];
+  if (params.systemPrompt) {
+    conversation.push({
+      role: "system",
+      content: params.systemPrompt.content
+    });
+  }
+  conversation.push({
+    role: "system",
+    content: [
+      "You are an insurance FNOL intake assistant.",
+      "Your job is to ask the user questions to collect the required fields listed below.",
+      "Ask concise questions, one or a few at a time.",
+      "When you have enough information OR when you are told to finish, output ONLY a single JSON object with the collected fields.",
+      "Do not include markdown fences. Do not include additional text outside the JSON.",
+      "",
+      "Fields to collect:",
+      fieldsToCollectText
+    ].join("\n")
+  });
+  conversation.push({
+    role: "user",
+    content: params.testCase.initialUserMessage
+  });
+  let doneReason;
+  let extracted;
+  const startedAt = Date.now();
+  for (let turn = 0; turn < params.testCase.maxTurns; turn++) {
+    const targetReply = await params.provider.forward({
+      model: params.runConfig.model,
+      temperature: params.runConfig.temperature,
+      messages: conversation
+    });
+    conversation.push({
+      role: "assistant",
+      content: targetReply.data
+    });
+    extracted = parseResponseAsJSON(targetReply.data);
+    if (extracted && hasAllRequiredFields({
+      extracted,
+      fieldsToCollect: params.testCase.fieldsToCollect
+    })) {
+      doneReason = FNOLDoneReason.modelProvidedJson;
+      break;
+    }
+    const lastAssistantMessage = targetReply.data;
+    const simulatedUser = await userSimulatorProvider.forward({
+      model: userSimulatorModel,
+      temperature: params.runConfig.userSimulatorTemperature,
+      messages: [
+        {
+          role: "system",
+          content: [
+            "You are simulating a real insurance customer (the user).",
+            "Answer the assistant's questions truthfully using ONLY the provided user profile and incident details.",
+            "If asked about something not present in the profile, say you don't know.",
+            "Be concise and natural. Do not invent new facts.",
+            "",
+            "User profile (JSON):",
+            JSON.stringify(params.testCase.userProfile)
+          ].join("\n")
+        },
+        {
+          role: "user",
+          content: lastAssistantMessage
+        }
+      ]
+    });
+    conversation.push({
+      role: "user",
+      content: simulatedUser.data
+    });
+  }
+  if (!doneReason) {
+    doneReason = FNOLDoneReason.reachedMaxTurns;
+    const forced = await params.provider.forward({
+      model: params.runConfig.model,
+      temperature: params.runConfig.temperature,
+      messages: [
+        ...conversation,
+        {
+          role: "user",
+          content: "Stop the interview now and output ONLY the final JSON object with the collected fields. No extra text."
+        }
+      ]
+    });
+    conversation.push({ role: "assistant", content: forced.data });
+    extracted = parseResponseAsJSON(forced.data);
+    if (extracted) {
+      doneReason = FNOLDoneReason.forcedFinalJson;
+    }
+  }
+  const completedAt = Date.now();
+  const lastAssistant = [...conversation].reverse().find((m) => m.role === "assistant");
+  const response = await FNOLResponseSchemaV1.newWithId(
+    {
+      data: typeof lastAssistant?.content === "string" ? lastAssistant.content : "",
+      startedAt,
+      completedAt,
+      testCaseId: params.testCase.id,
+      modelSlug: params.runConfig.model,
+      provider: params.provider.kind,
+      conversation: conversation.map((m) => ({
+        role: m.role === "user" ? "user" : "assistant",
+        content: String(m.content)
+      })),
+      turnsUsed: conversation.filter((m) => m.role === "assistant").length,
+      doneReason,
+      extracted
+    },
+    responseIdGenerator
+  );
+  if (params.scorer?.kind === "fnol.fields") {
+    const scorerResult = await params.scorer.score({
+      fieldsToCollect: params.testCase.fieldsToCollect,
+      extracted
+    });
+    const score = await FNOLFieldsScoreSchemaV1.newWithId(
+      {
+        responseId: response.id,
+        value: scorerResult.value,
+        explanation: scorerResult.explanation,
+        metadata: scorerResult.metadata,
+        scoringMethod: ScoringMethod.algo,
+        requiredKeys: scorerResult.requiredKeys,
+        presentKeys: scorerResult.presentKeys,
+        missingKeys: scorerResult.missingKeys,
+        mismatchedKeys: scorerResult.mismatchedKeys
+      },
+      scoreIdGenerator
+    );
+    return { response, score };
+  }
+  if (params.scorer?.kind === "llmJudge" && params.runConfig.llmJudgeModel) {
+    const scorerResult = await params.scorer.score({
+      task: "Evaluate whether the FNOL JSON contains the required fields and correct values.",
+      candidateAnswer: response.data,
+      referenceAnswer: JSON.stringify(
+        Object.fromEntries(
+          Object.entries(params.testCase.fieldsToCollect).map(([k, v]) => [
+            k,
+            v.expected
+          ])
+        )
+      ),
+      model: params.runConfig.llmJudgeModel,
+      meta: {
+        fieldsToCollect: params.testCase.fieldsToCollect,
+        doneReason
+      }
+    });
+    if (scorerResult !== null) {
+      const score = await FNOLLLMJudgeScoreSchemaV1.newWithId(
+        {
+          responseId: response.id,
+          value: scorerResult.value,
+          explanation: scorerResult.explanation,
+          metadata: scorerResult.metadata,
+          scoringMethod: ScoringMethod.ai,
+          verdict: scorerResult.verdict,
+          scorerAIProvider: scorerResult.provider,
+          scorerAIModelSlug: params.runConfig.llmJudgeModel,
+          scorerAIInputTokensUsed: scorerResult.inputTokensUsed,
+          scorerAIOutputTokensUsed: scorerResult.outputTokensUsed,
+          scorerAIInputCost: scorerResult.inputCost,
+          scorerAIOutputCost: scorerResult.outputCost
+        },
+        scoreIdGenerator
+      );
+      return { response, score };
+    }
+  }
+  return { response };
+}
+
+// src/benchmarks/fnol/scorer.ts
+function isMissing(value) {
+  return value === void 0 || value === null || value === "";
+}
+function normalizeString(value) {
+  return value.trim();
+}
+function valuesEqual(expected, actual) {
+  if (typeof expected === "string" && typeof actual === "string") {
+    return normalizeString(expected) === normalizeString(actual);
+  }
+  return stableStringify(expected) === stableStringify(actual);
+}
+var FNOLFieldsScorer = class extends AbstractScorer {
+  kind = "fnol.fields";
+  async score(params) {
+    const extracted = params.extracted ?? {};
+    const requiredKeys = Object.entries(params.fieldsToCollect).filter(([, field]) => field.required !== false).map(([key]) => key);
+    const presentKeys = [];
+    const missingKeys = [];
+    const mismatchedKeys = [];
+    for (const key of requiredKeys) {
+      const value = extracted[key];
+      if (isMissing(value)) {
+        missingKeys.push(key);
+        continue;
+      }
+      presentKeys.push(key);
+      const expected = params.fieldsToCollect[key]?.expected;
+      if (expected !== void 0 && !valuesEqual(expected, value)) {
+        mismatchedKeys.push(key);
+      }
+    }
+    const requiredCount = requiredKeys.length;
+    const correctCount = requiredCount - missingKeys.length - mismatchedKeys.length;
+    const score = requiredCount === 0 ? 1 : correctCount / requiredCount;
+    return {
+      value: Math.max(0, Math.min(1, score)),
+      explanation: missingKeys.length === 0 && mismatchedKeys.length === 0 ? "All required fields collected" : "Missing or mismatched fields",
+      requiredKeys,
+      presentKeys,
+      missingKeys,
+      mismatchedKeys,
+      metadata: {
+        requiredCount,
+        presentCount: presentKeys.length,
+        missingCount: missingKeys.length,
+        mismatchedCount: mismatchedKeys.length
+      }
+    };
+  }
+};
+export {
+  fnol_exports as fnol,
+  mmlu_pro_exports as mmluPro,
+  peerbench_exports as peerbench
+};
+//# sourceMappingURL=index.js.map