peerbench 0.0.2-alpha.0 → 0.0.2-alpha.2

package/dist/benchmarks/index.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/benchmarks/peerbench/index.ts","../../src/benchmarks/peerbench/test-cases/mcq.v1.ts","../../src/benchmarks/peerbench/score.ts","../../src/benchmarks/peerbench/test-cases/open-ended.v1.ts","../../src/benchmarks/peerbench/spec.ts","../../src/benchmarks/peerbench/loader.ts","../../src/benchmarks/peerbench/runner.ts","../../src/benchmarks/mmlu-pro/index.ts","../../src/benchmarks/mmlu-pro/score.ts","../../src/benchmarks/mmlu-pro/test-cases/main.v1.ts","../../src/benchmarks/mmlu-pro/loader.ts","../../src/benchmarks/mmlu-pro/runner.ts","../../src/benchmarks/mmlu-pro/spec.ts","../../src/benchmarks/fnol/index.ts","../../src/benchmarks/fnol/test-cases/fnol.v1.ts","../../src/benchmarks/fnol/score.ts","../../src/benchmarks/fnol/types.ts","../../src/benchmarks/fnol/runner.ts","../../src/benchmarks/fnol/scorer.ts"],"sourcesContent":["export * from \"./loader\";\nexport * from \"./runner\";\nexport * from \"./test-cases/mcq.v1\";\nexport * from \"./test-cases/open-ended.v1\";\n","import { z } from \"zod\";\nimport {\n  BaseTestCaseSchemaV1,\n  defineResponseSchema,\n  defineScoreSchema,\n  defineTestCaseSchema,\n} from \"@/schemas\";\nimport { BaseLLMChatResponseSchemaV1 } from \"@/schemas/llm/response\";\nimport { PeerbenchBaseScoreSchemaV1 } from \"../score\";\n\n// Test case schema\nexport const PeerbenchMultipleChoiceTestCaseSchemaV1 = defineTestCaseSchema({\n  baseSchema: BaseTestCaseSchemaV1,\n\n  kind: \"pb.ts.mcq\",\n  schemaVersion: 1,\n  fields: {\n    question: z.string(),\n    options: z.record(z.string(), z.string()),\n    answer: z.string(),\n    answerKey: z.string(),\n  },\n});\nexport type PeerbenchMultipleChoiceTestCaseV1 = z.infer<\n  typeof PeerbenchMultipleChoiceTestCaseSchemaV1\n>;\n\n// Response schema\nexport const PeerbenchMultipleChoiceResponseSchemaV1 = defineResponseSchema({\n  baseSchema: BaseLLMChatResponseSchemaV1,\n  kind: \"pb.rs.mcq\",\n  schemaVersion: 1,\n  fields: {},\n});\nexport type PeerbenchMultipleChoiceResponseV1 = z.infer<\n  typeof PeerbenchMultipleChoiceResponseSchemaV1\n>;\n\n// Score schema\nexport const PeerbenchMultipleChoiceScoreSchemaV1 = defineScoreSchema({\n  baseSchema: PeerbenchBaseScoreSchemaV1,\n  kind: \"pb.sc.mcq\",\n  schemaVersion: 1,\n  fields: {\n    extractedAnswers: z.array(z.string()),\n  },\n});\nexport type PeerbenchMultipleChoiceScoreV1 = z.infer<\n  typeof PeerbenchMultipleChoiceScoreSchemaV1\n>;\n","import { BaseScoreSchemaV1, defineScoreSchema } from \"@/schemas\";\nimport { ExtensionLLMAsAJudgeScorerFieldsV1 } from \"@/schemas/extensions/score/llm-as-a-judge-scorer\";\n\nexport const PeerbenchBaseScoreSchemaV1 = defineScoreSchema({\n  baseSchema: BaseScoreSchemaV1,\n  fields: {\n    ...ExtensionLLMAsAJudgeScorerFieldsV1,\n  },\n});\n","import { defineResponseSchema, defineScoreSchema } from \"@/schemas\";\nimport { BaseLLMChatResponseSchemaV1 } from \"@/schemas/llm/response\";\nimport {\n  BaseTestCaseSchemaV1,\n  defineTestCaseSchema,\n} from \"@/schemas/test-case\";\nimport { z } from \"zod\";\nimport { PeerbenchBaseScoreSchemaV1 } from \"../score\";\n\n// Test case schema\nexport const PeerbenchOpenEndedTestCaseSchemaV1 = defineTestCaseSchema({\n  kind: \"pb.ts.open-ended\",\n  schemaVersion: 1,\n  baseSchema: BaseTestCaseSchemaV1,\n  fields: {\n    question: z.string(),\n    answer: z.string().optional(),\n  },\n});\nexport type PeerbenchOpenEndedTestCaseV1 = z.infer<\n  typeof PeerbenchOpenEndedTestCaseSchemaV1\n>;\n\n// Response schema\nexport const PeerbenchOpenEndedResponseSchemaV1 = defineResponseSchema({\n  baseSchema: BaseLLMChatResponseSchemaV1,\n  kind: \"pb.rs.open-ended\",\n  schemaVersion: 1,\n});\nexport type PeerbenchOpenEndedResponseV1 = z.infer<\n  typeof PeerbenchOpenEndedResponseSchemaV1\n>;\n\n// Score schema\nexport const PeerbenchOpenEndedScoreSchemaV1 = defineScoreSchema({\n  baseSchema: PeerbenchBaseScoreSchemaV1,\n  kind: \"pb.sc.open-ended\",\n  schemaVersion: 1,\n  fields: {},\n});\nexport type PeerbenchOpenEndedScoreV1 = z.infer<\n  typeof PeerbenchOpenEndedScoreSchemaV1\n>;\n","import {\n  BaseBenchmarkSpecSchemaV1,\n  defineBenchmarkSpecSchema,\n} from \"@/schemas/benchmark-spec\";\nimport z from \"zod\";\n\nexport const PeerbenchBenchmarkSpecSchemaV1 = defineBenchmarkSpecSchema({\n  baseSchema: BaseBenchmarkSpecSchemaV1,\n  kind: \"pb.benchmark.spec\",\n  schemaVersion: 1,\n  fields: {\n    /**\n     * Big text contents that can be referred as <text>{key}</text> in a prompt or system prompt.\n     */\n    blobTexts: z.record(z.string(), z.string()).optional(),\n  },\n});\nexport type PeerbenchBenchmarkSpecV1 = z.infer<\n  typeof PeerbenchBenchmarkSpecSchemaV1\n>;\n","import { GenericJSONArrayDataLoader } from \"@/loaders/generic-array\";\nimport {\n  PeerbenchMultipleChoiceResponseSchemaV1,\n  PeerbenchMultipleChoiceScoreSchemaV1,\n  PeerbenchMultipleChoiceTestCaseSchemaV1,\n  PeerbenchMultipleChoiceResponseV1,\n  PeerbenchMultipleChoiceScoreV1,\n  PeerbenchMultipleChoiceTestCaseV1,\n} from \"./test-cases/mcq.v1\";\nimport {\n  PeerbenchOpenEndedResponseSchemaV1,\n  PeerbenchOpenEndedScoreSchemaV1,\n  PeerbenchOpenEndedTestCaseSchemaV1,\n  PeerbenchOpenEndedResponseV1,\n  PeerbenchOpenEndedScoreV1,\n  PeerbenchOpenEndedTestCaseV1,\n} from \"./test-cases/open-ended.v1\";\nimport {\n  PeerbenchBenchmarkSpecSchemaV1,\n  PeerbenchBenchmarkSpecV1,\n} from \"./spec\";\nimport z from \"zod\";\nimport { bufferToString } from \"@/utils\";\n\nexport class PeerbenchJSONDataLoader extends GenericJSONArrayDataLoader<\n  PeerbenchMultipleChoiceTestCaseV1 | PeerbenchOpenEndedTestCaseV1,\n  PeerbenchMultipleChoiceResponseV1 | PeerbenchOpenEndedResponseV1,\n  PeerbenchMultipleChoiceScoreV1 | PeerbenchOpenEndedScoreV1\n> {\n  override readonly kind = \"pb.load.json.data\";\n\n  async loadBenchmarkSpec(params: {\n    content: Uint8Array;\n  }): Promise<PeerbenchBenchmarkSpecV1> {\n    const content = bufferToString(params.content);\n    const parsed = PeerbenchBenchmarkSpecSchemaV1.parse(content);\n\n    return parsed;\n  }\n\n  protected override testCaseBuilder(data: any) {\n    const testCaseValidation = z\n      .union([\n        PeerbenchMultipleChoiceTestCaseSchemaV1,\n        PeerbenchOpenEndedTestCaseSchemaV1,\n      ])\n      .safeParse(data);\n    return testCaseValidation.success ? testCaseValidation.data : undefined;\n  }\n\n  protected override async responseBuilder(data: any) {\n    const responseValidation = z\n      .union([\n        PeerbenchMultipleChoiceResponseSchemaV1,\n        PeerbenchOpenEndedResponseSchemaV1,\n      ])\n      .safeParse(data);\n    return responseValidation.success ? responseValidation.data : undefined;\n  }\n\n  protected override async scoreBuilder(data: any) {\n    const scoreValidation = z\n      .union([\n        PeerbenchMultipleChoiceScoreSchemaV1,\n        PeerbenchOpenEndedScoreSchemaV1,\n      ])\n      .safeParse(data);\n    return scoreValidation.success ? scoreValidation.data : undefined;\n  }\n}\n","import { ChatCompletionMessageParam } from \"openai/resources/chat/completions\";\nimport { idGeneratorUUIDv7 } from \"@/utils/id-generator\";\nimport { IdGenerator, RunnerResult } from \"@/types\";\nimport { AbstractLLMProvider } from \"@/providers/abstract/llm\";\nimport { MCQScorer } from \"@/scorers/mcq\";\nimport { LLMJudgeScorer } from \"@/scorers/llm-judge\";\nimport {\n  PeerbenchMultipleChoiceResponseSchemaV1,\n  PeerbenchMultipleChoiceResponseV1,\n  PeerbenchMultipleChoiceScoreSchemaV1,\n  PeerbenchMultipleChoiceScoreV1,\n  PeerbenchMultipleChoiceTestCaseV1,\n} from \"./test-cases/mcq.v1\";\nimport { ScoringMethod } from \"@/types\";\nimport {\n  PeerbenchOpenEndedResponseSchemaV1,\n  PeerbenchOpenEndedResponseV1,\n  PeerbenchOpenEndedScoreSchemaV1,\n  PeerbenchOpenEndedScoreV1,\n  PeerbenchOpenEndedTestCaseV1,\n} from \"./test-cases/open-ended.v1\";\nimport { PeerbenchBenchmarkSpecV1 } from \"./spec\";\nimport { SimpleSystemPromptV1 } from \"@/schemas/llm\";\n\ntype ResponseTypes =\n  | PeerbenchMultipleChoiceResponseV1\n  | PeerbenchOpenEndedResponseV1;\ntype ScoreTypes = PeerbenchMultipleChoiceScoreV1 | PeerbenchOpenEndedScoreV1;\ntype TestCaseTypes =\n  | PeerbenchMultipleChoiceTestCaseV1\n  | PeerbenchOpenEndedTestCaseV1;\n\nexport async function runTestCase(params: {\n  testCase: TestCaseTypes;\n  provider: AbstractLLMProvider;\n  scorer?: MCQScorer | LLMJudgeScorer;\n  spec?: PeerbenchBenchmarkSpecV1;\n  runConfig: {\n    model: string;\n    llmJudgeModel?: string;\n  };\n  systemPrompt?: SimpleSystemPromptV1;\n  idGenerators?: {\n    response?: IdGenerator;\n    score?: IdGenerator;\n  };\n}): Promise<RunnerResult<ResponseTypes, ScoreTypes>> {\n  const { testCase } = params;\n  const responseIdGenerator =\n    params.idGenerators?.response ?? idGeneratorUUIDv7;\n  const scoreIdGenerator = params.idGenerators?.score ?? idGeneratorUUIDv7;\n  const messages: ChatCompletionMessageParam[] = [];\n\n  if (params.systemPrompt) {\n    messages.push({\n      role: \"system\",\n      content: params.systemPrompt.content,\n    });\n  }\n\n  if (testCase.kind === \"pb.ts.mcq\") {\n    const formattedPrompt = formatMCQPrompt(testCase);\n\n    messages.push({\n      role: \"user\",\n      content: formattedPrompt,\n    });\n\n    const providerResponse = await params.provider.forward({\n      model: params.runConfig.model,\n      messages,\n    });\n\n    const response = await PeerbenchMultipleChoiceResponseSchemaV1.newWithId(\n      {\n        data: providerResponse.data,\n        startedAt: providerResponse.startedAt,\n        completedAt: providerResponse.completedAt,\n        testCaseId: testCase.id,\n        modelSlug: params.runConfig.model,\n        provider: params.provider.kind,\n\n        inputTokensUsed: providerResponse.inputTokensUsed,\n        outputTokensUsed: providerResponse.outputTokensUsed,\n        inputCost: providerResponse.inputCost,\n        outputCost: providerResponse.outputCost,\n      },\n      responseIdGenerator\n    );\n\n    if (params.scorer?.kind === \"mcq\") {\n      const scorerResult = await params.scorer.score({\n        response: response.data,\n        choices: testCase.options ?? {},\n        correctAnswers: [testCase.answerKey],\n      });\n\n      if (scorerResult !== null) {\n        const score = await PeerbenchMultipleChoiceScoreSchemaV1.newWithId(\n          {\n            scoringMethod: ScoringMethod.algo,\n            value: scorerResult.value,\n            responseId: response.id,\n            extractedAnswers: scorerResult.extractedAnswers,\n            metadata: response.metadata,\n          },\n          scoreIdGenerator\n        );\n\n        return { response, score };\n      }\n    }\n\n    return { response };\n  } else if (testCase.kind === \"pb.ts.open-ended\") {\n    const messages: ChatCompletionMessageParam[] = [];\n\n    if (params.systemPrompt) {\n      messages.push({\n        role: \"system\",\n        content: params.systemPrompt.content,\n      });\n    }\n\n    messages.push({\n      role: \"user\",\n      content: testCase.question,\n    });\n\n    const providerResponse = await params.provider.forward({\n      model: params.runConfig.model,\n      messages,\n    });\n\n    const response = await PeerbenchOpenEndedResponseSchemaV1.newWithId(\n      {\n        data: providerResponse.data,\n        startedAt: providerResponse.startedAt,\n        completedAt: providerResponse.completedAt,\n        testCaseId: testCase.id,\n        modelSlug: params.runConfig.model,\n        provider: params.provider.kind,\n\n        inputTokensUsed: providerResponse.inputTokensUsed,\n        outputTokensUsed: providerResponse.outputTokensUsed,\n        inputCost: providerResponse.inputCost,\n        outputCost: providerResponse.outputCost,\n      },\n      responseIdGenerator\n    );\n\n    if (params.scorer?.kind === \"llmJudge\" && params.runConfig.llmJudgeModel) {\n      const scorerResult = await params.scorer.score({\n        task: testCase.question,\n        candidateAnswer: response.data,\n        referenceAnswer: testCase.answer,\n        model: params.runConfig.llmJudgeModel,\n      });\n\n      if (scorerResult !== null) {\n        const score = await PeerbenchOpenEndedScoreSchemaV1.newWithId(\n          {\n            scoringMethod: ScoringMethod.ai,\n            value: scorerResult.value,\n            responseId: response.id,\n            explanation: scorerResult.explanation,\n            metadata: scorerResult.metadata,\n\n            scorerAIProvider: scorerResult.provider,\n            scorerAIModelSlug: params.runConfig.llmJudgeModel,\n            scorerAIInputTokensUsed: scorerResult.inputTokensUsed,\n            scorerAIOutputTokensUsed: scorerResult.outputTokensUsed,\n            scorerAIInputCost: scorerResult.inputCost,\n            scorerAIOutputCost: scorerResult.outputCost,\n          },\n          scoreIdGenerator\n        );\n\n        return { response, score };\n      }\n    }\n\n    return { response };\n  }\n\n  throw new Error(\"Unsupported test case kind\");\n}\n\nfunction formatMCQPrompt(testCase: PeerbenchMultipleChoiceTestCaseV1) {\n  return `Question: ${testCase.question}\\nOptions:\\n${Object.entries(\n    testCase.options ?? {}\n  )\n    .map(([key, value]) => `${key}: ${value}`)\n    .join(\"\\n\")}`;\n}\n","export * from \"./loader\";\nexport * from \"./runner\";\nexport * from \"./score\";\nexport * from \"./spec\";\nexport * from \"./test-cases/main.v1\";\n","import { BaseScoreSchemaV1, defineScoreSchema } from \"@/schemas\";\nimport { ExtensionLLMAsAJudgeScorerFieldsV1 } from \"@/schemas/extensions/score/llm-as-a-judge-scorer\";\n\nexport const BaseMMLUProScoreSchemaV1 = defineScoreSchema({\n  baseSchema: BaseScoreSchemaV1,\n  fields: {\n    ...ExtensionLLMAsAJudgeScorerFieldsV1,\n  },\n});\n","import {\n  BaseTestCaseSchemaV1,\n  defineTestCaseSchema,\n} from \"@/schemas/test-case\";\nimport { BaseMMLUProScoreSchemaV1 } from \"../score\";\nimport { BaseLLMChatResponseSchemaV1 } from \"@/schemas/llm/response\";\nimport { defineResponseSchema, defineScoreSchema } from \"@/schemas\";\nimport { z } from \"zod\";\n\n// Test case schema\nexport const MMLUProMainTestCaseSchemaV1 = defineTestCaseSchema({\n  baseSchema: BaseTestCaseSchemaV1,\n  kind: \"mmlu-pro.ts.main\",\n  schemaVersion: 1,\n  fields: {\n    question: z.string(),\n    options: z.record(z.string(), z.string()),\n    answer: z.string(),\n    answerKey: z.string(),\n  },\n});\nexport type MMLUProMainTestCaseV1 = z.infer<typeof MMLUProMainTestCaseSchemaV1>;\n\n// Response schema\nexport const MMLUProMainResponseSchemaV1 = defineResponseSchema({\n  baseSchema: BaseLLMChatResponseSchemaV1,\n  kind: \"mmlu-pro.rs.main\",\n  schemaVersion: 1,\n});\nexport type MMLUProMainResponseV1 = z.infer<typeof MMLUProMainResponseSchemaV1>;\n\n// Score schema\nexport const MMLUProMainScoreSchemaV1 = defineScoreSchema({\n  baseSchema: BaseMMLUProScoreSchemaV1,\n  kind: \"mmlu-pro.sc.main\",\n  schemaVersion: 1,\n  fields: {\n    extractedAnswers: z.array(z.string()),\n  },\n});\nexport type MMLUProMainScoreV1 = z.infer<typeof MMLUProMainScoreSchemaV1>;\n","import { bufferToString } from \"@/utils\";\nimport {\n  MMLUProMainResponseV1,\n  MMLUProMainScoreV1,\n  MMLUProMainTestCaseSchemaV1,\n  MMLUProMainTestCaseV1,\n} from \"./test-cases/main.v1\";\nimport z from \"zod\";\nimport { AbstractDataLoader } from \"@/loaders/abstract/data\";\nimport { LoaderResult } from \"@/loaders/abstract/loader\";\nimport { parquetReadObjects } from \"hyparquet\";\nimport { MMLUProBenchmarkSpecV1 } from \"./spec\";\n\nconst jsonSchema = z\n  .object({\n    question_id: z.coerce.number(),\n    question: z.string(),\n    options: z.array(z.string()),\n    answer: z.string(),\n    answer_index: z.coerce.number(),\n    cot_content: z.string(),\n    category: z.string(),\n    src: z.string(),\n  })\n  .array();\n\nfunction mapData(\n  data: z.infer<typeof jsonSchema>\n): LoaderResult<\n  MMLUProMainTestCaseV1,\n  MMLUProMainResponseV1,\n  MMLUProMainScoreV1\n> {\n  return {\n    responses: [],\n    scores: [],\n    testCases: data.map((item) =>\n      MMLUProMainTestCaseSchemaV1.new({\n        id: `${item.src}-${item.category}-${item.question_id}`,\n        question: item.question,\n        answerKey: item.answer,\n        options: item.options.reduce(\n          (acc, option, index) => {\n            acc[String.fromCharCode(65 + index)] = option;\n            return acc;\n          },\n          {} as Record<string, string>\n        ),\n        answer: item.options[item.answer_index]!,\n        metadata: {\n          category: item.category,\n          src: item.src,\n          answer_index: item.answer_index,\n        },\n      })\n    ),\n  };\n}\n\nexport class MMLUProJSONDataLoader extends AbstractDataLoader {\n  override readonly kind = \"mmlu-pro.load.json.data\";\n\n  override loadData(params: {\n    content: Uint8Array;\n  }): LoaderResult<\n    MMLUProMainTestCaseV1,\n    MMLUProMainResponseV1,\n    MMLUProMainScoreV1\n  > {\n    const content =\n      typeof params.content === \"string\"\n        ? params.content\n        : bufferToString(params.content);\n\n    const parsed = jsonSchema.parse(JSON.parse(content));\n    return mapData(parsed);\n  }\n\n  // eslint-disable-next-line @typescript-eslint/no-unused-vars\n  override loadBenchmarkSpec(params: {\n    content: Uint8Array;\n  }): Promise<MMLUProBenchmarkSpecV1> {\n    throw new Error(\"Not implemented\");\n  }\n}\n\nexport class MMLUProParquetDataLoader extends AbstractDataLoader {\n  override readonly kind = \"mmlu-pro.load.parquet.data\";\n\n  override async loadData(params: {\n    content: Uint8Array;\n  }): Promise<\n    LoaderResult<\n      MMLUProMainTestCaseV1,\n      MMLUProMainResponseV1,\n      MMLUProMainScoreV1\n    >\n  > {\n    const data = await parquetReadObjects({\n      file: params.content.buffer as ArrayBuffer,\n    });\n    if (!data) {\n      throw new Error(\"Invalid Parquet file\");\n    }\n\n    return mapData(jsonSchema.parse(data));\n  }\n\n  // eslint-disable-next-line @typescript-eslint/no-unused-vars\n  override loadBenchmarkSpec(params: {\n    content: Uint8Array;\n  }): Promise<MMLUProBenchmarkSpecV1> {\n    throw new Error(\"Not implemented\");\n  }\n}\n","import { ChatCompletionMessageParam } from \"openai/resources/chat/completions\";\nimport { idGeneratorUUIDv7 } from \"@/utils/id-generator\";\nimport { IdGenerator, RunnerResult } from \"@/types\";\nimport { AbstractLLMProvider } from \"@/providers/abstract/llm\";\nimport { MCQScorer } from \"@/scorers/mcq\";\nimport { LLMJudgeScorer } from \"@/scorers/llm-judge\";\nimport { ScoringMethod } from \"@/types\";\nimport { SimpleSystemPromptV1 } from \"@/schemas/llm\";\nimport {\n  MMLUProMainResponseV1,\n  MMLUProMainTestCaseV1,\n  MMLUProMainScoreV1,\n  MMLUProMainResponseSchemaV1,\n  MMLUProMainScoreSchemaV1,\n} from \"./test-cases/main.v1\";\nimport { MMLUProBenchmarkSpecV1 } from \"./spec\";\n\nexport async function runTestCase(params: {\n  testCase: MMLUProMainTestCaseV1;\n  provider: AbstractLLMProvider;\n  scorer?: MCQScorer | LLMJudgeScorer;\n  spec?: MMLUProBenchmarkSpecV1;\n  runConfig: {\n    model: string;\n    llmJudgeModel?: string;\n  };\n  systemPrompt?: SimpleSystemPromptV1;\n  idGenerators?: {\n    response?: IdGenerator;\n    score?: IdGenerator;\n  };\n}): Promise<RunnerResult<MMLUProMainResponseV1, MMLUProMainScoreV1>> {\n  const { testCase } = params;\n  const responseIdGenerator =\n    params.idGenerators?.response ?? idGeneratorUUIDv7;\n  const scoreIdGenerator = params.idGenerators?.score ?? idGeneratorUUIDv7;\n  const messages: ChatCompletionMessageParam[] = [];\n\n  if (params.systemPrompt) {\n    messages.push({\n      role: \"system\",\n      content: params.systemPrompt.content,\n    });\n  }\n\n  if (testCase.kind === \"mmlu-pro.ts.main\") {\n    const formattedPrompt = formatMCQPrompt(testCase);\n\n    messages.push({\n      role: \"user\",\n      content: formattedPrompt,\n    });\n\n    const providerResponse = await params.provider.forward({\n      model: params.runConfig.model,\n      messages,\n    });\n\n    const response = await MMLUProMainResponseSchemaV1.newWithId(\n      {\n        data: providerResponse.data,\n        startedAt: providerResponse.startedAt,\n        completedAt: providerResponse.completedAt,\n        testCaseId: testCase.id,\n        modelSlug: params.runConfig.model,\n        provider: params.provider.kind,\n\n        inputTokensUsed: providerResponse.inputTokensUsed,\n        outputTokensUsed: providerResponse.outputTokensUsed,\n        inputCost: providerResponse.inputCost,\n        outputCost: providerResponse.outputCost,\n      },\n      responseIdGenerator\n    );\n\n    if (params.scorer?.kind === \"mcq\") {\n      const scorerResult = await params.scorer.score({\n        response: response.data,\n        choices: testCase.options ?? {},\n        correctAnswers: [testCase.answerKey],\n      });\n\n      if (scorerResult !== null) {\n        const score = await MMLUProMainScoreSchemaV1.newWithId(\n          {\n            scoringMethod: ScoringMethod.algo,\n            value: scorerResult.value,\n            responseId: response.id,\n            extractedAnswers: scorerResult.extractedAnswers,\n            metadata: response.metadata,\n          },\n          scoreIdGenerator\n        );\n\n        return { response, score };\n      }\n    }\n\n    return { response };\n  } else if (testCase.kind === \"pb.ts.open-ended\") {\n    const messages: ChatCompletionMessageParam[] = [];\n\n    if (params.systemPrompt) {\n      messages.push({\n        role: \"system\",\n        content: params.systemPrompt.content,\n      });\n    }\n\n    messages.push({\n      role: \"user\",\n      content: testCase.question,\n    });\n\n    const providerResponse = await params.provider.forward({\n      model: params.runConfig.model,\n      messages,\n    });\n\n    const response = await MMLUProMainResponseSchemaV1.newWithId(\n      {\n        data: providerResponse.data,\n        startedAt: providerResponse.startedAt,\n        completedAt: providerResponse.completedAt,\n        testCaseId: testCase.id,\n        modelSlug: params.runConfig.model,\n        provider: params.provider.kind,\n\n        inputTokensUsed: providerResponse.inputTokensUsed,\n        outputTokensUsed: providerResponse.outputTokensUsed,\n        inputCost: providerResponse.inputCost,\n        outputCost: providerResponse.outputCost,\n      },\n      responseIdGenerator\n    );\n\n    if (params.scorer?.kind === \"llmJudge\" && params.runConfig.llmJudgeModel) {\n      const scorerResult = await params.scorer.score({\n        task: testCase.question,\n        candidateAnswer: response.data,\n        referenceAnswer: testCase.answer,\n        model: params.runConfig.llmJudgeModel,\n      });\n\n      if (scorerResult !== null) {\n        const score = await MMLUProMainScoreSchemaV1.newWithId(\n          {\n            scoringMethod: ScoringMethod.ai,\n            value: scorerResult.value,\n            responseId: response.id,\n            explanation: scorerResult.explanation,\n            metadata: scorerResult.metadata,\n            extractedAnswers: [],\n\n            scorerAIProvider: scorerResult.provider,\n            scorerAIModelSlug: params.runConfig.llmJudgeModel,\n            scorerAIInputTokensUsed: scorerResult.inputTokensUsed,\n            scorerAIOutputTokensUsed: scorerResult.outputTokensUsed,\n            scorerAIInputCost: scorerResult.inputCost,\n            scorerAIOutputCost: scorerResult.outputCost,\n          },\n          scoreIdGenerator\n        );\n\n        return { response, score };\n      }\n    }\n\n    return { response };\n  }\n\n  throw new Error(\"Unsupported test case kind\");\n}\n\nfunction formatMCQPrompt(testCase: MMLUProMainTestCaseV1) {\n  return `Question: ${testCase.question}\\nOptions:\\n${Object.entries(\n    testCase.options ?? {}\n  )\n    .map(([key, value]) => `${key}: ${value}`)\n    .join(\"\\n\")}`;\n}\n","import {\n  BaseBenchmarkSpecSchemaV1,\n  defineBenchmarkSpecSchema,\n} from \"@/schemas/benchmark-spec\";\nimport z from \"zod\";\n\nexport const MMLUProBenchmarkSpecSchemaV1 = defineBenchmarkSpecSchema({\n  baseSchema: BaseBenchmarkSpecSchemaV1,\n  kind: \"mmlu-pro.benchmark.spec\",\n  schemaVersion: 1,\n  fields: {},\n});\nexport type MMLUProBenchmarkSpecV1 = z.infer<\n  typeof MMLUProBenchmarkSpecSchemaV1\n>;\n","export * from \"./runner\";\nexport * from \"./scorer\";\nexport * from \"./score\";\nexport * from \"./types\";\n\nexport * from \"./test-cases/fnol.v1\";\n","import { z } from \"zod\";\nimport {\n  BaseTestCaseSchemaV1,\n  defineResponseSchema,\n  defineScoreSchema,\n  defineTestCaseSchema,\n} from \"@/schemas\";\nimport { BaseLLMChatResponseSchemaV1 } from \"@/schemas/llm/response\";\nimport { FNOLBaseScoreSchemaV1 } from \"../score\";\nimport { FNOLDoneReason, FNOLFieldValueType } from \"../types\";\n\nexport const FNOLFieldSchemaV1 = z.object({\n  description: z.string(),\n  required: z.boolean().optional(),\n  /**\n   * Optional expected value used by the deterministic scorer.\n   * If omitted, the scorer will only check presence.\n   */\n  expected: z.unknown().optional(),\n  /**\n   * Optional type hint for the model/user simulation.\n   */\n  valueType: z.enum(FNOLFieldValueType).optional(),\n});\n\nexport const FNOLTestCaseSchemaV1 = defineTestCaseSchema({\n  baseSchema: BaseTestCaseSchemaV1,\n  kind: \"fnol.ts.v1\",\n  schemaVersion: 1,\n  fields: {\n    /**\n     * Scenario starter message. This is what the \"user\" would say initially.\n     */\n    initialUserMessage: z.string(),\n\n    /**\n     * Private/structured information about the user and the incident.\n     * This is used by the user simulator LLM to answer the target model questions.\n     */\n    userProfile: z.record(z.string(), z.unknown()),\n\n    /**\n     * The fields the target model must collect.\n     * Keys are canonical identifiers (e.g. \"policyNumber\", \"dateOfLoss\").\n     */\n    fieldsToCollect: z.record(z.string(), FNOLFieldSchemaV1),\n\n    /**\n     * Maximum number of back-and-forth turns (target question + user answer).\n     */\n    maxTurns: z.number().int().min(1).max(100).default(10),\n  },\n});\nexport type FNOLTestCaseV1 = z.infer<typeof FNOLTestCaseSchemaV1>;\n\nexport const FNOLConversationMessageSchemaV1 = z.object({\n  role: z.enum([\"system\", \"user\", \"assistant\"]),\n  content: z.string(),\n});\n\nexport const FNOLResponseSchemaV1 = defineResponseSchema({\n  baseSchema: BaseLLMChatResponseSchemaV1,\n  kind: \"fnol.rs.v1\",\n  schemaVersion: 1,\n  fields: {\n    /**\n     * Full conversation between the target model and simulated user.\n     */\n    conversation: z.array(FNOLConversationMessageSchemaV1),\n    turnsUsed: z.number().int(),\n    doneReason: z.enum(FNOLDoneReason),\n\n    /**\n     * Parsed JSON object from the target model's final answer, if available.\n     */\n    extracted: z.record(z.string(), z.unknown()).optional(),\n  },\n});\nexport type FNOLResponseV1 = z.infer<typeof FNOLResponseSchemaV1>;\n\nexport const FNOLFieldsScoreSchemaV1 = defineScoreSchema({\n  baseSchema: FNOLBaseScoreSchemaV1,\n  kind: \"fnol.sc.fields.v1\",\n  schemaVersion: 1,\n  fields: {\n    requiredKeys: z.array(z.string()),\n    presentKeys: z.array(z.string()),\n    missingKeys: z.array(z.string()),\n    mismatchedKeys: z.array(z.string()),\n  },\n});\nexport type FNOLFieldsScoreV1 = z.infer<typeof FNOLFieldsScoreSchemaV1>;\n\nexport const FNOLLLMJudgeScoreSchemaV1 = defineScoreSchema({\n  baseSchema: FNOLBaseScoreSchemaV1,\n  kind: \"fnol.sc.llm-judge.v1\",\n  schemaVersion: 1,\n  fields: {\n    verdict: z.enum([\"pass\", \"borderline\", \"fail\"]).optional(),\n  },\n});\nexport type FNOLLLMJudgeScoreV1 = z.infer<typeof FNOLLLMJudgeScoreSchemaV1>;\n","import { BaseScoreSchemaV1, defineScoreSchema } from \"@/schemas\";\nimport { ExtensionLLMAsAJudgeScorerFieldsV1 } from \"@/schemas/extensions/score/llm-as-a-judge-scorer\";\n\nexport const FNOLBaseScoreSchemaV1 = defineScoreSchema({\n  baseSchema: BaseScoreSchemaV1,\n  fields: {\n    ...ExtensionLLMAsAJudgeScorerFieldsV1,\n  },\n});\n","export const FNOLFieldValueType = {\n  string: \"string\",\n  number: \"number\",\n  boolean: \"boolean\",\n  object: \"object\",\n} as const;\n\nexport type FNOLFieldValueType =\n  (typeof FNOLFieldValueType)[keyof typeof FNOLFieldValueType];\n\nexport const FNOLDoneReason = {\n  modelProvidedJson: \"modelProvidedJson\",\n  reachedMaxTurns: \"reachedMaxTurns\",\n  forcedFinalJson: \"forcedFinalJson\",\n} as const;\n\nexport type FNOLDoneReason =\n  (typeof FNOLDoneReason)[keyof typeof FNOLDoneReason];\n\n","import { ChatCompletionMessageParam } from \"openai/resources/chat/completions\";\nimport { idGeneratorUUIDv7 } from \"@/utils/id-generator\";\nimport { IdGenerator, RunnerResult } from \"@/types\";\nimport { AbstractLLMProvider } from \"@/providers/abstract/llm\";\nimport { LLMJudgeScorer } from \"@/scorers/llm-judge\";\nimport { SimpleSystemPromptV1 } from \"@/schemas/llm\";\nimport { parseResponseAsJSON } from \"@/utils/llm\";\nimport { ScoringMethod } from \"@/types\";\nimport { FNOLFieldsScorer } from \"./scorer\";\nimport {\n  FNOLFieldsScoreSchemaV1,\n  FNOLFieldsScoreV1,\n  FNOLLLMJudgeScoreSchemaV1,\n  FNOLLLMJudgeScoreV1,\n  FNOLResponseSchemaV1,\n  FNOLResponseV1,\n  FNOLTestCaseV1,\n} from \"./test-cases/fnol.v1\";\nimport { FNOLDoneReason } from \"./types\";\n\nfunction formatFieldsToCollect(\n  fieldsToCollect: Record<string, { description: string; required?: boolean }>\n) {\n  return Object.entries(fieldsToCollect)\n    .map(([key, field]) => {\n      const required = field.required === false ? \"optional\" : \"required\";\n      return `- ${key} (${required}): ${field.description}`;\n    })\n    .join(\"\\n\");\n}\n\nfunction hasAllRequiredFields(params: {\n  extracted?: Record<string, unknown>;\n  fieldsToCollect: Record<string, { required?: boolean }>;\n}) {\n  const extracted = params.extracted ?? {};\n  for (const [key, field] of Object.entries(params.fieldsToCollect)) {\n    if (field.required === false) continue;\n    const value = extracted[key];\n    if (value === undefined || value === null || value === \"\") return false;\n  }\n  return true;\n}\n\nexport async function runTestCase(params: {\n  testCase: FNOLTestCaseV1;\n  provider: AbstractLLMProvider;\n  userSimulatorProvider?: AbstractLLMProvider;\n  scorer?: FNOLFieldsScorer | LLMJudgeScorer;\n  runConfig: {\n    model: string;\n    userSimulatorModel?: string;\n    llmJudgeModel?: string;\n    temperature?: number;\n    userSimulatorTemperature?: number;\n  };\n  systemPrompt?: SimpleSystemPromptV1;\n  idGenerators?: {\n    response?: IdGenerator;\n    score?: IdGenerator;\n  };\n}): Promise<\n  RunnerResult<FNOLResponseV1, FNOLFieldsScoreV1 | FNOLLLMJudgeScoreV1>\n> {\n  const responseIdGenerator =\n    params.idGenerators?.response ?? idGeneratorUUIDv7;\n  const scoreIdGenerator = params.idGenerators?.score ?? idGeneratorUUIDv7;\n\n  const userSimulatorProvider = params.userSimulatorProvider ?? params.provider;\n  const userSimulatorModel =\n    params.runConfig.userSimulatorModel ?? params.runConfig.model;\n\n  const fieldsToCollectText = formatFieldsToCollect(\n    params.testCase.fieldsToCollect\n  );\n\n  const conversation: ChatCompletionMessageParam[] = [];\n\n  if (params.systemPrompt) {\n    conversation.push({\n      role: \"system\",\n      content: params.systemPrompt.content,\n    });\n  }\n\n  conversation.push({\n    role: \"system\",\n    content: [\n      \"You are an insurance FNOL intake assistant.\",\n      \"Your job is to ask the user questions to collect the required fields listed below.\",\n      \"Ask concise questions, one or a few at a time.\",\n      \"When you have enough information OR when you are told to finish, output ONLY a single JSON object with the collected fields.\",\n      \"Do not include markdown fences. Do not include additional text outside the JSON.\",\n      \"\",\n      \"Fields to collect:\",\n      fieldsToCollectText,\n    ].join(\"\\n\"),\n  });\n\n  conversation.push({\n    role: \"user\",\n    content: params.testCase.initialUserMessage,\n  });\n\n  let doneReason: FNOLDoneReason | undefined;\n  let extracted: Record<string, unknown> | undefined;\n\n  const startedAt = Date.now();\n\n  for (let turn = 0; turn < params.testCase.maxTurns; turn++) {\n    const targetReply = await params.provider.forward({\n      model: params.runConfig.model,\n      temperature: params.runConfig.temperature,\n      messages: conversation,\n    });\n\n    conversation.push({\n      role: \"assistant\",\n      content: targetReply.data,\n    });\n\n    extracted = parseResponseAsJSON<Record<string, unknown>>(targetReply.data);\n    if (\n      extracted &&\n      hasAllRequiredFields({\n        extracted,\n        fieldsToCollect: params.testCase.fieldsToCollect,\n      })\n    ) {\n      doneReason = FNOLDoneReason.modelProvidedJson;\n      break;\n    }\n\n    const lastAssistantMessage = targetReply.data;\n\n    const simulatedUser = await userSimulatorProvider.forward({\n      model: userSimulatorModel,\n      temperature: params.runConfig.userSimulatorTemperature,\n      messages: [\n        {\n          role: \"system\",\n          content: [\n            \"You are simulating a real insurance customer (the user).\",\n            \"Answer the assistant's questions truthfully using ONLY the provided user profile and incident details.\",\n            \"If asked about something not present in the profile, say you don't know.\",\n            \"Be concise and natural. Do not invent new facts.\",\n            \"\",\n            \"User profile (JSON):\",\n            JSON.stringify(params.testCase.userProfile),\n          ].join(\"\\n\"),\n        },\n        {\n          role: \"user\",\n          content: lastAssistantMessage,\n        },\n      ],\n    });\n\n    conversation.push({\n      role: \"user\",\n      content: simulatedUser.data,\n    });\n  }\n\n  if (!doneReason) {\n    doneReason = FNOLDoneReason.reachedMaxTurns;\n\n    const forced = await params.provider.forward({\n      model: params.runConfig.model,\n      temperature: params.runConfig.temperature,\n      messages: [\n        ...conversation,\n        {\n          role: \"user\",\n          content:\n            \"Stop the interview now and output ONLY the final JSON object with the collected fields. No extra text.\",\n        },\n      ],\n    });\n\n    conversation.push({ role: \"assistant\", content: forced.data });\n    extracted = parseResponseAsJSON<Record<string, unknown>>(forced.data);\n    if (extracted) {\n      doneReason = FNOLDoneReason.forcedFinalJson;\n    }\n  }\n\n  const completedAt = Date.now();\n  const lastAssistant = [...conversation]\n    .reverse()\n    .find((m) => m.role === \"assistant\");\n\n  const response: FNOLResponseV1 = await FNOLResponseSchemaV1.newWithId(\n    {\n      data:\n        typeof lastAssistant?.content === \"string\" ? lastAssistant.content : \"\",\n      startedAt,\n      completedAt,\n      testCaseId: params.testCase.id,\n      modelSlug: params.runConfig.model,\n      provider: params.provider.kind,\n      conversation: conversation.map((m) => ({\n        role: m.role === \"user\" ? \"user\" : \"assistant\",\n        content: String(m.content),\n      })),\n      turnsUsed: conversation.filter((m) => m.role === \"assistant\").length,\n      doneReason,\n      extracted,\n    },\n    responseIdGenerator\n  );\n\n  if (params.scorer?.kind === \"fnol.fields\") {\n    const scorerResult = await params.scorer.score({\n      fieldsToCollect: params.testCase.fieldsToCollect,\n      extracted,\n    });\n\n    const score = await FNOLFieldsScoreSchemaV1.newWithId(\n      {\n        responseId: response.id,\n        value: scorerResult.value,\n        explanation: scorerResult.explanation,\n        metadata: scorerResult.metadata,\n        scoringMethod: ScoringMethod.algo,\n\n        requiredKeys: scorerResult.requiredKeys,\n        presentKeys: scorerResult.presentKeys,\n        missingKeys: scorerResult.missingKeys,\n        mismatchedKeys: scorerResult.mismatchedKeys,\n      },\n      scoreIdGenerator\n    );\n    return { response, score };\n  }\n\n  if (params.scorer?.kind === \"llmJudge\" && params.runConfig.llmJudgeModel) {\n    const scorerResult = await params.scorer.score({\n      task: \"Evaluate whether the FNOL JSON contains the required fields and correct values.\",\n      candidateAnswer: response.data,\n      referenceAnswer: JSON.stringify(\n        Object.fromEntries(\n          Object.entries(params.testCase.fieldsToCollect).map(([k, v]) => [\n            k,\n            v.expected,\n          ])\n        )\n      ),\n      model: params.runConfig.llmJudgeModel,\n      meta: {\n        fieldsToCollect: params.testCase.fieldsToCollect,\n        doneReason,\n      },\n    });\n\n    if (scorerResult !== null) {\n      const score = await FNOLLLMJudgeScoreSchemaV1.newWithId(\n        {\n          responseId: response.id,\n          value: scorerResult.value,\n          explanation: scorerResult.explanation,\n          metadata: scorerResult.metadata,\n          scoringMethod: ScoringMethod.ai,\n          verdict: scorerResult.verdict,\n\n          scorerAIProvider: scorerResult.provider,\n          scorerAIModelSlug: params.runConfig.llmJudgeModel,\n          scorerAIInputTokensUsed: scorerResult.inputTokensUsed,\n          scorerAIOutputTokensUsed: scorerResult.outputTokensUsed,\n          scorerAIInputCost: scorerResult.inputCost,\n          scorerAIOutputCost: scorerResult.outputCost,\n        },\n        scoreIdGenerator\n      );\n      return { response, score };\n    }\n  }\n\n  return { response };\n}\n","import { AbstractScorer, BaseScorerResult } from \"@/scorers/abstract\";\nimport { stableStringify } from \"@/utils/json\";\n\nfunction isMissing(value: unknown): boolean {\n  return value === undefined || value === null || value === \"\";\n}\n\nfunction normalizeString(value: string): string {\n  return value.trim();\n}\n\nfunction valuesEqual(expected: unknown, actual: unknown): boolean {\n  if (typeof expected === \"string\" && typeof actual === \"string\") {\n    return normalizeString(expected) === normalizeString(actual);\n  }\n  return stableStringify(expected) === stableStringify(actual);\n}\n\nexport class FNOLFieldsScorer extends AbstractScorer {\n  override readonly kind = \"fnol.fields\";\n\n  override async score(params: {\n    fieldsToCollect: Record<\n      string,\n      { required?: boolean; expected?: unknown; description?: string }\n    >;\n    extracted?: Record<string, unknown>;\n  }): Promise<\n    BaseScorerResult & {\n      requiredKeys: string[];\n      presentKeys: string[];\n      missingKeys: string[];\n      mismatchedKeys: string[];\n    }\n  > {\n    const extracted = params.extracted ?? {};\n    const requiredKeys = Object.entries(params.fieldsToCollect)\n      .filter(([, field]) => field.required !== false)\n      .map(([key]) => key);\n\n    const presentKeys: string[] = [];\n    const missingKeys: string[] = [];\n    const mismatchedKeys: string[] = [];\n\n    for (const key of requiredKeys) {\n      const value = extracted[key];\n      if (isMissing(value)) {\n        missingKeys.push(key);\n        continue;\n      }\n      presentKeys.push(key);\n\n      const expected = params.fieldsToCollect[key]?.expected;\n      if (expected !== undefined && !valuesEqual(expected, value)) {\n        mismatchedKeys.push(key);\n      }\n    }\n\n    const requiredCount = requiredKeys.length;\n    const correctCount = requiredCount - missingKeys.length - mismatchedKeys.length;\n    const score = requiredCount === 0 ? 1 : correctCount / requiredCount;\n\n    return {\n      value: Math.max(0, Math.min(1, score)),\n      explanation:\n        missingKeys.length === 0 && mismatchedKeys.length === 0\n          ? \"All required fields collected\"\n          : \"Missing or mismatched fields\",\n      requiredKeys,\n      presentKeys,\n      missingKeys,\n      mismatchedKeys,\n      metadata: {\n        requiredCount,\n        presentCount: presentKeys.length,\n        missingCount: missingKeys.length,\n        mismatchedCount: mismatchedKeys.length,\n      },\n    };\n  }\n}\n\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;;;ACAA,SAAS,SAAS;;;ACGX,IAAM,6BAA6B,kBAAkB;AAAA,EAC1D,YAAY;AAAA,EACZ,QAAQ;AAAA,IACN,GAAG;AAAA,EACL;AACF,CAAC;;;ADGM,IAAM,0CAA0C,qBAAqB;AAAA,EAC1E,YAAY;AAAA,EAEZ,MAAM;AAAA,EACN,eAAe;AAAA,EACf,QAAQ;AAAA,IACN,UAAU,EAAE,OAAO;AAAA,IACnB,SAAS,EAAE,OAAO,EAAE,OAAO,GAAG,EAAE,OAAO,CAAC;AAAA,IACxC,QAAQ,EAAE,OAAO;AAAA,IACjB,WAAW,EAAE,OAAO;AAAA,EACtB;AACF,CAAC;AAMM,IAAM,0CAA0C,qBAAqB;AAAA,EAC1E,YAAY;AAAA,EACZ,MAAM;AAAA,EACN,eAAe;AAAA,EACf,QAAQ,CAAC;AACX,CAAC;AAMM,IAAM,uCAAuC,kBAAkB;AAAA,EACpE,YAAY;AAAA,EACZ,MAAM;AAAA,EACN,eAAe;AAAA,EACf,QAAQ;AAAA,IACN,kBAAkB,EAAE,MAAM,EAAE,OAAO,CAAC;AAAA,EACtC;AACF,CAAC;;;AExCD,SAAS,KAAAA,UAAS;AAIX,IAAM,qCAAqC,qBAAqB;AAAA,EACrE,MAAM;AAAA,EACN,eAAe;AAAA,EACf,YAAY;AAAA,EACZ,QAAQ;AAAA,IACN,UAAUC,GAAE,OAAO;AAAA,IACnB,QAAQA,GAAE,OAAO,EAAE,SAAS;AAAA,EAC9B;AACF,CAAC;AAMM,IAAM,qCAAqC,qBAAqB;AAAA,EACrE,YAAY;AAAA,EACZ,MAAM;AAAA,EACN,eAAe;AACjB,CAAC;AAMM,IAAM,kCAAkC,kBAAkB;AAAA,EAC/D,YAAY;AAAA,EACZ,MAAM;AAAA,EACN,eAAe;AAAA,EACf,QAAQ,CAAC;AACX,CAAC;;;ACnCD,OAAOC,QAAO;AAEP,IAAM,iCAAiC,0BAA0B;AAAA,EACtE,YAAY;AAAA,EACZ,MAAM;AAAA,EACN,eAAe;AAAA,EACf,QAAQ;AAAA;AAAA;AAAA;AAAA,IAIN,WAAWA,GAAE,OAAOA,GAAE,OAAO,GAAGA,GAAE,OAAO,CAAC,EAAE,SAAS;AAAA,EACvD;AACF,CAAC;;;ACKD,OAAOC,QAAO;AAGP,IAAM,0BAAN,cAAsC,2BAI3C;AAAA,EACkB,OAAO;AAAA,EAEzB,MAAM,kBAAkB,QAEc;AACpC,UAAM,UAAU,eAAe,OAAO,OAAO;AAC7C,UAAM,SAAS,+BAA+B,MAAM,OAAO;AAE3D,WAAO;AAAA,EACT;AAAA,EAEmB,gBAAgB,MAAW;AAC5C,UAAM,qBAAqBC,GACxB,MAAM;AAAA,MACL;AAAA,MACA;AAAA,IACF,CAAC,EACA,UAAU,IAAI;AACjB,WAAO,mBAAmB,UAAU,mBAAmB,OAAO;AAAA,EAChE;AAAA,EAEA,MAAyB,gBAAgB,MAAW;AAClD,UAAM,qBAAqBA,GACxB,MAAM;AAAA,MACL;AAAA,MACA;AAAA,IACF,CAAC,EACA,UAAU,IAAI;AACjB,WAAO,mBAAmB,UAAU,mBAAmB,OAAO;AAAA,EAChE;AAAA,EAEA,MAAyB,aAAa,MAAW;AAC/C,UAAM,kBAAkBA,GACrB,MAAM;AAAA,MACL;AAAA,MACA;AAAA,IACF,CAAC,EACA,UAAU,IAAI;AACjB,WAAO,gBAAgB,UAAU,gBAAgB,OAAO;AAAA,EAC1D;AACF;;;ACrCA,eAAsB,YAAY,QAcmB;AACnD,QAAM,EAAE,SAAS,IAAI;AACrB,QAAM,sBACJ,OAAO,cAAc,YAAY;AACnC,QAAM,mBAAmB,OAAO,cAAc,SAAS;AACvD,QAAM,WAAyC,CAAC;AAEhD,MAAI,OAAO,cAAc;AACvB,aAAS,KAAK;AAAA,MACZ,MAAM;AAAA,MACN,SAAS,OAAO,aAAa;AAAA,IAC/B,CAAC;AAAA,EACH;AAEA,MAAI,SAAS,SAAS,aAAa;AACjC,UAAM,kBAAkB,gBAAgB,QAAQ;AAEhD,aAAS,KAAK;AAAA,MACZ,MAAM;AAAA,MACN,SAAS;AAAA,IACX,CAAC;AAED,UAAM,mBAAmB,MAAM,OAAO,SAAS,QAAQ;AAAA,MACrD,OAAO,OAAO,UAAU;AAAA,MACxB;AAAA,IACF,CAAC;AAED,UAAM,WAAW,MAAM,wCAAwC;AAAA,MAC7D;AAAA,QACE,MAAM,iBAAiB;AAAA,QACvB,WAAW,iBAAiB;AAAA,QAC5B,aAAa,iBAAiB;AAAA,QAC9B,YAAY,SAAS;AAAA,QACrB,WAAW,OAAO,UAAU;AAAA,QAC5B,UAAU,OAAO,SAAS;AAAA,QAE1B,iBAAiB,iBAAiB;AAAA,QAClC,kBAAkB,iBAAiB;AAAA,QACnC,WAAW,iBAAiB;AAAA,QAC5B,YAAY,iBAAiB;AAAA,MAC/B;AAAA,MACA;AAAA,IACF;AAEA,QAAI,OAAO,QAAQ,SAAS,OAAO;AACjC,YAAM,eAAe,MAAM,OAAO,OAAO,MAAM;AAAA,QAC7C,UAAU,SAAS;AAAA,QACnB,SAAS,SAAS,WAAW,CAAC;AAAA,QAC9B,gBAAgB,CAAC,SAAS,SAAS;AAAA,MACrC,CAAC;AAED,UAAI,iBAAiB,MAAM;AACzB,cAAM,QAAQ,MAAM,qCAAqC;AAAA,UACvD;AAAA,YACE,eAAe,cAAc;AAAA,YAC7B,OAAO,aAAa;AAAA,YACpB,YAAY,SAAS;AAAA,YACrB,kBAAkB,aAAa;AAAA,YAC/B,UAAU,SAAS;AAAA,UACrB;AAAA,UACA;AAAA,QACF;AAEA,eAAO,EAAE,UAAU,MAAM;AAAA,MAC3B;AAAA,IACF;AAEA,WAAO,EAAE,SAAS;AAAA,EACpB,WAAW,SAAS,SAAS,oBAAoB;AAC/C,UAAMC,YAAyC,CAAC;AAEhD,QAAI,OAAO,cAAc;AACvB,MAAAA,UAAS,KAAK;AAAA,QACZ,MAAM;AAAA,QACN,SAAS,OAAO,aAAa;AAAA,MAC/B,CAAC;AAAA,IACH;AAEA,IAAAA,UAAS,KAAK;AAAA,MACZ,MAAM;AAAA,MACN,SAAS,SAAS;AAAA,IACpB,CAAC;AAED,UAAM,mBAAmB,MAAM,OAAO,SAAS,QAAQ;AAAA,MACrD,OAAO,OAAO,UAAU;AAAA,MACxB,UAAAA;AAAA,IACF,CAAC;AAED,UAAM,WAAW,MAAM,mCAAmC;AAAA,MACxD;AAAA,QACE,MAAM,iBAAiB;AAAA,QACvB,WAAW,iBAAiB;AAAA,QAC5B,aAAa,iBAAiB;AAAA,QAC9B,YAAY,SAAS;AAAA,QACrB,WAAW,OAAO,UAAU;AAAA,QAC5B,UAAU,OAAO,SAAS;AAAA,QAE1B,iBAAiB,iBAAiB;AAAA,QAClC,kBAAkB,iBAAiB;AAAA,QACnC,WAAW,iBAAiB;AAAA,QAC5B,YAAY,iBAAiB;AAAA,MAC/B;AAAA,MACA;AAAA,IACF;AAEA,QAAI,OAAO,QAAQ,SAAS,cAAc,OAAO,UAAU,eAAe;AACxE,YAAM,eAAe,MAAM,OAAO,OAAO,MAAM;AAAA,QAC7C,MAAM,SAAS;AAAA,QACf,iBAAiB,SAAS;AAAA,QAC1B,iBAAiB,SAAS;AAAA,QAC1B,OAAO,OAAO,UAAU;AAAA,MAC1B,CAAC;AAED,UAAI,iBAAiB,MAAM;AACzB,cAAM,QAAQ,MAAM,gCAAgC;AAAA,UAClD;AAAA,YACE,eAAe,cAAc;AAAA,YAC7B,OAAO,aAAa;AAAA,YACpB,YAAY,SAAS;AAAA,YACrB,aAAa,aAAa;AAAA,YAC1B,UAAU,aAAa;AAAA,YAEvB,kBAAkB,aAAa;AAAA,YAC/B,mBAAmB,OAAO,UAAU;AAAA,YACpC,yBAAyB,aAAa;AAAA,YACtC,0BAA0B,aAAa;AAAA,YACvC,mBAAmB,aAAa;AAAA,YAChC,oBAAoB,aAAa;AAAA,UACnC;AAAA,UACA;AAAA,QACF;AAEA,eAAO,EAAE,UAAU,MAAM;AAAA,MAC3B;AAAA,IACF;AAEA,WAAO,EAAE,SAAS;AAAA,EACpB;AAEA,QAAM,IAAI,MAAM,4BAA4B;AAC9C;AAEA,SAAS,gBAAgB,UAA6C;AACpE,SAAO,aAAa,SAAS,QAAQ;AAAA;AAAA,EAAe,OAAO;AAAA,IACzD,SAAS,WAAW,CAAC;AAAA,EACvB,EACG,IAAI,CAAC,CAAC,KAAK,KAAK,MAAM,GAAG,GAAG,KAAK,KAAK,EAAE,EACxC,KAAK,IAAI,CAAC;AACf;;;AClMA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,qBAAAC;AAAA;;;ACGO,IAAM,2BAA2B,kBAAkB;AAAA,EACxD,YAAY;AAAA,EACZ,QAAQ;AAAA,IACN,GAAG;AAAA,EACL;AACF,CAAC;;;ACDD,SAAS,KAAAC,UAAS;AAGX,IAAM,8BAA8B,qBAAqB;AAAA,EAC9D,YAAY;AAAA,EACZ,MAAM;AAAA,EACN,eAAe;AAAA,EACf,QAAQ;AAAA,IACN,UAAUA,GAAE,OAAO;AAAA,IACnB,SAASA,GAAE,OAAOA,GAAE,OAAO,GAAGA,GAAE,OAAO,CAAC;AAAA,IACxC,QAAQA,GAAE,OAAO;AAAA,IACjB,WAAWA,GAAE,OAAO;AAAA,EACtB;AACF,CAAC;AAIM,IAAM,8BAA8B,qBAAqB;AAAA,EAC9D,YAAY;AAAA,EACZ,MAAM;AAAA,EACN,eAAe;AACjB,CAAC;AAIM,IAAM,2BAA2B,kBAAkB;AAAA,EACxD,YAAY;AAAA,EACZ,MAAM;AAAA,EACN,eAAe;AAAA,EACf,QAAQ;AAAA,IACN,kBAAkBA,GAAE,MAAMA,GAAE,OAAO,CAAC;AAAA,EACtC;AACF,CAAC;;;AChCD,OAAOC,QAAO;AAGd,SAAS,0BAA0B;AAGnC,IAAM,aAAaC,GAChB,OAAO;AAAA,EACN,aAAaA,GAAE,OAAO,OAAO;AAAA,EAC7B,UAAUA,GAAE,OAAO;AAAA,EACnB,SAASA,GAAE,MAAMA,GAAE,OAAO,CAAC;AAAA,EAC3B,QAAQA,GAAE,OAAO;AAAA,EACjB,cAAcA,GAAE,OAAO,OAAO;AAAA,EAC9B,aAAaA,GAAE,OAAO;AAAA,EACtB,UAAUA,GAAE,OAAO;AAAA,EACnB,KAAKA,GAAE,OAAO;AAChB,CAAC,EACA,MAAM;AAET,SAAS,QACP,MAKA;AACA,SAAO;AAAA,IACL,WAAW,CAAC;AAAA,IACZ,QAAQ,CAAC;AAAA,IACT,WAAW,KAAK;AAAA,MAAI,CAAC,SACnB,4BAA4B,IAAI;AAAA,QAC9B,IAAI,GAAG,KAAK,GAAG,IAAI,KAAK,QAAQ,IAAI,KAAK,WAAW;AAAA,QACpD,UAAU,KAAK;AAAA,QACf,WAAW,KAAK;AAAA,QAChB,SAAS,KAAK,QAAQ;AAAA,UACpB,CAAC,KAAK,QAAQ,UAAU;AACtB,gBAAI,OAAO,aAAa,KAAK,KAAK,CAAC,IAAI;AACvC,mBAAO;AAAA,UACT;AAAA,UACA,CAAC;AAAA,QACH;AAAA,QACA,QAAQ,KAAK,QAAQ,KAAK,YAAY;AAAA,QACtC,UAAU;AAAA,UACR,UAAU,KAAK;AAAA,UACf,KAAK,KAAK;AAAA,UACV,cAAc,KAAK;AAAA,QACrB;AAAA,MACF,CAAC;AAAA,IACH;AAAA,EACF;AACF;AAEO,IAAM,wBAAN,cAAoC,mBAAmB;AAAA,EAC1C,OAAO;AAAA,EAEhB,SAAS,QAMhB;AACA,UAAM,UACJ,OAAO,OAAO,YAAY,WACtB,OAAO,UACP,eAAe,OAAO,OAAO;AAEnC,UAAM,SAAS,WAAW,MAAM,KAAK,MAAM,OAAO,CAAC;AACnD,WAAO,QAAQ,MAAM;AAAA,EACvB;AAAA;AAAA,EAGS,kBAAkB,QAES;AAClC,UAAM,IAAI,MAAM,iBAAiB;AAAA,EACnC;AACF;AAEO,IAAM,2BAAN,cAAuC,mBAAmB;AAAA,EAC7C,OAAO;AAAA,EAEzB,MAAe,SAAS,QAQtB;AACA,UAAM,OAAO,MAAM,mBAAmB;AAAA,MACpC,MAAM,OAAO,QAAQ;AAAA,IACvB,CAAC;AACD,QAAI,CAAC,MAAM;AACT,YAAM,IAAI,MAAM,sBAAsB;AAAA,IACxC;AAEA,WAAO,QAAQ,WAAW,MAAM,IAAI,CAAC;AAAA,EACvC;AAAA;AAAA,EAGS,kBAAkB,QAES;AAClC,UAAM,IAAI,MAAM,iBAAiB;AAAA,EACnC;AACF;;;ACjGA,eAAsBC,aAAY,QAcmC;AACnE,QAAM,EAAE,SAAS,IAAI;AACrB,QAAM,sBACJ,OAAO,cAAc,YAAY;AACnC,QAAM,mBAAmB,OAAO,cAAc,SAAS;AACvD,QAAM,WAAyC,CAAC;AAEhD,MAAI,OAAO,cAAc;AACvB,aAAS,KAAK;AAAA,MACZ,MAAM;AAAA,MACN,SAAS,OAAO,aAAa;AAAA,IAC/B,CAAC;AAAA,EACH;AAEA,MAAI,SAAS,SAAS,oBAAoB;AACxC,UAAM,kBAAkBC,iBAAgB,QAAQ;AAEhD,aAAS,KAAK;AAAA,MACZ,MAAM;AAAA,MACN,SAAS;AAAA,IACX,CAAC;AAED,UAAM,mBAAmB,MAAM,OAAO,SAAS,QAAQ;AAAA,MACrD,OAAO,OAAO,UAAU;AAAA,MACxB;AAAA,IACF,CAAC;AAED,UAAM,WAAW,MAAM,4BAA4B;AAAA,MACjD;AAAA,QACE,MAAM,iBAAiB;AAAA,QACvB,WAAW,iBAAiB;AAAA,QAC5B,aAAa,iBAAiB;AAAA,QAC9B,YAAY,SAAS;AAAA,QACrB,WAAW,OAAO,UAAU;AAAA,QAC5B,UAAU,OAAO,SAAS;AAAA,QAE1B,iBAAiB,iBAAiB;AAAA,QAClC,kBAAkB,iBAAiB;AAAA,QACnC,WAAW,iBAAiB;AAAA,QAC5B,YAAY,iBAAiB;AAAA,MAC/B;AAAA,MACA;AAAA,IACF;AAEA,QAAI,OAAO,QAAQ,SAAS,OAAO;AACjC,YAAM,eAAe,MAAM,OAAO,OAAO,MAAM;AAAA,QAC7C,UAAU,SAAS;AAAA,QACnB,SAAS,SAAS,WAAW,CAAC;AAAA,QAC9B,gBAAgB,CAAC,SAAS,SAAS;AAAA,MACrC,CAAC;AAED,UAAI,iBAAiB,MAAM;AACzB,cAAM,QAAQ,MAAM,yBAAyB;AAAA,UAC3C;AAAA,YACE,eAAe,cAAc;AAAA,YAC7B,OAAO,aAAa;AAAA,YACpB,YAAY,SAAS;AAAA,YACrB,kBAAkB,aAAa;AAAA,YAC/B,UAAU,SAAS;AAAA,UACrB;AAAA,UACA;AAAA,QACF;AAEA,eAAO,EAAE,UAAU,MAAM;AAAA,MAC3B;AAAA,IACF;AAEA,WAAO,EAAE,SAAS;AAAA,EACpB,WAAW,SAAS,SAAS,oBAAoB;AAC/C,UAAMC,YAAyC,CAAC;AAEhD,QAAI,OAAO,cAAc;AACvB,MAAAA,UAAS,KAAK;AAAA,QACZ,MAAM;AAAA,QACN,SAAS,OAAO,aAAa;AAAA,MAC/B,CAAC;AAAA,IACH;AAEA,IAAAA,UAAS,KAAK;AAAA,MACZ,MAAM;AAAA,MACN,SAAS,SAAS;AAAA,IACpB,CAAC;AAED,UAAM,mBAAmB,MAAM,OAAO,SAAS,QAAQ;AAAA,MACrD,OAAO,OAAO,UAAU;AAAA,MACxB,UAAAA;AAAA,IACF,CAAC;AAED,UAAM,WAAW,MAAM,4BAA4B;AAAA,MACjD;AAAA,QACE,MAAM,iBAAiB;AAAA,QACvB,WAAW,iBAAiB;AAAA,QAC5B,aAAa,iBAAiB;AAAA,QAC9B,YAAY,SAAS;AAAA,QACrB,WAAW,OAAO,UAAU;AAAA,QAC5B,UAAU,OAAO,SAAS;AAAA,QAE1B,iBAAiB,iBAAiB;AAAA,QAClC,kBAAkB,iBAAiB;AAAA,QACnC,WAAW,iBAAiB;AAAA,QAC5B,YAAY,iBAAiB;AAAA,MAC/B;AAAA,MACA;AAAA,IACF;AAEA,QAAI,OAAO,QAAQ,SAAS,cAAc,OAAO,UAAU,eAAe;AACxE,YAAM,eAAe,MAAM,OAAO,OAAO,MAAM;AAAA,QAC7C,MAAM,SAAS;AAAA,QACf,iBAAiB,SAAS;AAAA,QAC1B,iBAAiB,SAAS;AAAA,QAC1B,OAAO,OAAO,UAAU;AAAA,MAC1B,CAAC;AAED,UAAI,iBAAiB,MAAM;AACzB,cAAM,QAAQ,MAAM,yBAAyB;AAAA,UAC3C;AAAA,YACE,eAAe,cAAc;AAAA,YAC7B,OAAO,aAAa;AAAA,YACpB,YAAY,SAAS;AAAA,YACrB,aAAa,aAAa;AAAA,YAC1B,UAAU,aAAa;AAAA,YACvB,kBAAkB,CAAC;AAAA,YAEnB,kBAAkB,aAAa;AAAA,YAC/B,mBAAmB,OAAO,UAAU;AAAA,YACpC,yBAAyB,aAAa;AAAA,YACtC,0BAA0B,aAAa;AAAA,YACvC,mBAAmB,aAAa;AAAA,YAChC,oBAAoB,aAAa;AAAA,UACnC;AAAA,UACA;AAAA,QACF;AAEA,eAAO,EAAE,UAAU,MAAM;AAAA,MAC3B;AAAA,IACF;AAEA,WAAO,EAAE,SAAS;AAAA,EACpB;AAEA,QAAM,IAAI,MAAM,4BAA4B;AAC9C;AAEA,SAASD,iBAAgB,UAAiC;AACxD,SAAO,aAAa,SAAS,QAAQ;AAAA;AAAA,EAAe,OAAO;AAAA,IACzD,SAAS,WAAW,CAAC;AAAA,EACvB,EACG,IAAI,CAAC,CAAC,KAAK,KAAK,MAAM,GAAG,GAAG,KAAK,KAAK,EAAE,EACxC,KAAK,IAAI,CAAC;AACf;;;AC9KO,IAAM,+BAA+B,0BAA0B;AAAA,EACpE,YAAY;AAAA,EACZ,MAAM;AAAA,EACN,eAAe;AAAA,EACf,QAAQ,CAAC;AACX,CAAC;;;ACXD;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,qBAAAE;AAAA;;;ACAA,SAAS,KAAAC,UAAS;;;ACGX,IAAM,wBAAwB,kBAAkB;AAAA,EACrD,YAAY;AAAA,EACZ,QAAQ;AAAA,IACN,GAAG;AAAA,EACL;AACF,CAAC;;;ACRM,IAAM,qBAAqB;AAAA,EAChC,QAAQ;AAAA,EACR,QAAQ;AAAA,EACR,SAAS;AAAA,EACT,QAAQ;AACV;AAKO,IAAM,iBAAiB;AAAA,EAC5B,mBAAmB;AAAA,EACnB,iBAAiB;AAAA,EACjB,iBAAiB;AACnB;;;AFHO,IAAM,oBAAoBC,GAAE,OAAO;AAAA,EACxC,aAAaA,GAAE,OAAO;AAAA,EACtB,UAAUA,GAAE,QAAQ,EAAE,SAAS;AAAA;AAAA;AAAA;AAAA;AAAA,EAK/B,UAAUA,GAAE,QAAQ,EAAE,SAAS;AAAA;AAAA;AAAA;AAAA,EAI/B,WAAWA,GAAE,KAAK,kBAAkB,EAAE,SAAS;AACjD,CAAC;AAEM,IAAM,uBAAuB,qBAAqB;AAAA,EACvD,YAAY;AAAA,EACZ,MAAM;AAAA,EACN,eAAe;AAAA,EACf,QAAQ;AAAA;AAAA;AAAA;AAAA,IAIN,oBAAoBA,GAAE,OAAO;AAAA;AAAA;AAAA;AAAA;AAAA,IAM7B,aAAaA,GAAE,OAAOA,GAAE,OAAO,GAAGA,GAAE,QAAQ,CAAC;AAAA;AAAA;AAAA;AAAA;AAAA,IAM7C,iBAAiBA,GAAE,OAAOA,GAAE,OAAO,GAAG,iBAAiB;AAAA;AAAA;AAAA;AAAA,IAKvD,UAAUA,GAAE,OAAO,EAAE,IAAI,EAAE,IAAI,CAAC,EAAE,IAAI,GAAG,EAAE,QAAQ,EAAE;AAAA,EACvD;AACF,CAAC;AAGM,IAAM,kCAAkCA,GAAE,OAAO;AAAA,EACtD,MAAMA,GAAE,KAAK,CAAC,UAAU,QAAQ,WAAW,CAAC;AAAA,EAC5C,SAASA,GAAE,OAAO;AACpB,CAAC;AAEM,IAAM,uBAAuB,qBAAqB;AAAA,EACvD,YAAY;AAAA,EACZ,MAAM;AAAA,EACN,eAAe;AAAA,EACf,QAAQ;AAAA;AAAA;AAAA;AAAA,IAIN,cAAcA,GAAE,MAAM,+BAA+B;AAAA,IACrD,WAAWA,GAAE,OAAO,EAAE,IAAI;AAAA,IAC1B,YAAYA,GAAE,KAAK,cAAc;AAAA;AAAA;AAAA;AAAA,IAKjC,WAAWA,GAAE,OAAOA,GAAE,OAAO,GAAGA,GAAE,QAAQ,CAAC,EAAE,SAAS;AAAA,EACxD;AACF,CAAC;AAGM,IAAM,0BAA0B,kBAAkB;AAAA,EACvD,YAAY;AAAA,EACZ,MAAM;AAAA,EACN,eAAe;AAAA,EACf,QAAQ;AAAA,IACN,cAAcA,GAAE,MAAMA,GAAE,OAAO,CAAC;AAAA,IAChC,aAAaA,GAAE,MAAMA,GAAE,OAAO,CAAC;AAAA,IAC/B,aAAaA,GAAE,MAAMA,GAAE,OAAO,CAAC;AAAA,IAC/B,gBAAgBA,GAAE,MAAMA,GAAE,OAAO,CAAC;AAAA,EACpC;AACF,CAAC;AAGM,IAAM,4BAA4B,kBAAkB;AAAA,EACzD,YAAY;AAAA,EACZ,MAAM;AAAA,EACN,eAAe;AAAA,EACf,QAAQ;AAAA,IACN,SAASA,GAAE,KAAK,CAAC,QAAQ,cAAc,MAAM,CAAC,EAAE,SAAS;AAAA,EAC3D;AACF,CAAC;;;AGhFD,SAAS,sBACP,iBACA;AACA,SAAO,OAAO,QAAQ,eAAe,EAClC,IAAI,CAAC,CAAC,KAAK,KAAK,MAAM;AACrB,UAAM,WAAW,MAAM,aAAa,QAAQ,aAAa;AACzD,WAAO,KAAK,GAAG,KAAK,QAAQ,MAAM,MAAM,WAAW;AAAA,EACrD,CAAC,EACA,KAAK,IAAI;AACd;AAEA,SAAS,qBAAqB,QAG3B;AACD,QAAM,YAAY,OAAO,aAAa,CAAC;AACvC,aAAW,CAAC,KAAK,KAAK,KAAK,OAAO,QAAQ,OAAO,eAAe,GAAG;AACjE,QAAI,MAAM,aAAa,MAAO;AAC9B,UAAM,QAAQ,UAAU,GAAG;AAC3B,QAAI,UAAU,UAAa,UAAU,QAAQ,UAAU,GAAI,QAAO;AAAA,EACpE;AACA,SAAO;AACT;AAEA,eAAsBC,aAAY,QAmBhC;AACA,QAAM,sBACJ,OAAO,cAAc,YAAY;AACnC,QAAM,mBAAmB,OAAO,cAAc,SAAS;AAEvD,QAAM,wBAAwB,OAAO,yBAAyB,OAAO;AACrE,QAAM,qBACJ,OAAO,UAAU,sBAAsB,OAAO,UAAU;AAE1D,QAAM,sBAAsB;AAAA,IAC1B,OAAO,SAAS;AAAA,EAClB;AAEA,QAAM,eAA6C,CAAC;AAEpD,MAAI,OAAO,cAAc;AACvB,iBAAa,KAAK;AAAA,MAChB,MAAM;AAAA,MACN,SAAS,OAAO,aAAa;AAAA,IAC/B,CAAC;AAAA,EACH;AAEA,eAAa,KAAK;AAAA,IAChB,MAAM;AAAA,IACN,SAAS;AAAA,MACP;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACF,EAAE,KAAK,IAAI;AAAA,EACb,CAAC;AAED,eAAa,KAAK;AAAA,IAChB,MAAM;AAAA,IACN,SAAS,OAAO,SAAS;AAAA,EAC3B,CAAC;AAED,MAAI;AACJ,MAAI;AAEJ,QAAM,YAAY,KAAK,IAAI;AAE3B,WAAS,OAAO,GAAG,OAAO,OAAO,SAAS,UAAU,QAAQ;AAC1D,UAAM,cAAc,MAAM,OAAO,SAAS,QAAQ;AAAA,MAChD,OAAO,OAAO,UAAU;AAAA,MACxB,aAAa,OAAO,UAAU;AAAA,MAC9B,UAAU;AAAA,IACZ,CAAC;AAED,iBAAa,KAAK;AAAA,MAChB,MAAM;AAAA,MACN,SAAS,YAAY;AAAA,IACvB,CAAC;AAED,gBAAY,oBAA6C,YAAY,IAAI;AACzE,QACE,aACA,qBAAqB;AAAA,MACnB;AAAA,MACA,iBAAiB,OAAO,SAAS;AAAA,IACnC,CAAC,GACD;AACA,mBAAa,eAAe;AAC5B;AAAA,IACF;AAEA,UAAM,uBAAuB,YAAY;AAEzC,UAAM,gBAAgB,MAAM,sBAAsB,QAAQ;AAAA,MACxD,OAAO;AAAA,MACP,aAAa,OAAO,UAAU;AAAA,MAC9B,UAAU;AAAA,QACR;AAAA,UACE,MAAM;AAAA,UACN,SAAS;AAAA,YACP;AAAA,YACA;AAAA,YACA;AAAA,YACA;AAAA,YACA;AAAA,YACA;AAAA,YACA,KAAK,UAAU,OAAO,SAAS,WAAW;AAAA,UAC5C,EAAE,KAAK,IAAI;AAAA,QACb;AAAA,QACA;AAAA,UACE,MAAM;AAAA,UACN,SAAS;AAAA,QACX;AAAA,MACF;AAAA,IACF,CAAC;AAED,iBAAa,KAAK;AAAA,MAChB,MAAM;AAAA,MACN,SAAS,cAAc;AAAA,IACzB,CAAC;AAAA,EACH;AAEA,MAAI,CAAC,YAAY;AACf,iBAAa,eAAe;AAE5B,UAAM,SAAS,MAAM,OAAO,SAAS,QAAQ;AAAA,MAC3C,OAAO,OAAO,UAAU;AAAA,MACxB,aAAa,OAAO,UAAU;AAAA,MAC9B,UAAU;AAAA,QACR,GAAG;AAAA,QACH;AAAA,UACE,MAAM;AAAA,UACN,SACE;AAAA,QACJ;AAAA,MACF;AAAA,IACF,CAAC;AAED,iBAAa,KAAK,EAAE,MAAM,aAAa,SAAS,OAAO,KAAK,CAAC;AAC7D,gBAAY,oBAA6C,OAAO,IAAI;AACpE,QAAI,WAAW;AACb,mBAAa,eAAe;AAAA,IAC9B;AAAA,EACF;AAEA,QAAM,cAAc,KAAK,IAAI;AAC7B,QAAM,gBAAgB,CAAC,GAAG,YAAY,EACnC,QAAQ,EACR,KAAK,CAAC,MAAM,EAAE,SAAS,WAAW;AAErC,QAAM,WAA2B,MAAM,qBAAqB;AAAA,IAC1D;AAAA,MACE,MACE,OAAO,eAAe,YAAY,WAAW,cAAc,UAAU;AAAA,MACvE;AAAA,MACA;AAAA,MACA,YAAY,OAAO,SAAS;AAAA,MAC5B,WAAW,OAAO,UAAU;AAAA,MAC5B,UAAU,OAAO,SAAS;AAAA,MAC1B,cAAc,aAAa,IAAI,CAAC,OAAO;AAAA,QACrC,MAAM,EAAE,SAAS,SAAS,SAAS;AAAA,QACnC,SAAS,OAAO,EAAE,OAAO;AAAA,MAC3B,EAAE;AAAA,MACF,WAAW,aAAa,OAAO,CAAC,MAAM,EAAE,SAAS,WAAW,EAAE;AAAA,MAC9D;AAAA,MACA;AAAA,IACF;AAAA,IACA;AAAA,EACF;AAEA,MAAI,OAAO,QAAQ,SAAS,eAAe;AACzC,UAAM,eAAe,MAAM,OAAO,OAAO,MAAM;AAAA,MAC7C,iBAAiB,OAAO,SAAS;AAAA,MACjC;AAAA,IACF,CAAC;AAED,UAAM,QAAQ,MAAM,wBAAwB;AAAA,MAC1C;AAAA,QACE,YAAY,SAAS;AAAA,QACrB,OAAO,aAAa;AAAA,QACpB,aAAa,aAAa;AAAA,QAC1B,UAAU,aAAa;AAAA,QACvB,eAAe,cAAc;AAAA,QAE7B,cAAc,aAAa;AAAA,QAC3B,aAAa,aAAa;AAAA,QAC1B,aAAa,aAAa;AAAA,QAC1B,gBAAgB,aAAa;AAAA,MAC/B;AAAA,MACA;AAAA,IACF;AACA,WAAO,EAAE,UAAU,MAAM;AAAA,EAC3B;AAEA,MAAI,OAAO,QAAQ,SAAS,cAAc,OAAO,UAAU,eAAe;AACxE,UAAM,eAAe,MAAM,OAAO,OAAO,MAAM;AAAA,MAC7C,MAAM;AAAA,MACN,iBAAiB,SAAS;AAAA,MAC1B,iBAAiB,KAAK;AAAA,QACpB,OAAO;AAAA,UACL,OAAO,QAAQ,OAAO,SAAS,eAAe,EAAE,IAAI,CAAC,CAAC,GAAG,CAAC,MAAM;AAAA,YAC9D;AAAA,YACA,EAAE;AAAA,UACJ,CAAC;AAAA,QACH;AAAA,MACF;AAAA,MACA,OAAO,OAAO,UAAU;AAAA,MACxB,MAAM;AAAA,QACJ,iBAAiB,OAAO,SAAS;AAAA,QACjC;AAAA,MACF;AAAA,IACF,CAAC;AAED,QAAI,iBAAiB,MAAM;AACzB,YAAM,QAAQ,MAAM,0BAA0B;AAAA,QAC5C;AAAA,UACE,YAAY,SAAS;AAAA,UACrB,OAAO,aAAa;AAAA,UACpB,aAAa,aAAa;AAAA,UAC1B,UAAU,aAAa;AAAA,UACvB,eAAe,cAAc;AAAA,UAC7B,SAAS,aAAa;AAAA,UAEtB,kBAAkB,aAAa;AAAA,UAC/B,mBAAmB,OAAO,UAAU;AAAA,UACpC,yBAAyB,aAAa;AAAA,UACtC,0BAA0B,aAAa;AAAA,UACvC,mBAAmB,aAAa;AAAA,UAChC,oBAAoB,aAAa;AAAA,QACnC;AAAA,QACA;AAAA,MACF;AACA,aAAO,EAAE,UAAU,MAAM;AAAA,IAC3B;AAAA,EACF;AAEA,SAAO,EAAE,SAAS;AACpB;;;ACpRA,SAAS,UAAU,OAAyB;AAC1C,SAAO,UAAU,UAAa,UAAU,QAAQ,UAAU;AAC5D;AAEA,SAAS,gBAAgB,OAAuB;AAC9C,SAAO,MAAM,KAAK;AACpB;AAEA,SAAS,YAAY,UAAmB,QAA0B;AAChE,MAAI,OAAO,aAAa,YAAY,OAAO,WAAW,UAAU;AAC9D,WAAO,gBAAgB,QAAQ,MAAM,gBAAgB,MAAM;AAAA,EAC7D;AACA,SAAO,gBAAgB,QAAQ,MAAM,gBAAgB,MAAM;AAC7D;AAEO,IAAM,mBAAN,cAA+B,eAAe;AAAA,EACjC,OAAO;AAAA,EAEzB,MAAe,MAAM,QAanB;AACA,UAAM,YAAY,OAAO,aAAa,CAAC;AACvC,UAAM,eAAe,OAAO,QAAQ,OAAO,eAAe,EACvD,OAAO,CAAC,CAAC,EAAE,KAAK,MAAM,MAAM,aAAa,KAAK,EAC9C,IAAI,CAAC,CAAC,GAAG,MAAM,GAAG;AAErB,UAAM,cAAwB,CAAC;AAC/B,UAAM,cAAwB,CAAC;AAC/B,UAAM,iBAA2B,CAAC;AAElC,eAAW,OAAO,cAAc;AAC9B,YAAM,QAAQ,UAAU,GAAG;AAC3B,UAAI,UAAU,KAAK,GAAG;AACpB,oBAAY,KAAK,GAAG;AACpB;AAAA,MACF;AACA,kBAAY,KAAK,GAAG;AAEpB,YAAM,WAAW,OAAO,gBAAgB,GAAG,GAAG;AAC9C,UAAI,aAAa,UAAa,CAAC,YAAY,UAAU,KAAK,GAAG;AAC3D,uBAAe,KAAK,GAAG;AAAA,MACzB;AAAA,IACF;AAEA,UAAM,gBAAgB,aAAa;AACnC,UAAM,eAAe,gBAAgB,YAAY,SAAS,eAAe;AACzE,UAAM,QAAQ,kBAAkB,IAAI,IAAI,eAAe;AAEvD,WAAO;AAAA,MACL,OAAO,KAAK,IAAI,GAAG,KAAK,IAAI,GAAG,KAAK,CAAC;AAAA,MACrC,aACE,YAAY,WAAW,KAAK,eAAe,WAAW,IAClD,kCACA;AAAA,MACN;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA,UAAU;AAAA,QACR;AAAA,QACA,cAAc,YAAY;AAAA,QAC1B,cAAc,YAAY;AAAA,QAC1B,iBAAiB,eAAe;AAAA,MAClC;AAAA,IACF;AAAA,EACF;AACF;","names":["z","z","z","z","z","messages","runTestCase","z","z","z","runTestCase","formatMCQPrompt","messages","runTestCase","z","z","runTestCase"]}
+{"version":3,"sources":["../../src/benchmarks/peerbench/index.ts","../../src/benchmarks/peerbench/schema-sets/mcq.v1.ts","../../src/benchmarks/peerbench/schema-sets/multi-turn.v1.ts","../../src/benchmarks/peerbench/schema-sets/qa.v1.ts","../../src/benchmarks/peerbench/runner.ts","../../src/benchmarks/peerbench/storages/json.ts"],"sourcesContent":["export * from \"./schema-sets/mcq.v1\";\nexport * from \"./schema-sets/multi-turn.v1\";\nexport * from \"./schema-sets/qa.v1\";\n\nexport * from \"./runner\";\n\nexport * from \"./storages/json\";\n","import { PEERBENCH_NAMESPACE } from \"@/constants\";\nimport {\n  BaseResponseSchemaV1,\n  BaseScoreSchemaV1,\n  BaseTestCaseSchemaV1,\n  defineResponseSchema,\n  defineScoreSchema,\n  defineTestCaseSchema,\n} from \"@/schemas\";\nimport { ExtensionLLMResponseFieldsV1 } from \"@/schemas/extensions/response/llm\";\nimport { ExtensionLLMAsAJudgeScoreFieldsV1 } from \"@/schemas/extensions/score/llm-as-a-judge-scorer\";\nimport { z } from \"zod\";\n\nexport const MCQKind = `llm/mcq` as const;\n\nexport const MCQTestCaseSchemaV1 = defineTestCaseSchema({\n  baseSchema: BaseTestCaseSchemaV1,\n  namespace: PEERBENCH_NAMESPACE,\n  kind: MCQKind,\n  schemaVersion: 1,\n  fields: {\n    question: z.string(),\n    options: z.record(z.string(), z.string()),\n    correctAnswerKeys: z.string().array(),\n  },\n});\nexport type MCQTestCaseV1 = z.infer<typeof MCQTestCaseSchemaV1>;\n\nexport const MCQResponseSchemaV1 = defineResponseSchema({\n  baseSchema: BaseResponseSchemaV1,\n  namespace: PEERBENCH_NAMESPACE,\n  kind: MCQKind,\n  schemaVersion: 1,\n  fields: {\n    ...ExtensionLLMResponseFieldsV1,\n  },\n});\nexport type MCQResponseV1 = z.infer<typeof MCQResponseSchemaV1>;\n\nexport const MCQScoreSchemaV1 = defineScoreSchema({\n  baseSchema: BaseScoreSchemaV1,\n  namespace: PEERBENCH_NAMESPACE,\n  kind: MCQKind,\n  schemaVersion: 1,\n  fields: {\n    ...ExtensionLLMAsAJudgeScoreFieldsV1,\n    extractedAnswers: z.array(z.string()),\n  },\n});\nexport type MCQScoreV1 = z.infer<typeof MCQScoreSchemaV1>;\n","import { PEERBENCH_NAMESPACE } from \"@/constants\";\nimport {\n  BaseResponseSchemaV1,\n  BaseScoreSchemaV1,\n  BaseTestCaseSchemaV1,\n  defineResponseSchema,\n  defineScoreSchema,\n  defineTestCaseSchema,\n} from \"@/schemas\";\nimport { ExtensionLLMResponseFieldsV1 } from \"@/schemas/extensions/response/llm\";\nimport { ExtensionLLMAsAJudgeScoreFieldsV1 } from \"@/schemas/extensions/score/llm-as-a-judge-scorer\";\nimport { z } from \"zod\";\n\nexport const MultiTurnKind = `llm/multi-turn` as const;\n\nexport const MultiTurnTestCaseSchemaV1 = defineTestCaseSchema({\n  baseSchema: BaseTestCaseSchemaV1,\n  namespace: PEERBENCH_NAMESPACE,\n  kind: MultiTurnKind,\n  schemaVersion: 1,\n  fields: {\n    messages: z\n      .object({\n        role: z.string(),\n        content: z.string(),\n        goodAnswers: z.string().array().optional(),\n        badAnswers: z.string().array().optional(),\n      })\n      .array(),\n\n    maxTurns: z.number().optional(),\n    expectedOutcome: z.string().optional(),\n  },\n});\nexport type MultiTurnTestCaseV1 = z.infer<typeof MultiTurnTestCaseSchemaV1>;\n\nexport const MultiTurnResponseSchemaV1 = defineResponseSchema({\n  baseSchema: BaseResponseSchemaV1,\n  namespace: PEERBENCH_NAMESPACE,\n  kind: MultiTurnKind,\n  schemaVersion: 1,\n  fields: {\n    ...ExtensionLLMResponseFieldsV1,\n    replies: z\n      .object({\n        messageIndex: z.number(),\n        startedAt: z.number(),\n        completedAt: z.number(),\n        data: z.string(),\n\n        inputTokensUsed: z.number().optional(),\n        outputTokensUsed: z.number().optional(),\n        inputCost: z.string().optional(),\n        outputCost: z.string().optional(),\n      })\n      .array(),\n  },\n});\nexport type MultiTurnResponseV1 = z.infer<typeof MultiTurnResponseSchemaV1>;\n\nexport const MultiTurnScoreSchemaV1 = defineScoreSchema({\n  baseSchema: BaseScoreSchemaV1,\n  namespace: PEERBENCH_NAMESPACE,\n  kind: MultiTurnKind,\n  schemaVersion: 1,\n  fields: {\n    ...ExtensionLLMAsAJudgeScoreFieldsV1,\n    individualScores: z\n      .object({\n        replyIndex: z.number(),\n        value: z.number(),\n      })\n      .array(),\n  },\n});\nexport type MultiTurnScoreV1 = z.infer<typeof MultiTurnScoreSchemaV1>;\n","import { PEERBENCH_NAMESPACE } from \"@/constants\";\nimport {\n  BaseResponseSchemaV1,\n  BaseScoreSchemaV1,\n  BaseTestCaseSchemaV1,\n  defineResponseSchema,\n  defineScoreSchema,\n  defineTestCaseSchema,\n} from \"@/schemas\";\nimport { ExtensionLLMResponseFieldsV1 } from \"@/schemas/extensions/response/llm\";\nimport { ExtensionLLMAsAJudgeScoreFieldsV1 } from \"@/schemas/extensions/score/llm-as-a-judge-scorer\";\nimport { z } from \"zod\";\n\nexport const QAKind = `llm/qa` as const;\n\nexport const QATestCaseSchemaV1 = defineTestCaseSchema({\n  baseSchema: BaseTestCaseSchemaV1,\n  namespace: PEERBENCH_NAMESPACE,\n  kind: QAKind,\n  schemaVersion: 1,\n  fields: {\n    question: z.string(),\n    goodAnswers: z.string().array(),\n    badAnswers: z.string().array(),\n  },\n});\nexport type QATestCaseV1 = z.infer<typeof QATestCaseSchemaV1>;\n\nexport const QAResponseSchemaV1 = defineResponseSchema({\n  baseSchema: BaseResponseSchemaV1,\n  namespace: PEERBENCH_NAMESPACE,\n  kind: QAKind,\n  schemaVersion: 1,\n  fields: {\n    ...ExtensionLLMResponseFieldsV1,\n  },\n});\nexport type QAResponseV1 = z.infer<typeof QAResponseSchemaV1>;\n\nexport const QAScoreSchemaV1 = defineScoreSchema({\n  baseSchema: BaseScoreSchemaV1,\n  namespace: PEERBENCH_NAMESPACE,\n  kind: QAKind,\n  schemaVersion: 1,\n  fields: {\n    ...ExtensionLLMAsAJudgeScoreFieldsV1,\n  },\n});\nexport type QAScoreV1 = z.infer<typeof QAScoreSchemaV1>;\n","import { defineRunner } from \"@/helpers/define-runner\";\nimport { AbstractLLMProvider } from \"@/providers\";\nimport {\n  SimpleSystemPromptSchemaV1,\n  SimpleSystemPromptV1,\n} from \"@/schemas/llm\";\nimport { LLMAsAJudgeScorer, MCQScorer } from \"@/scorers\";\nimport { IdGenerator, ScoringMethod } from \"@/types\";\nimport { idGeneratorUUIDv7 } from \"@/utils\";\nimport { ChatCompletionMessageParam } from \"openai/resources/index\";\nimport Handlebars from \"handlebars\";\nimport z from \"zod\";\nimport {\n  MCQResponseSchemaV1,\n  MCQScoreSchemaV1,\n  MCQTestCaseSchemaV1,\n  MCQTestCaseV1,\n} from \"./schema-sets/mcq.v1\";\nimport {\n  QAResponseSchemaV1,\n  QAScoreSchemaV1,\n  QATestCaseSchemaV1,\n  QATestCaseV1,\n} from \"./schema-sets/qa.v1\";\nimport { PEERBENCH_NAMESPACE } from \"@/constants\";\n\nexport const peerbenchRunner = defineRunner(\n  {\n    schemaSets: [\n      {\n        testCase: MCQTestCaseSchemaV1,\n        response: MCQResponseSchemaV1,\n        score: MCQScoreSchemaV1,\n      },\n      {\n        testCase: QATestCaseSchemaV1,\n        response: QAResponseSchemaV1,\n        score: QAScoreSchemaV1,\n      },\n    ],\n    providers: [AbstractLLMProvider],\n    scorers: [LLMAsAJudgeScorer, MCQScorer],\n\n    runConfigSchema: {\n      model: z.string(),\n      llmJudgeModel: z.string().optional(),\n      llmJudgeSystemPrompt: SimpleSystemPromptSchemaV1.optional(),\n      llmJudgeFieldsToExtract: z\n        .record(z.string(), z.custom<z.ZodType>())\n        .optional(),\n      systemPrompt: SimpleSystemPromptSchemaV1.optional(),\n      templateVariables: z.record(z.string(), z.string()).optional(),\n    },\n  },\n  async (params) => {\n    const { testCase, provider, scorer, runConfig } = params;\n    const messages: ChatCompletionMessageParam[] = [];\n\n    if (runConfig.systemPrompt) {\n      messages.push({\n        role: \"system\",\n        content: runConfig.systemPrompt.content,\n      });\n    }\n\n    if (testCase.kind === \"llm/mcq.tc\") {\n      messages.push({\n        role: \"user\",\n        content: formatMCQ(testCase),\n      });\n      templateMessages(messages, runConfig.templateVariables ?? {});\n\n      return runMCQ({\n        testCase,\n        messages,\n        provider,\n        scorer,\n        runConfig,\n        idGenerators: {\n          response: params.idGenerators?.response ?? idGeneratorUUIDv7,\n          score: params.idGenerators?.score ?? idGeneratorUUIDv7,\n        },\n      });\n    }\n\n    if (testCase.kind === \"llm/qa.tc\") {\n      if (\n        scorer &&\n        scorer?.kind !== (`${PEERBENCH_NAMESPACE}/llm-as-a-judge` as const)\n      ) {\n        throw new Error(\n          `QA test cases can only be scored with an LLM as a judge scorer, but ${scorer?.kind} was provided`\n        );\n      }\n\n      messages.push({\n        role: \"user\",\n        content: testCase.question,\n      });\n      templateMessages(messages, runConfig.templateVariables ?? {});\n\n      return runQA({\n        testCase,\n        messages,\n        provider,\n        scorer,\n        runConfig,\n        idGenerators: {\n          response: params.idGenerators?.response ?? idGeneratorUUIDv7,\n          score: params.idGenerators?.score ?? idGeneratorUUIDv7,\n        },\n      });\n    }\n\n    throw new Error(\"Unsupported test case kind\");\n  }\n);\n\nasync function runQA(params: {\n  messages: ChatCompletionMessageParam[];\n  testCase: QATestCaseV1;\n  provider: AbstractLLMProvider;\n  scorer?: LLMAsAJudgeScorer;\n  runConfig: {\n    model: string;\n    llmJudgeModel?: string;\n    llmJudgeSystemPrompt?: SimpleSystemPromptV1;\n    llmJudgeFieldsToExtract?: Record<string, z.ZodType>;\n    systemPrompt?: SimpleSystemPromptV1;\n  };\n  idGenerators: {\n    response: IdGenerator;\n    score: IdGenerator;\n  };\n}) {\n  const { messages, testCase, provider, scorer, runConfig } = params;\n\n  const providerResponse = await provider.forward({\n    model: runConfig.model,\n    messages,\n  });\n\n  const response = await QAResponseSchemaV1.newWithId(\n    {\n      data: providerResponse.data,\n      startedAt: providerResponse.startedAt,\n      completedAt: providerResponse.completedAt,\n      testCaseId: testCase.id,\n      modelSlug: runConfig.model,\n      provider: provider.kind,\n      systemPromptId: runConfig.systemPrompt?.id,\n\n      inputTokensUsed: providerResponse.inputTokensUsed,\n      outputTokensUsed: providerResponse.outputTokensUsed,\n      inputCost: providerResponse.inputCost,\n      outputCost: providerResponse.outputCost,\n    },\n    params.idGenerators?.response ?? idGeneratorUUIDv7\n  );\n\n  if (scorer?.kind === (`${PEERBENCH_NAMESPACE}/llm-as-a-judge` as const)) {\n    if (!runConfig.llmJudgeModel) {\n      throw new Error(\n        \"LLM judge model is required when using LLM as a judge scorer\"\n      );\n    }\n\n    const scorerResult = await scorer.score({\n      model: runConfig.llmJudgeModel,\n      response: response.data,\n      rubric: `Expected/Valid answers: ${testCase.goodAnswers.join(\"\\n\")}\\nInvalid answers: ${testCase.badAnswers.join(\"\\n\")}`,\n      systemPrompt: runConfig.llmJudgeSystemPrompt?.content,\n      criteria: [\n        {\n          id: \"correctness\",\n          description:\n            \"Is the response matches with the expected/valid answers in terms of meaning?\",\n          weight: 1,\n        },\n      ],\n      fieldsToExtract: runConfig.llmJudgeFieldsToExtract ?? {},\n    });\n\n    if (scorerResult !== null) {\n      const score = await QAScoreSchemaV1.newWithId(\n        {\n          scoringMethod: ScoringMethod.ai,\n          value: scorerResult.value,\n          responseId: response.id,\n          explanation: scorerResult.explanation,\n          scorerAIInputCost: scorerResult.inputCost,\n          scorerAIOutputCost: scorerResult.outputCost,\n          scorerAIInputTokensUsed: scorerResult.inputTokensUsed,\n          scorerAIOutputTokensUsed: scorerResult.outputTokensUsed,\n          scorerAIProvider: scorerResult.provider,\n          scorerAIModelSlug: runConfig.llmJudgeModel,\n          scorerAISystemPromptId: runConfig.llmJudgeSystemPrompt?.id,\n          metadata: {\n            ...scorerResult.metadata,\n            extractedFields: scorerResult.extractedFields,\n          },\n        },\n        params.idGenerators?.score ?? idGeneratorUUIDv7\n      );\n\n      return { response, score };\n    }\n  }\n\n  return { response };\n}\n\nasync function runMCQ(params: {\n  messages: ChatCompletionMessageParam[];\n  testCase: MCQTestCaseV1;\n  provider: AbstractLLMProvider;\n  scorer?: MCQScorer | LLMAsAJudgeScorer;\n  runConfig: {\n    model: string;\n    llmJudgeModel?: string;\n    llmJudgeSystemPrompt?: SimpleSystemPromptV1;\n    llmJudgeFieldsToExtract?: Record<string, z.ZodType>;\n    systemPrompt?: SimpleSystemPromptV1;\n  };\n  idGenerators: {\n    response: IdGenerator;\n    score: IdGenerator;\n  };\n}) {\n  const { messages, testCase, provider, scorer, runConfig } = params;\n\n  const providerResponse = await provider.forward({\n    model: runConfig.model,\n    messages,\n  });\n\n  const response = await MCQResponseSchemaV1.newWithId(\n    {\n      data: providerResponse.data,\n      startedAt: providerResponse.startedAt,\n      completedAt: providerResponse.completedAt,\n      testCaseId: testCase.id,\n      modelSlug: runConfig.model,\n      provider: provider.kind,\n      systemPromptId: runConfig.systemPrompt?.id,\n\n      inputTokensUsed: providerResponse.inputTokensUsed,\n      outputTokensUsed: providerResponse.outputTokensUsed,\n      inputCost: providerResponse.inputCost,\n      outputCost: providerResponse.outputCost,\n    },\n    params.idGenerators?.response ?? idGeneratorUUIDv7\n  );\n\n  if (scorer?.kind === (`${PEERBENCH_NAMESPACE}/mcq` as const)) {\n    const scorerResult = await scorer.score({\n      response: response.data,\n      choices: testCase.options,\n      correctAnswers: testCase.correctAnswerKeys,\n    });\n\n    if (scorerResult !== null) {\n      const score = await MCQScoreSchemaV1.newWithId(\n        {\n          scoringMethod: ScoringMethod.algo,\n          value: scorerResult.value,\n          responseId: response.id,\n          extractedAnswers: scorerResult.extractedAnswers,\n          explanation: scorerResult.explanation,\n          metadata: scorerResult.metadata,\n        },\n        params.idGenerators?.score ?? idGeneratorUUIDv7\n      );\n\n      return { response, score };\n    }\n  }\n\n  if (scorer?.kind === (`${PEERBENCH_NAMESPACE}/llm-as-a-judge` as const)) {\n    if (!runConfig.llmJudgeModel) {\n      throw new Error(\n        \"LLM judge model is required when using LLM as a judge scorer\"\n      );\n    }\n\n    const scorerResult = await scorer.score({\n      model: runConfig.llmJudgeModel,\n      criteria: [\n        {\n          id: \"correctness\",\n          description:\n            \"Is the given answer key matches with one of the correct answer keys?\",\n          weight: 1,\n        },\n      ],\n      rubric: `Answer text itself or the key (A, B, C) is accepted\nValid answer keys: ${testCase.correctAnswerKeys.map((key) => `- ${key}`).join(\"\\n\")}\nValid Answer texts: ${testCase.correctAnswerKeys.map((key) => `- ${testCase.options?.[key] ?? \"\"}`).join(\"\\n\")}`,\n      fieldsToExtract: {\n        extractedAnswers: z\n          .string()\n          .array()\n          .describe(\n            \"The extracted answer keys, valid or invalid (even if the answer text is provided rather than the key)\"\n          ),\n        ...(runConfig.llmJudgeFieldsToExtract ?? {}),\n      },\n      response: response.data,\n      systemPrompt: runConfig.llmJudgeSystemPrompt?.content,\n    });\n\n    if (scorerResult !== null) {\n      const { extractedAnswers, ...extractedFields } =\n        scorerResult.extractedFields;\n      const score = await MCQScoreSchemaV1.newWithId(\n        {\n          scoringMethod: ScoringMethod.ai,\n          value: scorerResult.value,\n          extractedAnswers,\n          responseId: response.id,\n          explanation: scorerResult.explanation,\n          scorerAIInputCost: scorerResult.inputCost,\n          scorerAIOutputCost: scorerResult.outputCost,\n          scorerAIInputTokensUsed: scorerResult.inputTokensUsed,\n          scorerAIOutputTokensUsed: scorerResult.outputTokensUsed,\n          scorerAIProvider: scorerResult.provider,\n          scorerAIModelSlug: runConfig.llmJudgeModel,\n          scorerAISystemPromptId: runConfig.llmJudgeSystemPrompt?.id,\n          metadata: {\n            ...scorerResult.metadata,\n            extractedFields,\n          },\n        },\n        params.idGenerators?.score ?? idGeneratorUUIDv7\n      );\n\n      return { response, score };\n    }\n  }\n\n  return { response };\n}\n\nfunction formatMCQ(testCase: MCQTestCaseV1) {\n  return `Question: ${testCase.question}\\nOptions:\\n${Object.entries(\n    testCase.options ?? {}\n  )\n    .map(([key, value]) => `${key}: ${value}`)\n    .join(\"\\n\")}`;\n}\n\nfunction templateMessages(\n  messages: ChatCompletionMessageParam[],\n  templateVariables: Record<string, string>\n) {\n  for (let i = 0; i < messages.length; i++) {\n    const template = Handlebars.compile(messages[i]!.content);\n    messages[i]!.content = template(templateVariables);\n  }\n}\n","import { JSONFileStorage } from \"@/storages/json-file\";\nimport {\n  MCQResponseSchemaV1,\n  MCQResponseV1,\n  MCQScoreSchemaV1,\n  MCQScoreV1,\n  MCQTestCaseSchemaV1,\n  MCQTestCaseV1,\n} from \"../schema-sets/mcq.v1\";\nimport {\n  QAResponseSchemaV1,\n  QAResponseV1,\n  QAScoreSchemaV1,\n  QAScoreV1,\n  QATestCaseSchemaV1,\n  QATestCaseV1,\n} from \"../schema-sets/qa.v1\";\nimport {\n  MultiTurnResponseSchemaV1,\n  MultiTurnResponseV1,\n  MultiTurnScoreSchemaV1,\n  MultiTurnScoreV1,\n  MultiTurnTestCaseSchemaV1,\n  MultiTurnTestCaseV1,\n} from \"../schema-sets/multi-turn.v1\";\nimport z from \"zod\";\n\nexport class PeerbenchJSONStorage extends JSONFileStorage<\n  | MCQTestCaseV1\n  | MCQResponseV1\n  | MCQScoreV1\n  | QATestCaseV1\n  | QAResponseV1\n  | QAScoreV1\n  | MultiTurnTestCaseV1\n  | MultiTurnResponseV1\n  | MultiTurnScoreV1\n> {\n  constructor(config: { path: string; chunkSize?: number }) {\n    super({\n      path: config.path,\n      chunkSize: config.chunkSize,\n\n      schema: z.union([\n        MCQTestCaseSchemaV1,\n        MCQResponseSchemaV1,\n        MCQScoreSchemaV1,\n        QATestCaseSchemaV1,\n        QAResponseSchemaV1,\n        QAScoreSchemaV1,\n        MultiTurnTestCaseSchemaV1,\n        MultiTurnResponseSchemaV1,\n        MultiTurnScoreSchemaV1,\n      ]),\n    });\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;;;ACWA,SAAS,SAAS;AAEX,IAAM,UAAU;AAEhB,IAAM,sBAAsB,qBAAqB;AAAA,EACtD,YAAY;AAAA,EACZ,WAAW;AAAA,EACX,MAAM;AAAA,EACN,eAAe;AAAA,EACf,QAAQ;AAAA,IACN,UAAU,EAAE,OAAO;AAAA,IACnB,SAAS,EAAE,OAAO,EAAE,OAAO,GAAG,EAAE,OAAO,CAAC;AAAA,IACxC,mBAAmB,EAAE,OAAO,EAAE,MAAM;AAAA,EACtC;AACF,CAAC;AAGM,IAAM,sBAAsB,qBAAqB;AAAA,EACtD,YAAY;AAAA,EACZ,WAAW;AAAA,EACX,MAAM;AAAA,EACN,eAAe;AAAA,EACf,QAAQ;AAAA,IACN,GAAG;AAAA,EACL;AACF,CAAC;AAGM,IAAM,mBAAmB,kBAAkB;AAAA,EAChD,YAAY;AAAA,EACZ,WAAW;AAAA,EACX,MAAM;AAAA,EACN,eAAe;AAAA,EACf,QAAQ;AAAA,IACN,GAAG;AAAA,IACH,kBAAkB,EAAE,MAAM,EAAE,OAAO,CAAC;AAAA,EACtC;AACF,CAAC;;;ACrCD,SAAS,KAAAA,UAAS;AAEX,IAAM,gBAAgB;AAEtB,IAAM,4BAA4B,qBAAqB;AAAA,EAC5D,YAAY;AAAA,EACZ,WAAW;AAAA,EACX,MAAM;AAAA,EACN,eAAe;AAAA,EACf,QAAQ;AAAA,IACN,UAAUA,GACP,OAAO;AAAA,MACN,MAAMA,GAAE,OAAO;AAAA,MACf,SAASA,GAAE,OAAO;AAAA,MAClB,aAAaA,GAAE,OAAO,EAAE,MAAM,EAAE,SAAS;AAAA,MACzC,YAAYA,GAAE,OAAO,EAAE,MAAM,EAAE,SAAS;AAAA,IAC1C,CAAC,EACA,MAAM;AAAA,IAET,UAAUA,GAAE,OAAO,EAAE,SAAS;AAAA,IAC9B,iBAAiBA,GAAE,OAAO,EAAE,SAAS;AAAA,EACvC;AACF,CAAC;AAGM,IAAM,4BAA4B,qBAAqB;AAAA,EAC5D,YAAY;AAAA,EACZ,WAAW;AAAA,EACX,MAAM;AAAA,EACN,eAAe;AAAA,EACf,QAAQ;AAAA,IACN,GAAG;AAAA,IACH,SAASA,GACN,OAAO;AAAA,MACN,cAAcA,GAAE,OAAO;AAAA,MACvB,WAAWA,GAAE,OAAO;AAAA,MACpB,aAAaA,GAAE,OAAO;AAAA,MACtB,MAAMA,GAAE,OAAO;AAAA,MAEf,iBAAiBA,GAAE,OAAO,EAAE,SAAS;AAAA,MACrC,kBAAkBA,GAAE,OAAO,EAAE,SAAS;AAAA,MACtC,WAAWA,GAAE,OAAO,EAAE,SAAS;AAAA,MAC/B,YAAYA,GAAE,OAAO,EAAE,SAAS;AAAA,IAClC,CAAC,EACA,MAAM;AAAA,EACX;AACF,CAAC;AAGM,IAAM,yBAAyB,kBAAkB;AAAA,EACtD,YAAY;AAAA,EACZ,WAAW;AAAA,EACX,MAAM;AAAA,EACN,eAAe;AAAA,EACf,QAAQ;AAAA,IACN,GAAG;AAAA,IACH,kBAAkBA,GACf,OAAO;AAAA,MACN,YAAYA,GAAE,OAAO;AAAA,MACrB,OAAOA,GAAE,OAAO;AAAA,IAClB,CAAC,EACA,MAAM;AAAA,EACX;AACF,CAAC;;;AC/DD,SAAS,KAAAC,UAAS;AAEX,IAAM,SAAS;AAEf,IAAM,qBAAqB,qBAAqB;AAAA,EACrD,YAAY;AAAA,EACZ,WAAW;AAAA,EACX,MAAM;AAAA,EACN,eAAe;AAAA,EACf,QAAQ;AAAA,IACN,UAAUA,GAAE,OAAO;AAAA,IACnB,aAAaA,GAAE,OAAO,EAAE,MAAM;AAAA,IAC9B,YAAYA,GAAE,OAAO,EAAE,MAAM;AAAA,EAC/B;AACF,CAAC;AAGM,IAAM,qBAAqB,qBAAqB;AAAA,EACrD,YAAY;AAAA,EACZ,WAAW;AAAA,EACX,MAAM;AAAA,EACN,eAAe;AAAA,EACf,QAAQ;AAAA,IACN,GAAG;AAAA,EACL;AACF,CAAC;AAGM,IAAM,kBAAkB,kBAAkB;AAAA,EAC/C,YAAY;AAAA,EACZ,WAAW;AAAA,EACX,MAAM;AAAA,EACN,eAAe;AAAA,EACf,QAAQ;AAAA,IACN,GAAG;AAAA,EACL;AACF,CAAC;;;ACrCD,OAAO,gBAAgB;AACvB,OAAOC,QAAO;AAeP,IAAM,kBAAkB;AAAA,EAC7B;AAAA,IACE,YAAY;AAAA,MACV;AAAA,QACE,UAAU;AAAA,QACV,UAAU;AAAA,QACV,OAAO;AAAA,MACT;AAAA,MACA;AAAA,QACE,UAAU;AAAA,QACV,UAAU;AAAA,QACV,OAAO;AAAA,MACT;AAAA,IACF;AAAA,IACA,WAAW,CAAC,mBAAmB;AAAA,IAC/B,SAAS,CAAC,mBAAmB,SAAS;AAAA,IAEtC,iBAAiB;AAAA,MACf,OAAOC,GAAE,OAAO;AAAA,MAChB,eAAeA,GAAE,OAAO,EAAE,SAAS;AAAA,MACnC,sBAAsB,2BAA2B,SAAS;AAAA,MAC1D,yBAAyBA,GACtB,OAAOA,GAAE,OAAO,GAAGA,GAAE,OAAkB,CAAC,EACxC,SAAS;AAAA,MACZ,cAAc,2BAA2B,SAAS;AAAA,MAClD,mBAAmBA,GAAE,OAAOA,GAAE,OAAO,GAAGA,GAAE,OAAO,CAAC,EAAE,SAAS;AAAA,IAC/D;AAAA,EACF;AAAA,EACA,OAAO,WAAW;AAChB,UAAM,EAAE,UAAU,UAAU,QAAQ,UAAU,IAAI;AAClD,UAAM,WAAyC,CAAC;AAEhD,QAAI,UAAU,cAAc;AAC1B,eAAS,KAAK;AAAA,QACZ,MAAM;AAAA,QACN,SAAS,UAAU,aAAa;AAAA,MAClC,CAAC;AAAA,IACH;AAEA,QAAI,SAAS,SAAS,cAAc;AAClC,eAAS,KAAK;AAAA,QACZ,MAAM;AAAA,QACN,SAAS,UAAU,QAAQ;AAAA,MAC7B,CAAC;AACD,uBAAiB,UAAU,UAAU,qBAAqB,CAAC,CAAC;AAE5D,aAAO,OAAO;AAAA,QACZ;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA,cAAc;AAAA,UACZ,UAAU,OAAO,cAAc,YAAY;AAAA,UAC3C,OAAO,OAAO,cAAc,SAAS;AAAA,QACvC;AAAA,MACF,CAAC;AAAA,IACH;AAEA,QAAI,SAAS,SAAS,aAAa;AACjC,UACE,UACA,QAAQ,SAAU,GAAG,mBAAmB,mBACxC;AACA,cAAM,IAAI;AAAA,UACR,uEAAuE,QAAQ,IAAI;AAAA,QACrF;AAAA,MACF;AAEA,eAAS,KAAK;AAAA,QACZ,MAAM;AAAA,QACN,SAAS,SAAS;AAAA,MACpB,CAAC;AACD,uBAAiB,UAAU,UAAU,qBAAqB,CAAC,CAAC;AAE5D,aAAO,MAAM;AAAA,QACX;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA,cAAc;AAAA,UACZ,UAAU,OAAO,cAAc,YAAY;AAAA,UAC3C,OAAO,OAAO,cAAc,SAAS;AAAA,QACvC;AAAA,MACF,CAAC;AAAA,IACH;AAEA,UAAM,IAAI,MAAM,4BAA4B;AAAA,EAC9C;AACF;AAEA,eAAe,MAAM,QAgBlB;AACD,QAAM,EAAE,UAAU,UAAU,UAAU,QAAQ,UAAU,IAAI;AAE5D,QAAM,mBAAmB,MAAM,SAAS,QAAQ;AAAA,IAC9C,OAAO,UAAU;AAAA,IACjB;AAAA,EACF,CAAC;AAED,QAAM,WAAW,MAAM,mBAAmB;AAAA,IACxC;AAAA,MACE,MAAM,iBAAiB;AAAA,MACvB,WAAW,iBAAiB;AAAA,MAC5B,aAAa,iBAAiB;AAAA,MAC9B,YAAY,SAAS;AAAA,MACrB,WAAW,UAAU;AAAA,MACrB,UAAU,SAAS;AAAA,MACnB,gBAAgB,UAAU,cAAc;AAAA,MAExC,iBAAiB,iBAAiB;AAAA,MAClC,kBAAkB,iBAAiB;AAAA,MACnC,WAAW,iBAAiB;AAAA,MAC5B,YAAY,iBAAiB;AAAA,IAC/B;AAAA,IACA,OAAO,cAAc,YAAY;AAAA,EACnC;AAEA,MAAI,QAAQ,SAAU,GAAG,mBAAmB,mBAA6B;AACvE,QAAI,CAAC,UAAU,eAAe;AAC5B,YAAM,IAAI;AAAA,QACR;AAAA,MACF;AAAA,IACF;AAEA,UAAM,eAAe,MAAM,OAAO,MAAM;AAAA,MACtC,OAAO,UAAU;AAAA,MACjB,UAAU,SAAS;AAAA,MACnB,QAAQ,2BAA2B,SAAS,YAAY,KAAK,IAAI,CAAC;AAAA,mBAAsB,SAAS,WAAW,KAAK,IAAI,CAAC;AAAA,MACtH,cAAc,UAAU,sBAAsB;AAAA,MAC9C,UAAU;AAAA,QACR;AAAA,UACE,IAAI;AAAA,UACJ,aACE;AAAA,UACF,QAAQ;AAAA,QACV;AAAA,MACF;AAAA,MACA,iBAAiB,UAAU,2BAA2B,CAAC;AAAA,IACzD,CAAC;AAED,QAAI,iBAAiB,MAAM;AACzB,YAAM,QAAQ,MAAM,gBAAgB;AAAA,QAClC;AAAA,UACE,eAAe,cAAc;AAAA,UAC7B,OAAO,aAAa;AAAA,UACpB,YAAY,SAAS;AAAA,UACrB,aAAa,aAAa;AAAA,UAC1B,mBAAmB,aAAa;AAAA,UAChC,oBAAoB,aAAa;AAAA,UACjC,yBAAyB,aAAa;AAAA,UACtC,0BAA0B,aAAa;AAAA,UACvC,kBAAkB,aAAa;AAAA,UAC/B,mBAAmB,UAAU;AAAA,UAC7B,wBAAwB,UAAU,sBAAsB;AAAA,UACxD,UAAU;AAAA,YACR,GAAG,aAAa;AAAA,YAChB,iBAAiB,aAAa;AAAA,UAChC;AAAA,QACF;AAAA,QACA,OAAO,cAAc,SAAS;AAAA,MAChC;AAEA,aAAO,EAAE,UAAU,MAAM;AAAA,IAC3B;AAAA,EACF;AAEA,SAAO,EAAE,SAAS;AACpB;AAEA,eAAe,OAAO,QAgBnB;AACD,QAAM,EAAE,UAAU,UAAU,UAAU,QAAQ,UAAU,IAAI;AAE5D,QAAM,mBAAmB,MAAM,SAAS,QAAQ;AAAA,IAC9C,OAAO,UAAU;AAAA,IACjB;AAAA,EACF,CAAC;AAED,QAAM,WAAW,MAAM,oBAAoB;AAAA,IACzC;AAAA,MACE,MAAM,iBAAiB;AAAA,MACvB,WAAW,iBAAiB;AAAA,MAC5B,aAAa,iBAAiB;AAAA,MAC9B,YAAY,SAAS;AAAA,MACrB,WAAW,UAAU;AAAA,MACrB,UAAU,SAAS;AAAA,MACnB,gBAAgB,UAAU,cAAc;AAAA,MAExC,iBAAiB,iBAAiB;AAAA,MAClC,kBAAkB,iBAAiB;AAAA,MACnC,WAAW,iBAAiB;AAAA,MAC5B,YAAY,iBAAiB;AAAA,IAC/B;AAAA,IACA,OAAO,cAAc,YAAY;AAAA,EACnC;AAEA,MAAI,QAAQ,SAAU,GAAG,mBAAmB,QAAkB;AAC5D,UAAM,eAAe,MAAM,OAAO,MAAM;AAAA,MACtC,UAAU,SAAS;AAAA,MACnB,SAAS,SAAS;AAAA,MAClB,gBAAgB,SAAS;AAAA,IAC3B,CAAC;AAED,QAAI,iBAAiB,MAAM;AACzB,YAAM,QAAQ,MAAM,iBAAiB;AAAA,QACnC;AAAA,UACE,eAAe,cAAc;AAAA,UAC7B,OAAO,aAAa;AAAA,UACpB,YAAY,SAAS;AAAA,UACrB,kBAAkB,aAAa;AAAA,UAC/B,aAAa,aAAa;AAAA,UAC1B,UAAU,aAAa;AAAA,QACzB;AAAA,QACA,OAAO,cAAc,SAAS;AAAA,MAChC;AAEA,aAAO,EAAE,UAAU,MAAM;AAAA,IAC3B;AAAA,EACF;AAEA,MAAI,QAAQ,SAAU,GAAG,mBAAmB,mBAA6B;AACvE,QAAI,CAAC,UAAU,eAAe;AAC5B,YAAM,IAAI;AAAA,QACR;AAAA,MACF;AAAA,IACF;AAEA,UAAM,eAAe,MAAM,OAAO,MAAM;AAAA,MACtC,OAAO,UAAU;AAAA,MACjB,UAAU;AAAA,QACR;AAAA,UACE,IAAI;AAAA,UACJ,aACE;AAAA,UACF,QAAQ;AAAA,QACV;AAAA,MACF;AAAA,MACA,QAAQ;AAAA,qBACO,SAAS,kBAAkB,IAAI,CAAC,QAAQ,KAAK,GAAG,EAAE,EAAE,KAAK,IAAI,CAAC;AAAA,sBAC7D,SAAS,kBAAkB,IAAI,CAAC,QAAQ,KAAK,SAAS,UAAU,GAAG,KAAK,EAAE,EAAE,EAAE,KAAK,IAAI,CAAC;AAAA,MACxG,iBAAiB;AAAA,QACf,kBAAkBA,GACf,OAAO,EACP,MAAM,EACN;AAAA,UACC;AAAA,QACF;AAAA,QACF,GAAI,UAAU,2BAA2B,CAAC;AAAA,MAC5C;AAAA,MACA,UAAU,SAAS;AAAA,MACnB,cAAc,UAAU,sBAAsB;AAAA,IAChD,CAAC;AAED,QAAI,iBAAiB,MAAM;AACzB,YAAM,EAAE,kBAAkB,GAAG,gBAAgB,IAC3C,aAAa;AACf,YAAM,QAAQ,MAAM,iBAAiB;AAAA,QACnC;AAAA,UACE,eAAe,cAAc;AAAA,UAC7B,OAAO,aAAa;AAAA,UACpB;AAAA,UACA,YAAY,SAAS;AAAA,UACrB,aAAa,aAAa;AAAA,UAC1B,mBAAmB,aAAa;AAAA,UAChC,oBAAoB,aAAa;AAAA,UACjC,yBAAyB,aAAa;AAAA,UACtC,0BAA0B,aAAa;AAAA,UACvC,kBAAkB,aAAa;AAAA,UAC/B,mBAAmB,UAAU;AAAA,UAC7B,wBAAwB,UAAU,sBAAsB;AAAA,UACxD,UAAU;AAAA,YACR,GAAG,aAAa;AAAA,YAChB;AAAA,UACF;AAAA,QACF;AAAA,QACA,OAAO,cAAc,SAAS;AAAA,MAChC;AAEA,aAAO,EAAE,UAAU,MAAM;AAAA,IAC3B;AAAA,EACF;AAEA,SAAO,EAAE,SAAS;AACpB;AAEA,SAAS,UAAU,UAAyB;AAC1C,SAAO,aAAa,SAAS,QAAQ;AAAA;AAAA,EAAe,OAAO;AAAA,IACzD,SAAS,WAAW,CAAC;AAAA,EACvB,EACG,IAAI,CAAC,CAAC,KAAK,KAAK,MAAM,GAAG,GAAG,KAAK,KAAK,EAAE,EACxC,KAAK,IAAI,CAAC;AACf;AAEA,SAAS,iBACP,UACA,mBACA;AACA,WAAS,IAAI,GAAG,IAAI,SAAS,QAAQ,KAAK;AACxC,UAAM,WAAW,WAAW,QAAQ,SAAS,CAAC,EAAG,OAAO;AACxD,aAAS,CAAC,EAAG,UAAU,SAAS,iBAAiB;AAAA,EACnD;AACF;;;AC9UA,OAAOC,QAAO;AAEP,IAAM,uBAAN,cAAmC,gBAUxC;AAAA,EACA,YAAY,QAA8C;AACxD,UAAM;AAAA,MACJ,MAAM,OAAO;AAAA,MACb,WAAW,OAAO;AAAA,MAElB,QAAQA,GAAE,MAAM;AAAA,QACd;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,MACF,CAAC;AAAA,IACH,CAAC;AAAA,EACH;AACF;","names":["z","z","z","z","z"]}

package/dist/{chunk-DUBKY73H.js → chunk-4UBK6452.js}

@@ -1,13 +1,3 @@
-// src/utils/string.ts
-function bufferToString(buffer) {
-  const decoder = new TextDecoder();
-  return decoder.decode(buffer);
-}
-function stringToBuffer(str) {
-  const encoder = new TextEncoder();
-  return encoder.encode(str);
-}
-
 // src/utils/sleep.ts
 async function sleep(ms, signal) {
   return new Promise((resolve, reject) => {
@@ -111,6 +101,16 @@ var RateLimiter = class {
   }
 };
 
+// src/utils/string.ts
+function bufferToString(buffer, encoding = "utf-8") {
+  const decoder = new TextDecoder(encoding);
+  return decoder.decode(buffer);
+}
+function stringToBuffer(str) {
+  const encoder = new TextEncoder();
+  return encoder.encode(str);
+}
+
 // src/utils/id-generator.ts
 import { v7 as uuidv7 } from "uuid";
 var idGeneratorUUIDv7 = () => {
@@ -118,11 +118,11 @@ var idGeneratorUUIDv7 = () => {
 };
 
 export {
-  bufferToString,
-  stringToBuffer,
   sleep,
   parseResponseAsJSON,
   RateLimiter,
+  bufferToString,
+  stringToBuffer,
   idGeneratorUUIDv7
 };
-//# sourceMappingURL=chunk-DUBKY73H.js.map
+//# sourceMappingURL=chunk-4UBK6452.js.map

package/dist/chunk-4UBK6452.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/utils/sleep.ts","../src/utils/llm.ts","../src/utils/rate-limiter.ts","../src/utils/string.ts","../src/utils/id-generator.ts"],"sourcesContent":["export async function sleep(ms: number, signal?: AbortSignal) {\n  return new Promise<void>((resolve, reject) => {\n    if (signal?.aborted) {\n      return reject(new Error(\"Aborted\"));\n    }\n\n    const timeout = setTimeout(() => {\n      cleanup();\n      resolve();\n    }, ms);\n\n    const onAbort = () => {\n      cleanup();\n      reject(new Error(\"Aborted\"));\n    };\n\n    const cleanup = () => {\n      clearTimeout(timeout);\n      signal?.removeEventListener(\"abort\", onAbort);\n    };\n\n    signal?.addEventListener(\"abort\", onAbort);\n  });\n}\n","import { jsonrepair } from \"jsonrepair\";\n\n/**\n * Tries to repair and parse LLM response as a JSON object. LLM must\n * be configured to return a JSON object. This function only helps to\n * get rid out of some additional formatting (e.g. ```json) and repair\n * the JSON syntax (e.g missing comma, single quotes instead double).\n */\nexport function parseResponseAsJSON<T>(response: string) {\n  try {\n    return JSON.parse(jsonrepair(response)) as T;\n  } catch (e) {\n    if (process?.env?.PB_SDK_DEBUG) {\n      console.log(\"Original response\", JSON.stringify(response));\n      console.error(\"Error parsing response as JSON\", e);\n    }\n  }\n}\n","import { sleep } from \"./sleep\";\n\nexport interface RateLimiterOptions {\n  /**\n   * Maximum weight of calls allowed within a time window. Each call has a weight of 1 by default.\n   * Rate limiting is disabled when this is 0 or negative.\n   * @default 25\n   */\n  maxWeight?: number;\n\n  /**\n   * Time window in milliseconds for rate limiting\n   * @default 1000\n   */\n  timeWindow?: number;\n}\n\nexport interface RateLimiterCallOptions {\n  /**\n   * Weight of the call. Rate limiting is applied based on the total weight of the calls.\n   * @default 1\n   */\n  weight?: number;\n\n  /**\n   * Abort signal for cancellation\n   */\n  signal?: AbortSignal;\n}\n\n/**\n * Generic rate limiter. It can be used to limit async function calls\n * by a given number of calls within a time window.\n */\nexport class RateLimiter {\n  maxWeight: number;\n  timeWindow: number;\n\n  private timestamps: number[] = [];\n\n  constructor(options: RateLimiterOptions = {}) {\n    this.maxWeight = options.maxWeight ?? 25;\n    this.timeWindow = options.timeWindow ?? 1000;\n  }\n\n  /**\n   * Checks if rate limiting is disabled\n   */\n  isDisabled(): boolean {\n    return this.maxWeight <= 0;\n  }\n\n  /**\n   * Disables rate limiting. Set `maxWeight` to re-enable it.\n   */\n  disable() {\n    this.maxWeight = 0;\n  }\n\n  /**\n   * Returns how many weight of calls are there in the current time window\n   */\n  getCurrentCalls(): number {\n    const now = Date.now();\n    this.timestamps = this.timestamps.filter(\n      (ts) => now - ts < this.timeWindow\n    );\n    return this.timestamps.length;\n  }\n\n  /**\n   * Executes the given function with rate limiting applied\n   */\n  async execute<T = unknown>(\n    func: () => Promise<T>,\n    { weight = 1, signal }: RateLimiterCallOptions = {}\n  ): Promise<T> {\n    // If maxWeight is zero/negative, execute immediately (no rate limiting)\n    if (this.maxWeight <= 0) {\n      return await func();\n    }\n\n    await this.waitForRateLimit(weight, signal);\n    return await func();\n  }\n\n  /**\n   * Waits until rate limit allows the specified weight of calls\n   */\n  private async waitForRateLimit(\n    weight: number,\n    signal?: AbortSignal\n  ): Promise<void> {\n    const now = Date.now();\n\n    // Clear timestamps from the old time period\n    this.timestamps = this.timestamps.filter(\n      (ts) => now - ts < this.timeWindow\n    );\n\n    // Check if there are empty slots for the requests\n    if (this.timestamps.length + weight <= this.maxWeight) {\n      for (let i = 0; i < weight; i++) {\n        this.timestamps.push(now);\n      }\n      return;\n    }\n\n    // Calculate how much time to wait for the next time window\n    const earliest = this.timestamps[0];\n    const waitTime = this.timeWindow - (now - (earliest ?? 0));\n\n    await sleep(waitTime, signal);\n\n    // Retry after waiting\n    return this.waitForRateLimit(weight, signal);\n  }\n\n  /**\n   * Resets the rate limited weight of calls\n   */\n  reset(): void {\n    this.timestamps = [];\n  }\n}\n","/**\n * Converts the given byte array to a string\n */\nexport function bufferToString(\n  buffer: Uint8Array,\n  encoding: BufferEncoding = \"utf-8\"\n): string {\n  const decoder = new TextDecoder(encoding);\n  return decoder.decode(buffer);\n}\n\n/**\n * Converts the given string to a byte array\n */\nexport function stringToBuffer(str: string): Uint8Array {\n  const encoder = new TextEncoder();\n  return encoder.encode(str);\n}\n","import { IdGenerator } from \"@/types\";\nimport { v7 as uuidv7 } from \"uuid\";\n\nexport const idGeneratorUUIDv7: IdGenerator = () => {\n  return uuidv7();\n};\n"],"mappings":";AAAA,eAAsB,MAAM,IAAY,QAAsB;AAC5D,SAAO,IAAI,QAAc,CAAC,SAAS,WAAW;AAC5C,QAAI,QAAQ,SAAS;AACnB,aAAO,OAAO,IAAI,MAAM,SAAS,CAAC;AAAA,IACpC;AAEA,UAAM,UAAU,WAAW,MAAM;AAC/B,cAAQ;AACR,cAAQ;AAAA,IACV,GAAG,EAAE;AAEL,UAAM,UAAU,MAAM;AACpB,cAAQ;AACR,aAAO,IAAI,MAAM,SAAS,CAAC;AAAA,IAC7B;AAEA,UAAM,UAAU,MAAM;AACpB,mBAAa,OAAO;AACpB,cAAQ,oBAAoB,SAAS,OAAO;AAAA,IAC9C;AAEA,YAAQ,iBAAiB,SAAS,OAAO;AAAA,EAC3C,CAAC;AACH;;;ACvBA,SAAS,kBAAkB;AAQpB,SAAS,oBAAuB,UAAkB;AACvD,MAAI;AACF,WAAO,KAAK,MAAM,WAAW,QAAQ,CAAC;AAAA,EACxC,SAAS,GAAG;AACV,QAAI,SAAS,KAAK,cAAc;AAC9B,cAAQ,IAAI,qBAAqB,KAAK,UAAU,QAAQ,CAAC;AACzD,cAAQ,MAAM,kCAAkC,CAAC;AAAA,IACnD;AAAA,EACF;AACF;;;ACiBO,IAAM,cAAN,MAAkB;AAAA,EACvB;AAAA,EACA;AAAA,EAEQ,aAAuB,CAAC;AAAA,EAEhC,YAAY,UAA8B,CAAC,GAAG;AAC5C,SAAK,YAAY,QAAQ,aAAa;AACtC,SAAK,aAAa,QAAQ,cAAc;AAAA,EAC1C;AAAA;AAAA;AAAA;AAAA,EAKA,aAAsB;AACpB,WAAO,KAAK,aAAa;AAAA,EAC3B;AAAA;AAAA;AAAA;AAAA,EAKA,UAAU;AACR,SAAK,YAAY;AAAA,EACnB;AAAA;AAAA;AAAA;AAAA,EAKA,kBAA0B;AACxB,UAAM,MAAM,KAAK,IAAI;AACrB,SAAK,aAAa,KAAK,WAAW;AAAA,MAChC,CAAC,OAAO,MAAM,KAAK,KAAK;AAAA,IAC1B;AACA,WAAO,KAAK,WAAW;AAAA,EACzB;AAAA;AAAA;AAAA;AAAA,EAKA,MAAM,QACJ,MACA,EAAE,SAAS,GAAG,OAAO,IAA4B,CAAC,GACtC;AAEZ,QAAI,KAAK,aAAa,GAAG;AACvB,aAAO,MAAM,KAAK;AAAA,IACpB;AAEA,UAAM,KAAK,iBAAiB,QAAQ,MAAM;AAC1C,WAAO,MAAM,KAAK;AAAA,EACpB;AAAA;AAAA;AAAA;AAAA,EAKA,MAAc,iBACZ,QACA,QACe;AACf,UAAM,MAAM,KAAK,IAAI;AAGrB,SAAK,aAAa,KAAK,WAAW;AAAA,MAChC,CAAC,OAAO,MAAM,KAAK,KAAK;AAAA,IAC1B;AAGA,QAAI,KAAK,WAAW,SAAS,UAAU,KAAK,WAAW;AACrD,eAAS,IAAI,GAAG,IAAI,QAAQ,KAAK;AAC/B,aAAK,WAAW,KAAK,GAAG;AAAA,MAC1B;AACA;AAAA,IACF;AAGA,UAAM,WAAW,KAAK,WAAW,CAAC;AAClC,UAAM,WAAW,KAAK,cAAc,OAAO,YAAY;AAEvD,UAAM,MAAM,UAAU,MAAM;AAG5B,WAAO,KAAK,iBAAiB,QAAQ,MAAM;AAAA,EAC7C;AAAA;AAAA;AAAA;AAAA,EAKA,QAAc;AACZ,SAAK,aAAa,CAAC;AAAA,EACrB;AACF;;;ACzHO,SAAS,eACd,QACA,WAA2B,SACnB;AACR,QAAM,UAAU,IAAI,YAAY,QAAQ;AACxC,SAAO,QAAQ,OAAO,MAAM;AAC9B;AAKO,SAAS,eAAe,KAAyB;AACtD,QAAM,UAAU,IAAI,YAAY;AAChC,SAAO,QAAQ,OAAO,GAAG;AAC3B;;;AChBA,SAAS,MAAM,cAAc;AAEtB,IAAM,oBAAiC,MAAM;AAClD,SAAO,OAAO;AAChB;","names":[]}

package/dist/chunk-ERALDEZY.js

@@ -0,0 +1,112 @@
+import {
+  bufferToString
+} from "./chunk-4UBK6452.js";
+
+// src/storages/abstract.ts
+var AbstractStorage = class {
+};
+
+// src/storages/file.ts
+import { open } from "fs/promises";
+var FileStorage = class extends AbstractStorage {
+  path;
+  codec;
+  fileHandle;
+  constructor(config) {
+    super();
+    this.path = config.path;
+    this.codec = config.codec;
+  }
+  async init() {
+    this.fileHandle = await open(this.path, "r+");
+  }
+  async read(_key, _params) {
+    throw new Error("Method not implemented.");
+  }
+  async readAll(_params) {
+    this.assertInitialized();
+    return await this.codec.readAll({
+      fileHandle: {
+        readAt: this.readAt.bind(this),
+        readChunks: this.readChunks.bind(this),
+        size: this.size.bind(this)
+      }
+    });
+  }
+  async write(_key, _value, _params) {
+    throw new Error("Method not implemented.");
+  }
+  async *readChunks(params) {
+    this.assertInitialized();
+    const fileSize = await this.size();
+    let offset = params.startOffset ?? 0;
+    while (offset < fileSize) {
+      const length = Math.min(params.chunkSize, fileSize - offset);
+      if (length === 0) break;
+      const bytes = await this.readAt(offset, length);
+      if (bytes.length === 0) break;
+      yield { offset, bytes };
+      offset += bytes.length;
+    }
+  }
+  async size() {
+    this.assertInitialized();
+    return await this.fileHandle.stat().then((stat) => stat.size);
+  }
+  async readAt(offset, length) {
+    this.assertInitialized();
+    const buffer = new Uint8Array(length);
+    const result = await this.fileHandle.read(buffer, 0, length, offset);
+    return new Uint8Array(buffer.subarray(0, result.bytesRead));
+  }
+  assertInitialized() {
+    if (!this.fileHandle) {
+      throw new Error("File Storage is not initialized");
+    }
+  }
+};
+var AbstractFileStorageCodec = class {
+  // TODO: Add other methods like write, append, read one etc.
+};
+
+// src/storages/json-file.ts
+var JSONFileStorage = class extends FileStorage {
+  constructor(config) {
+    super({
+      path: config.path,
+      codec: new JSONFileStorageCodec({
+        chunkSize: config.chunkSize,
+        schema: config.schema
+      })
+    });
+  }
+};
+var JSONFileStorageCodec = class extends AbstractFileStorageCodec {
+  chunkSize;
+  schema;
+  constructor(config) {
+    super();
+    this.chunkSize = config?.chunkSize ?? 64 * 1024;
+    this.schema = config.schema;
+  }
+  async readAll(params) {
+    const wholeFile = new Uint8Array(
+      await params.fileHandle.size()
+    );
+    for await (const chunk of params.fileHandle.readChunks({
+      chunkSize: this.chunkSize
+    })) {
+      wholeFile.set(chunk.bytes, chunk.offset);
+    }
+    return this.schema.array().parse(JSON.parse(bufferToString(wholeFile)));
+  }
+};
+
+export {
+  AbstractStorage,
+  FileStorage,
+  AbstractFileStorageCodec,
+  JSONFileStorage,
+  JSONFileStorageCodec
+};
+//# sourceMappingURL=chunk-ERALDEZY.js.map

package/dist/chunk-ERALDEZY.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/storages/abstract.ts","../src/storages/file.ts","../src/storages/json-file.ts"],"sourcesContent":["export abstract class AbstractStorage<TObject> {\n  abstract init(params?: unknown): Promise<void>;\n  abstract read(key: string, params?: unknown): Promise<TObject | null>;\n  abstract readAll(params?: unknown): Promise<TObject[]>;\n  abstract write(\n    key: string,\n    value: TObject,\n    params?: unknown\n  ): Promise<unknown>;\n}\n","import { FileHandle, open } from \"node:fs/promises\";\nimport { AbstractStorage } from \"./abstract\";\n\nexport class FileStorage<TObject> extends AbstractStorage<TObject> {\n  protected path: string;\n  protected codec: AbstractFileStorageCodec<TObject>;\n  protected fileHandle: FileHandle | undefined;\n\n  constructor(config: {\n    path: string;\n    codec: AbstractFileStorageCodec<TObject>;\n  }) {\n    super();\n    this.path = config.path;\n    this.codec = config.codec;\n  }\n\n  async init(): Promise<void> {\n    this.fileHandle = await open(this.path, \"r+\");\n  }\n\n  override async read(\n    _key: string,\n    _params?: unknown\n  ): Promise<TObject | null> {\n    throw new Error(\"Method not implemented.\");\n  }\n\n  override async readAll(_params?: unknown): Promise<TObject[]> {\n    this.assertInitialized();\n\n    return await this.codec.readAll({\n      fileHandle: {\n        readAt: this.readAt.bind(this),\n        readChunks: this.readChunks.bind(this),\n        size: this.size.bind(this),\n      },\n    });\n  }\n\n  override async write(\n    _key: string,\n    _value: TObject,\n    _params?: unknown\n  ): Promise<unknown> {\n    throw new Error(\"Method not implemented.\");\n  }\n\n  protected async *readChunks(params: {\n    startOffset?: number;\n    chunkSize: number;\n  }): AsyncIterable<{ offset: number; bytes: Uint8Array }> {\n    this.assertInitialized();\n\n    const fileSize = await this.size();\n    let offset = params.startOffset ?? 0;\n\n    // NOTE: What if the file size changed during the iteration?\n    while (offset < fileSize) {\n      const length = Math.min(params.chunkSize, fileSize - offset);\n      if (length === 0) break;\n\n      const bytes = await this.readAt(offset, length);\n      if (bytes.length === 0) break;\n      yield { offset, bytes };\n      offset += bytes.length;\n    }\n  }\n\n  protected async size(): Promise<number> {\n    this.assertInitialized();\n\n    return await this.fileHandle.stat().then((stat) => stat.size);\n  }\n\n  protected async readAt(offset: number, length: number): Promise<Uint8Array> {\n    this.assertInitialized();\n\n    const buffer = new Uint8Array(length);\n    const result = await this.fileHandle.read(buffer, 0, length, offset);\n\n    return new Uint8Array(buffer.subarray(0, result.bytesRead));\n  }\n\n  protected assertInitialized(): asserts this is this & {\n    fileHandle: FileHandle;\n  } {\n    if (!this.fileHandle) {\n      throw new Error(\"File Storage is not initialized\");\n    }\n  }\n}\n\nexport type FileStorageFileHandle = {\n  size(): Promise<number>;\n  readAt(offset: number, length: number): Promise<Uint8Array>;\n  readChunks(params: {\n    startOffset?: number;\n    chunkSize: number;\n  }): AsyncIterable<{ offset: number; bytes: Uint8Array }>;\n};\n\nexport abstract class AbstractFileStorageCodec<TObject> {\n  abstract readAll(params: {\n    fileHandle: FileStorageFileHandle;\n  }): Promise<TObject[]>;\n  // TODO: Add other methods like write, append, read one etc.\n}\n","import { bufferToString } from \"@/utils\";\nimport {\n  AbstractFileStorageCodec,\n  FileStorage,\n  FileStorageFileHandle,\n} from \"./file\";\nimport z from \"zod\";\n\nexport class JSONFileStorage<TObject> extends FileStorage<TObject> {\n  declare codec: JSONFileStorageCodec<TObject>;\n\n  constructor(config: {\n    path: string;\n    chunkSize?: number;\n    schema: z.ZodType<TObject>;\n  }) {\n    super({\n      path: config.path,\n      codec: new JSONFileStorageCodec({\n        chunkSize: config.chunkSize,\n        schema: config.schema,\n      }),\n    });\n  }\n}\n\nexport class JSONFileStorageCodec<\n  TObject,\n> extends AbstractFileStorageCodec<TObject> {\n  private chunkSize: number;\n  private schema: z.ZodType<TObject>;\n\n  constructor(config: { chunkSize?: number; schema: z.ZodType<TObject> }) {\n    super();\n    this.chunkSize = config?.chunkSize ?? 64 * 1024;\n    this.schema = config.schema;\n  }\n\n  async readAll(params: {\n    fileHandle: FileStorageFileHandle;\n  }): Promise<TObject[]> {\n    const wholeFile: Uint8Array = new Uint8Array(\n      await params.fileHandle.size()\n    );\n    for await (const chunk of params.fileHandle.readChunks({\n      chunkSize: this.chunkSize,\n    })) {\n      wholeFile.set(chunk.bytes, chunk.offset);\n    }\n\n    return this.schema.array().parse(JSON.parse(bufferToString(wholeFile)));\n  }\n}\n"],"mappings":";;;;;AAAO,IAAe,kBAAf,MAAwC;AAS/C;;;ACTA,SAAqB,YAAY;AAG1B,IAAM,cAAN,cAAmC,gBAAyB;AAAA,EACvD;AAAA,EACA;AAAA,EACA;AAAA,EAEV,YAAY,QAGT;AACD,UAAM;AACN,SAAK,OAAO,OAAO;AACnB,SAAK,QAAQ,OAAO;AAAA,EACtB;AAAA,EAEA,MAAM,OAAsB;AAC1B,SAAK,aAAa,MAAM,KAAK,KAAK,MAAM,IAAI;AAAA,EAC9C;AAAA,EAEA,MAAe,KACb,MACA,SACyB;AACzB,UAAM,IAAI,MAAM,yBAAyB;AAAA,EAC3C;AAAA,EAEA,MAAe,QAAQ,SAAuC;AAC5D,SAAK,kBAAkB;AAEvB,WAAO,MAAM,KAAK,MAAM,QAAQ;AAAA,MAC9B,YAAY;AAAA,QACV,QAAQ,KAAK,OAAO,KAAK,IAAI;AAAA,QAC7B,YAAY,KAAK,WAAW,KAAK,IAAI;AAAA,QACrC,MAAM,KAAK,KAAK,KAAK,IAAI;AAAA,MAC3B;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEA,MAAe,MACb,MACA,QACA,SACkB;AAClB,UAAM,IAAI,MAAM,yBAAyB;AAAA,EAC3C;AAAA,EAEA,OAAiB,WAAW,QAG6B;AACvD,SAAK,kBAAkB;AAEvB,UAAM,WAAW,MAAM,KAAK,KAAK;AACjC,QAAI,SAAS,OAAO,eAAe;AAGnC,WAAO,SAAS,UAAU;AACxB,YAAM,SAAS,KAAK,IAAI,OAAO,WAAW,WAAW,MAAM;AAC3D,UAAI,WAAW,EAAG;AAElB,YAAM,QAAQ,MAAM,KAAK,OAAO,QAAQ,MAAM;AAC9C,UAAI,MAAM,WAAW,EAAG;AACxB,YAAM,EAAE,QAAQ,MAAM;AACtB,gBAAU,MAAM;AAAA,IAClB;AAAA,EACF;AAAA,EAEA,MAAgB,OAAwB;AACtC,SAAK,kBAAkB;AAEvB,WAAO,MAAM,KAAK,WAAW,KAAK,EAAE,KAAK,CAAC,SAAS,KAAK,IAAI;AAAA,EAC9D;AAAA,EAEA,MAAgB,OAAO,QAAgB,QAAqC;AAC1E,SAAK,kBAAkB;AAEvB,UAAM,SAAS,IAAI,WAAW,MAAM;AACpC,UAAM,SAAS,MAAM,KAAK,WAAW,KAAK,QAAQ,GAAG,QAAQ,MAAM;AAEnE,WAAO,IAAI,WAAW,OAAO,SAAS,GAAG,OAAO,SAAS,CAAC;AAAA,EAC5D;AAAA,EAEU,oBAER;AACA,QAAI,CAAC,KAAK,YAAY;AACpB,YAAM,IAAI,MAAM,iCAAiC;AAAA,IACnD;AAAA,EACF;AACF;AAWO,IAAe,2BAAf,MAAiD;AAAA;AAKxD;;;ACnGO,IAAM,kBAAN,cAAuC,YAAqB;AAAA,EAGjE,YAAY,QAIT;AACD,UAAM;AAAA,MACJ,MAAM,OAAO;AAAA,MACb,OAAO,IAAI,qBAAqB;AAAA,QAC9B,WAAW,OAAO;AAAA,QAClB,QAAQ,OAAO;AAAA,MACjB,CAAC;AAAA,IACH,CAAC;AAAA,EACH;AACF;AAEO,IAAM,uBAAN,cAEG,yBAAkC;AAAA,EAClC;AAAA,EACA;AAAA,EAER,YAAY,QAA4D;AACtE,UAAM;AACN,SAAK,YAAY,QAAQ,aAAa,KAAK;AAC3C,SAAK,SAAS,OAAO;AAAA,EACvB;AAAA,EAEA,MAAM,QAAQ,QAES;AACrB,UAAM,YAAwB,IAAI;AAAA,MAChC,MAAM,OAAO,WAAW,KAAK;AAAA,IAC/B;AACA,qBAAiB,SAAS,OAAO,WAAW,WAAW;AAAA,MACrD,WAAW,KAAK;AAAA,IAClB,CAAC,GAAG;AACF,gBAAU,IAAI,MAAM,OAAO,MAAM,MAAM;AAAA,IACzC;AAEA,WAAO,KAAK,OAAO,MAAM,EAAE,MAAM,KAAK,MAAM,eAAe,SAAS,CAAC,CAAC;AAAA,EACxE;AACF;","names":[]}

package/dist/{chunk-ZJWSK4VO.js → chunk-HMQYGCKI.js}

@@ -8,4 +8,4 @@ var ScoringMethod = {
 export {
   ScoringMethod
 };
-//# sourceMappingURL=chunk-ZJWSK4VO.js.map
+//# sourceMappingURL=chunk-HMQYGCKI.js.map

package/dist/chunk-HMQYGCKI.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/types/index.ts"],"sourcesContent":["export * from \"./runner\";\n\nimport { AbstractProvider } from \"@/providers\";\nimport { IdSchema } from \"@/schemas/id\";\nimport { AbstractScorer } from \"@/scorers/abstract\";\nimport { AbstractClassConstructor, ClassConstructor } from \"@/utilities\";\nimport z from \"zod\";\n\nexport type Id = z.infer<typeof IdSchema>;\n\nexport type IdGenerator<TInput = unknown> = (input: TInput) => MaybePromise<Id>;\n\nexport type MaybePromise<T> = T | Promise<T>;\n\nexport const ScoringMethod = {\n  ai: \"ai\",\n  human: \"human\",\n  algo: \"algo\",\n} as const;\nexport type ScoringMethod = (typeof ScoringMethod)[keyof typeof ScoringMethod];\n\nexport type ScorerCtor =\n  | ClassConstructor<AbstractScorer>\n  | AbstractClassConstructor<AbstractScorer>;\nexport type ProviderCtor =\n  | ClassConstructor<AbstractProvider>\n  | AbstractClassConstructor<AbstractProvider>;\n"],"mappings":";AAcO,IAAM,gBAAgB;AAAA,EAC3B,IAAI;AAAA,EACJ,OAAO;AAAA,EACP,MAAM;AACR;","names":[]}

package/dist/chunk-NUEOE3K5.js

@@ -0,0 +1,8 @@
+// src/schemas/id.ts
+import z from "zod";
+var IdSchema = z.string();
+
+export {
+  IdSchema
+};
+//# sourceMappingURL=chunk-NUEOE3K5.js.map

package/dist/chunk-NUEOE3K5.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/schemas/id.ts"],"sourcesContent":["import z from \"zod\";\n\nexport const IdSchema = z.string();\n"],"mappings":";AAAA,OAAO,OAAO;AAEP,IAAM,WAAW,EAAE,OAAO;","names":[]}

package/dist/chunk-OQE6TQXZ.js

@@ -0,0 +1,42 @@
+// src/schemas/schema-definer.ts
+import z from "zod";
+function buildSchemaDefiner(_schema, kindSuffix) {
+  return function(config) {
+    const kind = `${config.kind}.${kindSuffix}`;
+    const schema = config.baseSchema.extend({
+      ...config.fields ?? {},
+      namespace: config.namespace !== void 0 ? z.literal(config.namespace) : config.baseSchema.shape.namespace,
+      kind: config.kind !== void 0 ? z.literal(kind) : config.baseSchema.shape.kind,
+      schemaVersion: config.schemaVersion !== void 0 ? z.literal(config.schemaVersion) : config.baseSchema.shape.schemaVersion
+    });
+    return Object.assign(schema, {
+      new(input) {
+        return schema.parse({
+          ...input,
+          kind,
+          namespace: config.namespace,
+          schemaVersion: config.schemaVersion
+        });
+      },
+      async newWithId(input, generator) {
+        const obj = schema.parse({
+          ...input,
+          id: "",
+          kind,
+          namespace: config.namespace,
+          schemaVersion: config.schemaVersion
+        });
+        const id = await generator(obj);
+        return {
+          ...obj,
+          id
+        };
+      }
+    });
+  };
+}
+
+export {
+  buildSchemaDefiner
+};
+//# sourceMappingURL=chunk-OQE6TQXZ.js.map

package/dist/chunk-OQE6TQXZ.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/schemas/schema-definer.ts"],"sourcesContent":["import { IdGenerator } from \"@/types\";\nimport { WidenZodObject } from \"@/utilities\";\nimport z from \"zod\";\n\nexport function buildSchemaDefiner<\n  TBaseShape extends {\n    kind: z.ZodString | z.ZodLiteral<string>;\n    namespace: z.ZodString | z.ZodLiteral<string>;\n    schemaVersion: z.ZodNumber | z.ZodLiteral<number>;\n  },\n  TKindSuffix extends string = \"\",\n>(_schema: z.ZodObject<TBaseShape>, kindSuffix: TKindSuffix) {\n  return function <\n    TBaseSchema extends WidenZodObject<z.ZodObject<TBaseShape>>,\n    TNamespace extends string | undefined = undefined,\n    TKind extends string | undefined = undefined,\n    TSchemaVersion extends number | undefined = undefined,\n    TFields extends z.ZodRawShape = {},\n  >(config: {\n    baseSchema: TBaseSchema;\n    namespace?: TNamespace;\n    kind?: TKind;\n    schemaVersion?: TSchemaVersion;\n    fields?: TFields;\n  }) {\n    const kind = `${config.kind}.${kindSuffix}` as const;\n    const schema = config.baseSchema.extend({\n      ...((config.fields ?? {}) as TFields),\n      namespace: (config.namespace !== undefined\n        ? z.literal(config.namespace)\n        : config.baseSchema.shape.namespace) as TNamespace extends undefined\n        ? TBaseSchema[\"shape\"][\"namespace\"]\n        : z.ZodLiteral<TNamespace>,\n      kind: (config.kind !== undefined\n        ? z.literal(kind)\n        : config.baseSchema.shape.kind) as TKind extends undefined\n        ? TBaseSchema[\"shape\"][\"kind\"]\n        : z.ZodLiteral<`${TKind}.${TKindSuffix}`>,\n      schemaVersion: (config.schemaVersion !== undefined\n        ? z.literal(config.schemaVersion)\n        : config.baseSchema.shape\n            .schemaVersion) as TSchemaVersion extends undefined\n        ? TBaseSchema[\"shape\"][\"schemaVersion\"]\n        : z.ZodLiteral<TSchemaVersion>,\n    });\n\n    type SchemaType =\n      TBaseSchema extends z.ZodObject<infer U>\n        ? z.ZodObject<\n            Omit<U, \"kind\" | \"schemaVersion\" | \"namespace\"> &\n              TFields & {\n                namespace: TNamespace extends undefined\n                  ? U[\"namespace\"]\n                  : z.ZodLiteral<TNamespace>;\n                kind: TKind extends undefined\n                  ? U[\"kind\"]\n                  : z.ZodLiteral<`${TKind}.${TKindSuffix}`>;\n                schemaVersion: TSchemaVersion extends undefined\n                  ? U[\"schemaVersion\"]\n                  : z.ZodLiteral<TSchemaVersion>;\n              }\n          >\n        : never;\n\n    return Object.assign(schema, {\n      new(\n        input: Omit<\n          z.infer<typeof schema>,\n          \"kind\" | \"schemaVersion\" | \"namespace\"\n        >\n      ) {\n        return schema.parse({\n          ...input,\n          kind,\n          namespace: config.namespace,\n          schemaVersion: config.schemaVersion,\n        });\n      },\n      async newWithId(\n        input: Omit<\n          z.infer<typeof schema>,\n          \"kind\" | \"schemaVersion\" | \"id\" | \"namespace\"\n        >,\n        generator: IdGenerator\n      ) {\n        const obj = schema.parse({\n          ...input,\n          id: \"\",\n          kind,\n          namespace: config.namespace,\n          schemaVersion: config.schemaVersion,\n        });\n        const id = await generator(obj);\n\n        return {\n          ...obj,\n          id,\n        };\n      },\n    }) as unknown as SchemaType & {\n      new: (\n        input: Omit<z.infer<SchemaType>, \"kind\" | \"schemaVersion\" | \"namespace\">\n      ) => z.infer<SchemaType>;\n\n      newWithId(\n        input: Omit<\n          z.infer<SchemaType>,\n          \"id\" | \"kind\" | \"schemaVersion\" | \"namespace\"\n        >,\n        generator: IdGenerator\n      ): Promise<z.infer<SchemaType>>;\n    };\n  };\n}\n"],"mappings":";AAEA,OAAO,OAAO;AAEP,SAAS,mBAOd,SAAkC,YAAyB;AAC3D,SAAO,SAML,QAMC;AACD,UAAM,OAAO,GAAG,OAAO,IAAI,IAAI,UAAU;AACzC,UAAM,SAAS,OAAO,WAAW,OAAO;AAAA,MACtC,GAAK,OAAO,UAAU,CAAC;AAAA,MACvB,WAAY,OAAO,cAAc,SAC7B,EAAE,QAAQ,OAAO,SAAS,IAC1B,OAAO,WAAW,MAAM;AAAA,MAG5B,MAAO,OAAO,SAAS,SACnB,EAAE,QAAQ,IAAI,IACd,OAAO,WAAW,MAAM;AAAA,MAG5B,eAAgB,OAAO,kBAAkB,SACrC,EAAE,QAAQ,OAAO,aAAa,IAC9B,OAAO,WAAW,MACf;AAAA,IAGT,CAAC;AAoBD,WAAO,OAAO,OAAO,QAAQ;AAAA,MAC3B,IACE,OAIA;AACA,eAAO,OAAO,MAAM;AAAA,UAClB,GAAG;AAAA,UACH;AAAA,UACA,WAAW,OAAO;AAAA,UAClB,eAAe,OAAO;AAAA,QACxB,CAAC;AAAA,MACH;AAAA,MACA,MAAM,UACJ,OAIA,WACA;AACA,cAAM,MAAM,OAAO,MAAM;AAAA,UACvB,GAAG;AAAA,UACH,IAAI;AAAA,UACJ;AAAA,UACA,WAAW,OAAO;AAAA,UAClB,eAAe,OAAO;AAAA,QACxB,CAAC;AACD,cAAM,KAAK,MAAM,UAAU,GAAG;AAE9B,eAAO;AAAA,UACL,GAAG;AAAA,UACH;AAAA,QACF;AAAA,MACF;AAAA,IACF,CAAC;AAAA,EAaH;AACF;","names":[]}

package/dist/chunk-QY5MPNNB.js

@@ -0,0 +1,28 @@
+import {
+  idGeneratorUUIDv7
+} from "./chunk-4UBK6452.js";
+
+// src/helpers/define-runner.ts
+import z from "zod";
+function defineRunner(config, fn) {
+  return async (params) => {
+    if (config.runConfigSchema && config.parseRunConfig !== false) {
+      z.object(config.runConfigSchema).parse(params.runConfig);
+    }
+    if (params.idGenerators && !params.idGenerators.response) {
+      params.idGenerators.response = config.defaults?.responseIdGenerator ?? idGeneratorUUIDv7;
+    }
+    if (params.idGenerators && !params.idGenerators.score) {
+      params.idGenerators.score = config.defaults?.scoreIdGenerator ?? idGeneratorUUIDv7;
+    }
+    if (params.scorer === void 0) {
+      params.scorer = config.defaults?.scorer ?? void 0;
+    }
+    return await fn(params);
+  };
+}
+
+export {
+  defineRunner
+};
+//# sourceMappingURL=chunk-QY5MPNNB.js.map

package/dist/chunk-QY5MPNNB.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/helpers/define-runner.ts"],"sourcesContent":["import {\n  IdGenerator,\n  InferRunConfig,\n  ProviderCtor,\n  Runner,\n  ScorerCtor,\n} from \"@/types\";\nimport { idGeneratorUUIDv7 } from \"@/utils\";\nimport z from \"zod\";\n\nexport function defineRunner<\n  const TProviders extends ProviderCtor[],\n  const TScorers extends ScorerCtor[],\n  const TSchemaSets extends SchemaSetDefinition[],\n  const TRunConfigSchema extends z.ZodRawShape = {},\n>(\n  config: {\n    schemaSets: TSchemaSets;\n    providers: TProviders;\n    scorers: TScorers;\n    runConfigSchema?: TRunConfigSchema;\n\n    /**\n     * @default true\n     */\n    parseRunConfig?: boolean;\n    defaults?: {\n      scorer?: InstanceType<TScorers[number]>;\n      responseIdGenerator?: IdGenerator;\n      scoreIdGenerator?: IdGenerator;\n    };\n  },\n  fn: Runner<\n    TSchemaSets[number][\"testCase\"],\n    TSchemaSets[number][\"response\"],\n    TSchemaSets[number][\"score\"],\n    InstanceType<TProviders[number]>,\n    InstanceType<TScorers[number]>,\n    InferRunConfig<TRunConfigSchema>\n  >\n) {\n  return async (params: Parameters<typeof fn>[0]) => {\n    if (config.runConfigSchema && config.parseRunConfig !== false) {\n      z.object(config.runConfigSchema).parse(params.runConfig);\n    }\n\n    if (params.idGenerators && !params.idGenerators.response) {\n      params.idGenerators.response =\n        config.defaults?.responseIdGenerator ?? idGeneratorUUIDv7;\n    }\n\n    if (params.idGenerators && !params.idGenerators.score) {\n      params.idGenerators.score =\n        config.defaults?.scoreIdGenerator ?? idGeneratorUUIDv7;\n    }\n\n    if (params.scorer === undefined) {\n      params.scorer = config.defaults?.scorer ?? undefined;\n    }\n\n    return await fn(params);\n  };\n}\n\ntype SchemaSetDefinition<\n  TTestCase extends z.ZodObject = z.ZodObject,\n  TResponse extends z.ZodObject = z.ZodObject,\n  TScore extends z.ZodObject = z.ZodObject,\n> = {\n  testCase: TTestCase;\n  response: TResponse;\n  score: TScore;\n};\n"],"mappings":";;;;;AAQA,OAAO,OAAO;AAEP,SAAS,aAMd,QAgBA,IAQA;AACA,SAAO,OAAO,WAAqC;AACjD,QAAI,OAAO,mBAAmB,OAAO,mBAAmB,OAAO;AAC7D,QAAE,OAAO,OAAO,eAAe,EAAE,MAAM,OAAO,SAAS;AAAA,IACzD;AAEA,QAAI,OAAO,gBAAgB,CAAC,OAAO,aAAa,UAAU;AACxD,aAAO,aAAa,WAClB,OAAO,UAAU,uBAAuB;AAAA,IAC5C;AAEA,QAAI,OAAO,gBAAgB,CAAC,OAAO,aAAa,OAAO;AACrD,aAAO,aAAa,QAClB,OAAO,UAAU,oBAAoB;AAAA,IACzC;AAEA,QAAI,OAAO,WAAW,QAAW;AAC/B,aAAO,SAAS,OAAO,UAAU,UAAU;AAAA,IAC7C;AAEA,WAAO,MAAM,GAAG,MAAM;AAAA,EACxB;AACF;","names":[]}