peerbench 0.0.1 → 0.0.2-alpha.0

package/dist/benchmarks/index.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/benchmarks/peerbench/index.ts","../../src/benchmarks/peerbench/test-cases/mcq.v1.ts","../../src/benchmarks/peerbench/score.ts","../../src/benchmarks/peerbench/test-cases/open-ended.v1.ts","../../src/benchmarks/peerbench/spec.ts","../../src/benchmarks/peerbench/loader.ts","../../src/benchmarks/peerbench/runner.ts","../../src/benchmarks/mmlu-pro/index.ts","../../src/benchmarks/mmlu-pro/score.ts","../../src/benchmarks/mmlu-pro/test-cases/main.v1.ts","../../src/benchmarks/mmlu-pro/loader.ts","../../src/benchmarks/mmlu-pro/runner.ts","../../src/benchmarks/mmlu-pro/spec.ts","../../src/benchmarks/fnol/index.ts","../../src/benchmarks/fnol/test-cases/fnol.v1.ts","../../src/benchmarks/fnol/score.ts","../../src/benchmarks/fnol/types.ts","../../src/benchmarks/fnol/runner.ts","../../src/benchmarks/fnol/scorer.ts"],"sourcesContent":["export * from \"./loader\";\nexport * from \"./runner\";\nexport * from \"./test-cases/mcq.v1\";\nexport * from \"./test-cases/open-ended.v1\";\n","import { z } from \"zod\";\nimport {\n  BaseTestCaseSchemaV1,\n  defineResponseSchema,\n  defineScoreSchema,\n  defineTestCaseSchema,\n} from \"@/schemas\";\nimport { BaseLLMChatResponseSchemaV1 } from \"@/schemas/llm/response\";\nimport { PeerbenchBaseScoreSchemaV1 } from \"../score\";\n\n// Test case schema\nexport const PeerbenchMultipleChoiceTestCaseSchemaV1 = defineTestCaseSchema({\n  baseSchema: BaseTestCaseSchemaV1,\n\n  kind: \"pb.ts.mcq\",\n  schemaVersion: 1,\n  fields: {\n    question: z.string(),\n    options: z.record(z.string(), z.string()),\n    answer: z.string(),\n    answerKey: z.string(),\n  },\n});\nexport type PeerbenchMultipleChoiceTestCaseV1 = z.infer<\n  typeof PeerbenchMultipleChoiceTestCaseSchemaV1\n>;\n\n// Response schema\nexport const PeerbenchMultipleChoiceResponseSchemaV1 = defineResponseSchema({\n  baseSchema: BaseLLMChatResponseSchemaV1,\n  kind: \"pb.rs.mcq\",\n  schemaVersion: 1,\n  fields: {},\n});\nexport type PeerbenchMultipleChoiceResponseV1 = z.infer<\n  typeof PeerbenchMultipleChoiceResponseSchemaV1\n>;\n\n// Score schema\nexport const PeerbenchMultipleChoiceScoreSchemaV1 = defineScoreSchema({\n  baseSchema: PeerbenchBaseScoreSchemaV1,\n  kind: \"pb.sc.mcq\",\n  schemaVersion: 1,\n  fields: {\n    extractedAnswers: z.array(z.string()),\n  },\n});\nexport type PeerbenchMultipleChoiceScoreV1 = z.infer<\n  typeof PeerbenchMultipleChoiceScoreSchemaV1\n>;\n","import { BaseScoreSchemaV1, defineScoreSchema } from \"@/schemas\";\nimport { ExtensionLLMAsAJudgeScorerFieldsV1 } from \"@/schemas/extensions/score/llm-as-a-judge-scorer\";\n\nexport const PeerbenchBaseScoreSchemaV1 = defineScoreSchema({\n  baseSchema: BaseScoreSchemaV1,\n  fields: {\n    ...ExtensionLLMAsAJudgeScorerFieldsV1,\n  },\n});\n","import { defineResponseSchema, defineScoreSchema } from \"@/schemas\";\nimport { BaseLLMChatResponseSchemaV1 } from \"@/schemas/llm/response\";\nimport {\n  BaseTestCaseSchemaV1,\n  defineTestCaseSchema,\n} from \"@/schemas/test-case\";\nimport { z } from \"zod\";\nimport { PeerbenchBaseScoreSchemaV1 } from \"../score\";\n\n// Test case schema\nexport const PeerbenchOpenEndedTestCaseSchemaV1 = defineTestCaseSchema({\n  kind: \"pb.ts.open-ended\",\n  schemaVersion: 1,\n  baseSchema: BaseTestCaseSchemaV1,\n  fields: {\n    question: z.string(),\n    answer: z.string().optional(),\n  },\n});\nexport type PeerbenchOpenEndedTestCaseV1 = z.infer<\n  typeof PeerbenchOpenEndedTestCaseSchemaV1\n>;\n\n// Response schema\nexport const PeerbenchOpenEndedResponseSchemaV1 = defineResponseSchema({\n  baseSchema: BaseLLMChatResponseSchemaV1,\n  kind: \"pb.rs.open-ended\",\n  schemaVersion: 1,\n});\nexport type PeerbenchOpenEndedResponseV1 = z.infer<\n  typeof PeerbenchOpenEndedResponseSchemaV1\n>;\n\n// Score schema\nexport const PeerbenchOpenEndedScoreSchemaV1 = defineScoreSchema({\n  baseSchema: PeerbenchBaseScoreSchemaV1,\n  kind: \"pb.sc.open-ended\",\n  schemaVersion: 1,\n  fields: {},\n});\nexport type PeerbenchOpenEndedScoreV1 = z.infer<\n  typeof PeerbenchOpenEndedScoreSchemaV1\n>;\n","import {\n  BaseBenchmarkSpecSchemaV1,\n  defineBenchmarkSpecSchema,\n} from \"@/schemas/benchmark-spec\";\nimport z from \"zod\";\n\nexport const PeerbenchBenchmarkSpecSchemaV1 = defineBenchmarkSpecSchema({\n  baseSchema: BaseBenchmarkSpecSchemaV1,\n  kind: \"pb.benchmark.spec\",\n  schemaVersion: 1,\n  fields: {\n    /**\n     * Big text contents that can be referred as <text>{key}</text> in a prompt or system prompt.\n     */\n    blobTexts: z.record(z.string(), z.string()).optional(),\n  },\n});\nexport type PeerbenchBenchmarkSpecV1 = z.infer<\n  typeof PeerbenchBenchmarkSpecSchemaV1\n>;\n","import { GenericJSONArrayDataLoader } from \"@/loaders/generic-array\";\nimport {\n  PeerbenchMultipleChoiceResponseSchemaV1,\n  PeerbenchMultipleChoiceScoreSchemaV1,\n  PeerbenchMultipleChoiceTestCaseSchemaV1,\n  PeerbenchMultipleChoiceResponseV1,\n  PeerbenchMultipleChoiceScoreV1,\n  PeerbenchMultipleChoiceTestCaseV1,\n} from \"./test-cases/mcq.v1\";\nimport {\n  PeerbenchOpenEndedResponseSchemaV1,\n  PeerbenchOpenEndedScoreSchemaV1,\n  PeerbenchOpenEndedTestCaseSchemaV1,\n  PeerbenchOpenEndedResponseV1,\n  PeerbenchOpenEndedScoreV1,\n  PeerbenchOpenEndedTestCaseV1,\n} from \"./test-cases/open-ended.v1\";\nimport {\n  PeerbenchBenchmarkSpecSchemaV1,\n  PeerbenchBenchmarkSpecV1,\n} from \"./spec\";\nimport z from \"zod\";\nimport { bufferToString } from \"@/utils\";\n\nexport class PeerbenchJSONDataLoader extends GenericJSONArrayDataLoader<\n  PeerbenchMultipleChoiceTestCaseV1 | PeerbenchOpenEndedTestCaseV1,\n  PeerbenchMultipleChoiceResponseV1 | PeerbenchOpenEndedResponseV1,\n  PeerbenchMultipleChoiceScoreV1 | PeerbenchOpenEndedScoreV1\n> {\n  override readonly kind = \"pb.load.json.data\";\n\n  async loadBenchmarkSpec(params: {\n    content: Uint8Array;\n  }): Promise<PeerbenchBenchmarkSpecV1> {\n    const content = bufferToString(params.content);\n    const parsed = PeerbenchBenchmarkSpecSchemaV1.parse(content);\n\n    return parsed;\n  }\n\n  protected override testCaseBuilder(data: any) {\n    const testCaseValidation = z\n      .union([\n        PeerbenchMultipleChoiceTestCaseSchemaV1,\n        PeerbenchOpenEndedTestCaseSchemaV1,\n      ])\n      .safeParse(data);\n    return testCaseValidation.success ? testCaseValidation.data : undefined;\n  }\n\n  protected override async responseBuilder(data: any) {\n    const responseValidation = z\n      .union([\n        PeerbenchMultipleChoiceResponseSchemaV1,\n        PeerbenchOpenEndedResponseSchemaV1,\n      ])\n      .safeParse(data);\n    return responseValidation.success ? responseValidation.data : undefined;\n  }\n\n  protected override async scoreBuilder(data: any) {\n    const scoreValidation = z\n      .union([\n        PeerbenchMultipleChoiceScoreSchemaV1,\n        PeerbenchOpenEndedScoreSchemaV1,\n      ])\n      .safeParse(data);\n    return scoreValidation.success ? scoreValidation.data : undefined;\n  }\n}\n","import { ChatCompletionMessageParam } from \"openai/resources/chat/completions\";\nimport { idGeneratorUUIDv7 } from \"@/utils/id-generator\";\nimport { IdGenerator, RunnerResult } from \"@/types\";\nimport { AbstractLLMProvider } from \"@/providers/abstract/llm\";\nimport { MCQScorer } from \"@/scorers/mcq\";\nimport { LLMJudgeScorer } from \"@/scorers/llm-judge\";\nimport {\n  PeerbenchMultipleChoiceResponseSchemaV1,\n  PeerbenchMultipleChoiceResponseV1,\n  PeerbenchMultipleChoiceScoreSchemaV1,\n  PeerbenchMultipleChoiceScoreV1,\n  PeerbenchMultipleChoiceTestCaseV1,\n} from \"./test-cases/mcq.v1\";\nimport { ScoringMethod } from \"@/types\";\nimport {\n  PeerbenchOpenEndedResponseSchemaV1,\n  PeerbenchOpenEndedResponseV1,\n  PeerbenchOpenEndedScoreSchemaV1,\n  PeerbenchOpenEndedScoreV1,\n  PeerbenchOpenEndedTestCaseV1,\n} from \"./test-cases/open-ended.v1\";\nimport { PeerbenchBenchmarkSpecV1 } from \"./spec\";\nimport { SimpleSystemPromptV1 } from \"@/schemas/llm\";\n\ntype ResponseTypes =\n  | PeerbenchMultipleChoiceResponseV1\n  | PeerbenchOpenEndedResponseV1;\ntype ScoreTypes = PeerbenchMultipleChoiceScoreV1 | PeerbenchOpenEndedScoreV1;\ntype TestCaseTypes =\n  | PeerbenchMultipleChoiceTestCaseV1\n  | PeerbenchOpenEndedTestCaseV1;\n\nexport async function runTestCase(params: {\n  testCase: TestCaseTypes;\n  provider: AbstractLLMProvider;\n  scorer?: MCQScorer | LLMJudgeScorer;\n  spec?: PeerbenchBenchmarkSpecV1;\n  runConfig: {\n    model: string;\n    llmJudgeModel?: string;\n  };\n  systemPrompt?: SimpleSystemPromptV1;\n  idGenerators?: {\n    response?: IdGenerator;\n    score?: IdGenerator;\n  };\n}): Promise<RunnerResult<ResponseTypes, ScoreTypes>> {\n  const { testCase } = params;\n  const responseIdGenerator =\n    params.idGenerators?.response ?? idGeneratorUUIDv7;\n  const scoreIdGenerator = params.idGenerators?.score ?? idGeneratorUUIDv7;\n  const messages: ChatCompletionMessageParam[] = [];\n\n  if (params.systemPrompt) {\n    messages.push({\n      role: \"system\",\n      content: params.systemPrompt.content,\n    });\n  }\n\n  if (testCase.kind === \"pb.ts.mcq\") {\n    const formattedPrompt = formatMCQPrompt(testCase);\n\n    messages.push({\n      role: \"user\",\n      content: formattedPrompt,\n    });\n\n    const providerResponse = await params.provider.forward({\n      model: params.runConfig.model,\n      messages,\n    });\n\n    const response = await PeerbenchMultipleChoiceResponseSchemaV1.newWithId(\n      {\n        data: providerResponse.data,\n        startedAt: providerResponse.startedAt,\n        completedAt: providerResponse.completedAt,\n        testCaseId: testCase.id,\n        modelSlug: params.runConfig.model,\n        provider: params.provider.kind,\n\n        inputTokensUsed: providerResponse.inputTokensUsed,\n        outputTokensUsed: providerResponse.outputTokensUsed,\n        inputCost: providerResponse.inputCost,\n        outputCost: providerResponse.outputCost,\n      },\n      responseIdGenerator\n    );\n\n    if (params.scorer?.kind === \"mcq\") {\n      const scorerResult = await params.scorer.score({\n        response: response.data,\n        choices: testCase.options ?? {},\n        correctAnswers: [testCase.answerKey],\n      });\n\n      if (scorerResult !== null) {\n        const score = await PeerbenchMultipleChoiceScoreSchemaV1.newWithId(\n          {\n            scoringMethod: ScoringMethod.algo,\n            value: scorerResult.value,\n            responseId: response.id,\n            extractedAnswers: scorerResult.extractedAnswers,\n            metadata: response.metadata,\n          },\n          scoreIdGenerator\n        );\n\n        return { response, score };\n      }\n    }\n\n    return { response };\n  } else if (testCase.kind === \"pb.ts.open-ended\") {\n    const messages: ChatCompletionMessageParam[] = [];\n\n    if (params.systemPrompt) {\n      messages.push({\n        role: \"system\",\n        content: params.systemPrompt.content,\n      });\n    }\n\n    messages.push({\n      role: \"user\",\n      content: testCase.question,\n    });\n\n    const providerResponse = await params.provider.forward({\n      model: params.runConfig.model,\n      messages,\n    });\n\n    const response = await PeerbenchOpenEndedResponseSchemaV1.newWithId(\n      {\n        data: providerResponse.data,\n        startedAt: providerResponse.startedAt,\n        completedAt: providerResponse.completedAt,\n        testCaseId: testCase.id,\n        modelSlug: params.runConfig.model,\n        provider: params.provider.kind,\n\n        inputTokensUsed: providerResponse.inputTokensUsed,\n        outputTokensUsed: providerResponse.outputTokensUsed,\n        inputCost: providerResponse.inputCost,\n        outputCost: providerResponse.outputCost,\n      },\n      responseIdGenerator\n    );\n\n    if (params.scorer?.kind === \"llmJudge\" && params.runConfig.llmJudgeModel) {\n      const scorerResult = await params.scorer.score({\n        task: testCase.question,\n        candidateAnswer: response.data,\n        referenceAnswer: testCase.answer,\n        model: params.runConfig.llmJudgeModel,\n      });\n\n      if (scorerResult !== null) {\n        const score = await PeerbenchOpenEndedScoreSchemaV1.newWithId(\n          {\n            scoringMethod: ScoringMethod.ai,\n            value: scorerResult.value,\n            responseId: response.id,\n            explanation: scorerResult.explanation,\n            metadata: scorerResult.metadata,\n\n            scorerAIProvider: scorerResult.provider,\n            scorerAIModelSlug: params.runConfig.llmJudgeModel,\n            scorerAIInputTokensUsed: scorerResult.inputTokensUsed,\n            scorerAIOutputTokensUsed: scorerResult.outputTokensUsed,\n            scorerAIInputCost: scorerResult.inputCost,\n            scorerAIOutputCost: scorerResult.outputCost,\n          },\n          scoreIdGenerator\n        );\n\n        return { response, score };\n      }\n    }\n\n    return { response };\n  }\n\n  throw new Error(\"Unsupported test case kind\");\n}\n\nfunction formatMCQPrompt(testCase: PeerbenchMultipleChoiceTestCaseV1) {\n  return `Question: ${testCase.question}\\nOptions:\\n${Object.entries(\n    testCase.options ?? {}\n  )\n    .map(([key, value]) => `${key}: ${value}`)\n    .join(\"\\n\")}`;\n}\n","export * from \"./loader\";\nexport * from \"./runner\";\nexport * from \"./score\";\nexport * from \"./spec\";\nexport * from \"./test-cases/main.v1\";\n","import { BaseScoreSchemaV1, defineScoreSchema } from \"@/schemas\";\nimport { ExtensionLLMAsAJudgeScorerFieldsV1 } from \"@/schemas/extensions/score/llm-as-a-judge-scorer\";\n\nexport const BaseMMLUProScoreSchemaV1 = defineScoreSchema({\n  baseSchema: BaseScoreSchemaV1,\n  fields: {\n    ...ExtensionLLMAsAJudgeScorerFieldsV1,\n  },\n});\n","import {\n  BaseTestCaseSchemaV1,\n  defineTestCaseSchema,\n} from \"@/schemas/test-case\";\nimport { BaseMMLUProScoreSchemaV1 } from \"../score\";\nimport { BaseLLMChatResponseSchemaV1 } from \"@/schemas/llm/response\";\nimport { defineResponseSchema, defineScoreSchema } from \"@/schemas\";\nimport { z } from \"zod\";\n\n// Test case schema\nexport const MMLUProMainTestCaseSchemaV1 = defineTestCaseSchema({\n  baseSchema: BaseTestCaseSchemaV1,\n  kind: \"mmlu-pro.ts.main\",\n  schemaVersion: 1,\n  fields: {\n    question: z.string(),\n    options: z.record(z.string(), z.string()),\n    answer: z.string(),\n    answerKey: z.string(),\n  },\n});\nexport type MMLUProMainTestCaseV1 = z.infer<typeof MMLUProMainTestCaseSchemaV1>;\n\n// Response schema\nexport const MMLUProMainResponseSchemaV1 = defineResponseSchema({\n  baseSchema: BaseLLMChatResponseSchemaV1,\n  kind: \"mmlu-pro.rs.main\",\n  schemaVersion: 1,\n});\nexport type MMLUProMainResponseV1 = z.infer<typeof MMLUProMainResponseSchemaV1>;\n\n// Score schema\nexport const MMLUProMainScoreSchemaV1 = defineScoreSchema({\n  baseSchema: BaseMMLUProScoreSchemaV1,\n  kind: \"mmlu-pro.sc.main\",\n  schemaVersion: 1,\n  fields: {\n    extractedAnswers: z.array(z.string()),\n  },\n});\nexport type MMLUProMainScoreV1 = z.infer<typeof MMLUProMainScoreSchemaV1>;\n","import { bufferToString } from \"@/utils\";\nimport {\n  MMLUProMainResponseV1,\n  MMLUProMainScoreV1,\n  MMLUProMainTestCaseSchemaV1,\n  MMLUProMainTestCaseV1,\n} from \"./test-cases/main.v1\";\nimport z from \"zod\";\nimport { AbstractDataLoader } from \"@/loaders/abstract/data\";\nimport { LoaderResult } from \"@/loaders/abstract/loader\";\nimport { parquetReadObjects } from \"hyparquet\";\nimport { MMLUProBenchmarkSpecV1 } from \"./spec\";\n\nconst jsonSchema = z\n  .object({\n    question_id: z.coerce.number(),\n    question: z.string(),\n    options: z.array(z.string()),\n    answer: z.string(),\n    answer_index: z.coerce.number(),\n    cot_content: z.string(),\n    category: z.string(),\n    src: z.string(),\n  })\n  .array();\n\nfunction mapData(\n  data: z.infer<typeof jsonSchema>\n): LoaderResult<\n  MMLUProMainTestCaseV1,\n  MMLUProMainResponseV1,\n  MMLUProMainScoreV1\n> {\n  return {\n    responses: [],\n    scores: [],\n    testCases: data.map((item) =>\n      MMLUProMainTestCaseSchemaV1.new({\n        id: `${item.src}-${item.category}-${item.question_id}`,\n        question: item.question,\n        answerKey: item.answer,\n        options: item.options.reduce(\n          (acc, option, index) => {\n            acc[String.fromCharCode(65 + index)] = option;\n            return acc;\n          },\n          {} as Record<string, string>\n        ),\n        answer: item.options[item.answer_index]!,\n        metadata: {\n          category: item.category,\n          src: item.src,\n          answer_index: item.answer_index,\n        },\n      })\n    ),\n  };\n}\n\nexport class MMLUProJSONDataLoader extends AbstractDataLoader {\n  override readonly kind = \"mmlu-pro.load.json.data\";\n\n  override loadData(params: {\n    content: Uint8Array;\n  }): LoaderResult<\n    MMLUProMainTestCaseV1,\n    MMLUProMainResponseV1,\n    MMLUProMainScoreV1\n  > {\n    const content =\n      typeof params.content === \"string\"\n        ? params.content\n        : bufferToString(params.content);\n\n    const parsed = jsonSchema.parse(JSON.parse(content));\n    return mapData(parsed);\n  }\n\n  // eslint-disable-next-line @typescript-eslint/no-unused-vars\n  override loadBenchmarkSpec(params: {\n    content: Uint8Array;\n  }): Promise<MMLUProBenchmarkSpecV1> {\n    throw new Error(\"Not implemented\");\n  }\n}\n\nexport class MMLUProParquetDataLoader extends AbstractDataLoader {\n  override readonly kind = \"mmlu-pro.load.parquet.data\";\n\n  override async loadData(params: {\n    content: Uint8Array;\n  }): Promise<\n    LoaderResult<\n      MMLUProMainTestCaseV1,\n      MMLUProMainResponseV1,\n      MMLUProMainScoreV1\n    >\n  > {\n    const data = await parquetReadObjects({\n      file: params.content.buffer as ArrayBuffer,\n    });\n    if (!data) {\n      throw new Error(\"Invalid Parquet file\");\n    }\n\n    return mapData(jsonSchema.parse(data));\n  }\n\n  // eslint-disable-next-line @typescript-eslint/no-unused-vars\n  override loadBenchmarkSpec(params: {\n    content: Uint8Array;\n  }): Promise<MMLUProBenchmarkSpecV1> {\n    throw new Error(\"Not implemented\");\n  }\n}\n","import { ChatCompletionMessageParam } from \"openai/resources/chat/completions\";\nimport { idGeneratorUUIDv7 } from \"@/utils/id-generator\";\nimport { IdGenerator, RunnerResult } from \"@/types\";\nimport { AbstractLLMProvider } from \"@/providers/abstract/llm\";\nimport { MCQScorer } from \"@/scorers/mcq\";\nimport { LLMJudgeScorer } from \"@/scorers/llm-judge\";\nimport { ScoringMethod } from \"@/types\";\nimport { SimpleSystemPromptV1 } from \"@/schemas/llm\";\nimport {\n  MMLUProMainResponseV1,\n  MMLUProMainTestCaseV1,\n  MMLUProMainScoreV1,\n  MMLUProMainResponseSchemaV1,\n  MMLUProMainScoreSchemaV1,\n} from \"./test-cases/main.v1\";\nimport { MMLUProBenchmarkSpecV1 } from \"./spec\";\n\nexport async function runTestCase(params: {\n  testCase: MMLUProMainTestCaseV1;\n  provider: AbstractLLMProvider;\n  scorer?: MCQScorer | LLMJudgeScorer;\n  spec?: MMLUProBenchmarkSpecV1;\n  runConfig: {\n    model: string;\n    llmJudgeModel?: string;\n  };\n  systemPrompt?: SimpleSystemPromptV1;\n  idGenerators?: {\n    response?: IdGenerator;\n    score?: IdGenerator;\n  };\n}): Promise<RunnerResult<MMLUProMainResponseV1, MMLUProMainScoreV1>> {\n  const { testCase } = params;\n  const responseIdGenerator =\n    params.idGenerators?.response ?? idGeneratorUUIDv7;\n  const scoreIdGenerator = params.idGenerators?.score ?? idGeneratorUUIDv7;\n  const messages: ChatCompletionMessageParam[] = [];\n\n  if (params.systemPrompt) {\n    messages.push({\n      role: \"system\",\n      content: params.systemPrompt.content,\n    });\n  }\n\n  if (testCase.kind === \"mmlu-pro.ts.main\") {\n    const formattedPrompt = formatMCQPrompt(testCase);\n\n    messages.push({\n      role: \"user\",\n      content: formattedPrompt,\n    });\n\n    const providerResponse = await params.provider.forward({\n      model: params.runConfig.model,\n      messages,\n    });\n\n    const response = await MMLUProMainResponseSchemaV1.newWithId(\n      {\n        data: providerResponse.data,\n        startedAt: providerResponse.startedAt,\n        completedAt: providerResponse.completedAt,\n        testCaseId: testCase.id,\n        modelSlug: params.runConfig.model,\n        provider: params.provider.kind,\n\n        inputTokensUsed: providerResponse.inputTokensUsed,\n        outputTokensUsed: providerResponse.outputTokensUsed,\n        inputCost: providerResponse.inputCost,\n        outputCost: providerResponse.outputCost,\n      },\n      responseIdGenerator\n    );\n\n    if (params.scorer?.kind === \"mcq\") {\n      const scorerResult = await params.scorer.score({\n        response: response.data,\n        choices: testCase.options ?? {},\n        correctAnswers: [testCase.answerKey],\n      });\n\n      if (scorerResult !== null) {\n        const score = await MMLUProMainScoreSchemaV1.newWithId(\n          {\n            scoringMethod: ScoringMethod.algo,\n            value: scorerResult.value,\n            responseId: response.id,\n            extractedAnswers: scorerResult.extractedAnswers,\n            metadata: response.metadata,\n          },\n          scoreIdGenerator\n        );\n\n        return { response, score };\n      }\n    }\n\n    return { response };\n  } else if (testCase.kind === \"pb.ts.open-ended\") {\n    const messages: ChatCompletionMessageParam[] = [];\n\n    if (params.systemPrompt) {\n      messages.push({\n        role: \"system\",\n        content: params.systemPrompt.content,\n      });\n    }\n\n    messages.push({\n      role: \"user\",\n      content: testCase.question,\n    });\n\n    const providerResponse = await params.provider.forward({\n      model: params.runConfig.model,\n      messages,\n    });\n\n    const response = await MMLUProMainResponseSchemaV1.newWithId(\n      {\n        data: providerResponse.data,\n        startedAt: providerResponse.startedAt,\n        completedAt: providerResponse.completedAt,\n        testCaseId: testCase.id,\n        modelSlug: params.runConfig.model,\n        provider: params.provider.kind,\n\n        inputTokensUsed: providerResponse.inputTokensUsed,\n        outputTokensUsed: providerResponse.outputTokensUsed,\n        inputCost: providerResponse.inputCost,\n        outputCost: providerResponse.outputCost,\n      },\n      responseIdGenerator\n    );\n\n    if (params.scorer?.kind === \"llmJudge\" && params.runConfig.llmJudgeModel) {\n      const scorerResult = await params.scorer.score({\n        task: testCase.question,\n        candidateAnswer: response.data,\n        referenceAnswer: testCase.answer,\n        model: params.runConfig.llmJudgeModel,\n      });\n\n      if (scorerResult !== null) {\n        const score = await MMLUProMainScoreSchemaV1.newWithId(\n          {\n            scoringMethod: ScoringMethod.ai,\n            value: scorerResult.value,\n            responseId: response.id,\n            explanation: scorerResult.explanation,\n            metadata: scorerResult.metadata,\n            extractedAnswers: [],\n\n            scorerAIProvider: scorerResult.provider,\n            scorerAIModelSlug: params.runConfig.llmJudgeModel,\n            scorerAIInputTokensUsed: scorerResult.inputTokensUsed,\n            scorerAIOutputTokensUsed: scorerResult.outputTokensUsed,\n            scorerAIInputCost: scorerResult.inputCost,\n            scorerAIOutputCost: scorerResult.outputCost,\n          },\n          scoreIdGenerator\n        );\n\n        return { response, score };\n      }\n    }\n\n    return { response };\n  }\n\n  throw new Error(\"Unsupported test case kind\");\n}\n\nfunction formatMCQPrompt(testCase: MMLUProMainTestCaseV1) {\n  return `Question: ${testCase.question}\\nOptions:\\n${Object.entries(\n    testCase.options ?? {}\n  )\n    .map(([key, value]) => `${key}: ${value}`)\n    .join(\"\\n\")}`;\n}\n","import {\n  BaseBenchmarkSpecSchemaV1,\n  defineBenchmarkSpecSchema,\n} from \"@/schemas/benchmark-spec\";\nimport z from \"zod\";\n\nexport const MMLUProBenchmarkSpecSchemaV1 = defineBenchmarkSpecSchema({\n  baseSchema: BaseBenchmarkSpecSchemaV1,\n  kind: \"mmlu-pro.benchmark.spec\",\n  schemaVersion: 1,\n  fields: {},\n});\nexport type MMLUProBenchmarkSpecV1 = z.infer<\n  typeof MMLUProBenchmarkSpecSchemaV1\n>;\n","export * from \"./runner\";\nexport * from \"./scorer\";\nexport * from \"./score\";\nexport * from \"./types\";\n\nexport * from \"./test-cases/fnol.v1\";\n","import { z } from \"zod\";\nimport {\n  BaseTestCaseSchemaV1,\n  defineResponseSchema,\n  defineScoreSchema,\n  defineTestCaseSchema,\n} from \"@/schemas\";\nimport { BaseLLMChatResponseSchemaV1 } from \"@/schemas/llm/response\";\nimport { FNOLBaseScoreSchemaV1 } from \"../score\";\nimport { FNOLDoneReason, FNOLFieldValueType } from \"../types\";\n\nexport const FNOLFieldSchemaV1 = z.object({\n  description: z.string(),\n  required: z.boolean().optional(),\n  /**\n   * Optional expected value used by the deterministic scorer.\n   * If omitted, the scorer will only check presence.\n   */\n  expected: z.unknown().optional(),\n  /**\n   * Optional type hint for the model/user simulation.\n   */\n  valueType: z.enum(FNOLFieldValueType).optional(),\n});\n\nexport const FNOLTestCaseSchemaV1 = defineTestCaseSchema({\n  baseSchema: BaseTestCaseSchemaV1,\n  kind: \"fnol.ts.v1\",\n  schemaVersion: 1,\n  fields: {\n    /**\n     * Scenario starter message. This is what the \"user\" would say initially.\n     */\n    initialUserMessage: z.string(),\n\n    /**\n     * Private/structured information about the user and the incident.\n     * This is used by the user simulator LLM to answer the target model questions.\n     */\n    userProfile: z.record(z.string(), z.unknown()),\n\n    /**\n     * The fields the target model must collect.\n     * Keys are canonical identifiers (e.g. \"policyNumber\", \"dateOfLoss\").\n     */\n    fieldsToCollect: z.record(z.string(), FNOLFieldSchemaV1),\n\n    /**\n     * Maximum number of back-and-forth turns (target question + user answer).\n     */\n    maxTurns: z.number().int().min(1).max(100).default(10),\n  },\n});\nexport type FNOLTestCaseV1 = z.infer<typeof FNOLTestCaseSchemaV1>;\n\nexport const FNOLConversationMessageSchemaV1 = z.object({\n  role: z.enum([\"system\", \"user\", \"assistant\"]),\n  content: z.string(),\n});\n\nexport const FNOLResponseSchemaV1 = defineResponseSchema({\n  baseSchema: BaseLLMChatResponseSchemaV1,\n  kind: \"fnol.rs.v1\",\n  schemaVersion: 1,\n  fields: {\n    /**\n     * Full conversation between the target model and simulated user.\n     */\n    conversation: z.array(FNOLConversationMessageSchemaV1),\n    turnsUsed: z.number().int(),\n    doneReason: z.enum(FNOLDoneReason),\n\n    /**\n     * Parsed JSON object from the target model's final answer, if available.\n     */\n    extracted: z.record(z.string(), z.unknown()).optional(),\n  },\n});\nexport type FNOLResponseV1 = z.infer<typeof FNOLResponseSchemaV1>;\n\nexport const FNOLFieldsScoreSchemaV1 = defineScoreSchema({\n  baseSchema: FNOLBaseScoreSchemaV1,\n  kind: \"fnol.sc.fields.v1\",\n  schemaVersion: 1,\n  fields: {\n    requiredKeys: z.array(z.string()),\n    presentKeys: z.array(z.string()),\n    missingKeys: z.array(z.string()),\n    mismatchedKeys: z.array(z.string()),\n  },\n});\nexport type FNOLFieldsScoreV1 = z.infer<typeof FNOLFieldsScoreSchemaV1>;\n\nexport const FNOLLLMJudgeScoreSchemaV1 = defineScoreSchema({\n  baseSchema: FNOLBaseScoreSchemaV1,\n  kind: \"fnol.sc.llm-judge.v1\",\n  schemaVersion: 1,\n  fields: {\n    verdict: z.enum([\"pass\", \"borderline\", \"fail\"]).optional(),\n  },\n});\nexport type FNOLLLMJudgeScoreV1 = z.infer<typeof FNOLLLMJudgeScoreSchemaV1>;\n","import { BaseScoreSchemaV1, defineScoreSchema } from \"@/schemas\";\nimport { ExtensionLLMAsAJudgeScorerFieldsV1 } from \"@/schemas/extensions/score/llm-as-a-judge-scorer\";\n\nexport const FNOLBaseScoreSchemaV1 = defineScoreSchema({\n  baseSchema: BaseScoreSchemaV1,\n  fields: {\n    ...ExtensionLLMAsAJudgeScorerFieldsV1,\n  },\n});\n","export const FNOLFieldValueType = {\n  string: \"string\",\n  number: \"number\",\n  boolean: \"boolean\",\n  object: \"object\",\n} as const;\n\nexport type FNOLFieldValueType =\n  (typeof FNOLFieldValueType)[keyof typeof FNOLFieldValueType];\n\nexport const FNOLDoneReason = {\n  modelProvidedJson: \"modelProvidedJson\",\n  reachedMaxTurns: \"reachedMaxTurns\",\n  forcedFinalJson: \"forcedFinalJson\",\n} as const;\n\nexport type FNOLDoneReason =\n  (typeof FNOLDoneReason)[keyof typeof FNOLDoneReason];\n\n","import { ChatCompletionMessageParam } from \"openai/resources/chat/completions\";\nimport { idGeneratorUUIDv7 } from \"@/utils/id-generator\";\nimport { IdGenerator, RunnerResult } from \"@/types\";\nimport { AbstractLLMProvider } from \"@/providers/abstract/llm\";\nimport { LLMJudgeScorer } from \"@/scorers/llm-judge\";\nimport { SimpleSystemPromptV1 } from \"@/schemas/llm\";\nimport { parseResponseAsJSON } from \"@/utils/llm\";\nimport { ScoringMethod } from \"@/types\";\nimport { FNOLFieldsScorer } from \"./scorer\";\nimport {\n  FNOLFieldsScoreSchemaV1,\n  FNOLFieldsScoreV1,\n  FNOLLLMJudgeScoreSchemaV1,\n  FNOLLLMJudgeScoreV1,\n  FNOLResponseSchemaV1,\n  FNOLResponseV1,\n  FNOLTestCaseV1,\n} from \"./test-cases/fnol.v1\";\nimport { FNOLDoneReason } from \"./types\";\n\nfunction formatFieldsToCollect(\n  fieldsToCollect: Record<string, { description: string; required?: boolean }>\n) {\n  return Object.entries(fieldsToCollect)\n    .map(([key, field]) => {\n      const required = field.required === false ? \"optional\" : \"required\";\n      return `- ${key} (${required}): ${field.description}`;\n    })\n    .join(\"\\n\");\n}\n\nfunction hasAllRequiredFields(params: {\n  extracted?: Record<string, unknown>;\n  fieldsToCollect: Record<string, { required?: boolean }>;\n}) {\n  const extracted = params.extracted ?? {};\n  for (const [key, field] of Object.entries(params.fieldsToCollect)) {\n    if (field.required === false) continue;\n    const value = extracted[key];\n    if (value === undefined || value === null || value === \"\") return false;\n  }\n  return true;\n}\n\nexport async function runTestCase(params: {\n  testCase: FNOLTestCaseV1;\n  provider: AbstractLLMProvider;\n  userSimulatorProvider?: AbstractLLMProvider;\n  scorer?: FNOLFieldsScorer | LLMJudgeScorer;\n  runConfig: {\n    model: string;\n    userSimulatorModel?: string;\n    llmJudgeModel?: string;\n    temperature?: number;\n    userSimulatorTemperature?: number;\n  };\n  systemPrompt?: SimpleSystemPromptV1;\n  idGenerators?: {\n    response?: IdGenerator;\n    score?: IdGenerator;\n  };\n}): Promise<\n  RunnerResult<FNOLResponseV1, FNOLFieldsScoreV1 | FNOLLLMJudgeScoreV1>\n> {\n  const responseIdGenerator =\n    params.idGenerators?.response ?? idGeneratorUUIDv7;\n  const scoreIdGenerator = params.idGenerators?.score ?? idGeneratorUUIDv7;\n\n  const userSimulatorProvider = params.userSimulatorProvider ?? params.provider;\n  const userSimulatorModel =\n    params.runConfig.userSimulatorModel ?? params.runConfig.model;\n\n  const fieldsToCollectText = formatFieldsToCollect(\n    params.testCase.fieldsToCollect\n  );\n\n  const conversation: ChatCompletionMessageParam[] = [];\n\n  if (params.systemPrompt) {\n    conversation.push({\n      role: \"system\",\n      content: params.systemPrompt.content,\n    });\n  }\n\n  conversation.push({\n    role: \"system\",\n    content: [\n      \"You are an insurance FNOL intake assistant.\",\n      \"Your job is to ask the user questions to collect the required fields listed below.\",\n      \"Ask concise questions, one or a few at a time.\",\n      \"When you have enough information OR when you are told to finish, output ONLY a single JSON object with the collected fields.\",\n      \"Do not include markdown fences. Do not include additional text outside the JSON.\",\n      \"\",\n      \"Fields to collect:\",\n      fieldsToCollectText,\n    ].join(\"\\n\"),\n  });\n\n  conversation.push({\n    role: \"user\",\n    content: params.testCase.initialUserMessage,\n  });\n\n  let doneReason: FNOLDoneReason | undefined;\n  let extracted: Record<string, unknown> | undefined;\n\n  const startedAt = Date.now();\n\n  for (let turn = 0; turn < params.testCase.maxTurns; turn++) {\n    const targetReply = await params.provider.forward({\n      model: params.runConfig.model,\n      temperature: params.runConfig.temperature,\n      messages: conversation,\n    });\n\n    conversation.push({\n      role: \"assistant\",\n      content: targetReply.data,\n    });\n\n    extracted = parseResponseAsJSON<Record<string, unknown>>(targetReply.data);\n    if (\n      extracted &&\n      hasAllRequiredFields({\n        extracted,\n        fieldsToCollect: params.testCase.fieldsToCollect,\n      })\n    ) {\n      doneReason = FNOLDoneReason.modelProvidedJson;\n      break;\n    }\n\n    const lastAssistantMessage = targetReply.data;\n\n    const simulatedUser = await userSimulatorProvider.forward({\n      model: userSimulatorModel,\n      temperature: params.runConfig.userSimulatorTemperature,\n      messages: [\n        {\n          role: \"system\",\n          content: [\n            \"You are simulating a real insurance customer (the user).\",\n            \"Answer the assistant's questions truthfully using ONLY the provided user profile and incident details.\",\n            \"If asked about something not present in the profile, say you don't know.\",\n            \"Be concise and natural. Do not invent new facts.\",\n            \"\",\n            \"User profile (JSON):\",\n            JSON.stringify(params.testCase.userProfile),\n          ].join(\"\\n\"),\n        },\n        {\n          role: \"user\",\n          content: lastAssistantMessage,\n        },\n      ],\n    });\n\n    conversation.push({\n      role: \"user\",\n      content: simulatedUser.data,\n    });\n  }\n\n  if (!doneReason) {\n    doneReason = FNOLDoneReason.reachedMaxTurns;\n\n    const forced = await params.provider.forward({\n      model: params.runConfig.model,\n      temperature: params.runConfig.temperature,\n      messages: [\n        ...conversation,\n        {\n          role: \"user\",\n          content:\n            \"Stop the interview now and output ONLY the final JSON object with the collected fields. No extra text.\",\n        },\n      ],\n    });\n\n    conversation.push({ role: \"assistant\", content: forced.data });\n    extracted = parseResponseAsJSON<Record<string, unknown>>(forced.data);\n    if (extracted) {\n      doneReason = FNOLDoneReason.forcedFinalJson;\n    }\n  }\n\n  const completedAt = Date.now();\n  const lastAssistant = [...conversation]\n    .reverse()\n    .find((m) => m.role === \"assistant\");\n\n  const response: FNOLResponseV1 = await FNOLResponseSchemaV1.newWithId(\n    {\n      data:\n        typeof lastAssistant?.content === \"string\" ? lastAssistant.content : \"\",\n      startedAt,\n      completedAt,\n      testCaseId: params.testCase.id,\n      modelSlug: params.runConfig.model,\n      provider: params.provider.kind,\n      conversation: conversation.map((m) => ({\n        role: m.role === \"user\" ? \"user\" : \"assistant\",\n        content: String(m.content),\n      })),\n      turnsUsed: conversation.filter((m) => m.role === \"assistant\").length,\n      doneReason,\n      extracted,\n    },\n    responseIdGenerator\n  );\n\n  if (params.scorer?.kind === \"fnol.fields\") {\n    const scorerResult = await params.scorer.score({\n      fieldsToCollect: params.testCase.fieldsToCollect,\n      extracted,\n    });\n\n    const score = await FNOLFieldsScoreSchemaV1.newWithId(\n      {\n        responseId: response.id,\n        value: scorerResult.value,\n        explanation: scorerResult.explanation,\n        metadata: scorerResult.metadata,\n        scoringMethod: ScoringMethod.algo,\n\n        requiredKeys: scorerResult.requiredKeys,\n        presentKeys: scorerResult.presentKeys,\n        missingKeys: scorerResult.missingKeys,\n        mismatchedKeys: scorerResult.mismatchedKeys,\n      },\n      scoreIdGenerator\n    );\n    return { response, score };\n  }\n\n  if (params.scorer?.kind === \"llmJudge\" && params.runConfig.llmJudgeModel) {\n    const scorerResult = await params.scorer.score({\n      task: \"Evaluate whether the FNOL JSON contains the required fields and correct values.\",\n      candidateAnswer: response.data,\n      referenceAnswer: JSON.stringify(\n        Object.fromEntries(\n          Object.entries(params.testCase.fieldsToCollect).map(([k, v]) => [\n            k,\n            v.expected,\n          ])\n        )\n      ),\n      model: params.runConfig.llmJudgeModel,\n      meta: {\n        fieldsToCollect: params.testCase.fieldsToCollect,\n        doneReason,\n      },\n    });\n\n    if (scorerResult !== null) {\n      const score = await FNOLLLMJudgeScoreSchemaV1.newWithId(\n        {\n          responseId: response.id,\n          value: scorerResult.value,\n          explanation: scorerResult.explanation,\n          metadata: scorerResult.metadata,\n          scoringMethod: ScoringMethod.ai,\n          verdict: scorerResult.verdict,\n\n          scorerAIProvider: scorerResult.provider,\n          scorerAIModelSlug: params.runConfig.llmJudgeModel,\n          scorerAIInputTokensUsed: scorerResult.inputTokensUsed,\n          scorerAIOutputTokensUsed: scorerResult.outputTokensUsed,\n          scorerAIInputCost: scorerResult.inputCost,\n          scorerAIOutputCost: scorerResult.outputCost,\n        },\n        scoreIdGenerator\n      );\n      return { response, score };\n    }\n  }\n\n  return { response };\n}\n","import { AbstractScorer, BaseScorerResult } from \"@/scorers/abstract\";\nimport { stableStringify } from \"@/utils/json\";\n\nfunction isMissing(value: unknown): boolean {\n  return value === undefined || value === null || value === \"\";\n}\n\nfunction normalizeString(value: string): string {\n  return value.trim();\n}\n\nfunction valuesEqual(expected: unknown, actual: unknown): boolean {\n  if (typeof expected === \"string\" && typeof actual === \"string\") {\n    return normalizeString(expected) === normalizeString(actual);\n  }\n  return stableStringify(expected) === stableStringify(actual);\n}\n\nexport class FNOLFieldsScorer extends AbstractScorer {\n  override readonly kind = \"fnol.fields\";\n\n  override async score(params: {\n    fieldsToCollect: Record<\n      string,\n      { required?: boolean; expected?: unknown; description?: string }\n    >;\n    extracted?: Record<string, unknown>;\n  }): Promise<\n    BaseScorerResult & {\n      requiredKeys: string[];\n      presentKeys: string[];\n      missingKeys: string[];\n      mismatchedKeys: string[];\n    }\n  > {\n    const extracted = params.extracted ?? {};\n    const requiredKeys = Object.entries(params.fieldsToCollect)\n      .filter(([, field]) => field.required !== false)\n      .map(([key]) => key);\n\n    const presentKeys: string[] = [];\n    const missingKeys: string[] = [];\n    const mismatchedKeys: string[] = [];\n\n    for (const key of requiredKeys) {\n      const value = extracted[key];\n      if (isMissing(value)) {\n        missingKeys.push(key);\n        continue;\n      }\n      presentKeys.push(key);\n\n      const expected = params.fieldsToCollect[key]?.expected;\n      if (expected !== undefined && !valuesEqual(expected, value)) {\n        mismatchedKeys.push(key);\n      }\n    }\n\n    const requiredCount = requiredKeys.length;\n    const correctCount = requiredCount - missingKeys.length - mismatchedKeys.length;\n    const score = requiredCount === 0 ? 1 : correctCount / requiredCount;\n\n    return {\n      value: Math.max(0, Math.min(1, score)),\n      explanation:\n        missingKeys.length === 0 && mismatchedKeys.length === 0\n          ? \"All required fields collected\"\n          : \"Missing or mismatched fields\",\n      requiredKeys,\n      presentKeys,\n      missingKeys,\n      mismatchedKeys,\n      metadata: {\n        requiredCount,\n        presentCount: presentKeys.length,\n        missingCount: missingKeys.length,\n        mismatchedCount: mismatchedKeys.length,\n      },\n    };\n  }\n}\n\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;;;ACAA,SAAS,SAAS;;;ACGX,IAAM,6BAA6B,kBAAkB;AAAA,EAC1D,YAAY;AAAA,EACZ,QAAQ;AAAA,IACN,GAAG;AAAA,EACL;AACF,CAAC;;;ADGM,IAAM,0CAA0C,qBAAqB;AAAA,EAC1E,YAAY;AAAA,EAEZ,MAAM;AAAA,EACN,eAAe;AAAA,EACf,QAAQ;AAAA,IACN,UAAU,EAAE,OAAO;AAAA,IACnB,SAAS,EAAE,OAAO,EAAE,OAAO,GAAG,EAAE,OAAO,CAAC;AAAA,IACxC,QAAQ,EAAE,OAAO;AAAA,IACjB,WAAW,EAAE,OAAO;AAAA,EACtB;AACF,CAAC;AAMM,IAAM,0CAA0C,qBAAqB;AAAA,EAC1E,YAAY;AAAA,EACZ,MAAM;AAAA,EACN,eAAe;AAAA,EACf,QAAQ,CAAC;AACX,CAAC;AAMM,IAAM,uCAAuC,kBAAkB;AAAA,EACpE,YAAY;AAAA,EACZ,MAAM;AAAA,EACN,eAAe;AAAA,EACf,QAAQ;AAAA,IACN,kBAAkB,EAAE,MAAM,EAAE,OAAO,CAAC;AAAA,EACtC;AACF,CAAC;;;AExCD,SAAS,KAAAA,UAAS;AAIX,IAAM,qCAAqC,qBAAqB;AAAA,EACrE,MAAM;AAAA,EACN,eAAe;AAAA,EACf,YAAY;AAAA,EACZ,QAAQ;AAAA,IACN,UAAUC,GAAE,OAAO;AAAA,IACnB,QAAQA,GAAE,OAAO,EAAE,SAAS;AAAA,EAC9B;AACF,CAAC;AAMM,IAAM,qCAAqC,qBAAqB;AAAA,EACrE,YAAY;AAAA,EACZ,MAAM;AAAA,EACN,eAAe;AACjB,CAAC;AAMM,IAAM,kCAAkC,kBAAkB;AAAA,EAC/D,YAAY;AAAA,EACZ,MAAM;AAAA,EACN,eAAe;AAAA,EACf,QAAQ,CAAC;AACX,CAAC;;;ACnCD,OAAOC,QAAO;AAEP,IAAM,iCAAiC,0BAA0B;AAAA,EACtE,YAAY;AAAA,EACZ,MAAM;AAAA,EACN,eAAe;AAAA,EACf,QAAQ;AAAA;AAAA;AAAA;AAAA,IAIN,WAAWA,GAAE,OAAOA,GAAE,OAAO,GAAGA,GAAE,OAAO,CAAC,EAAE,SAAS;AAAA,EACvD;AACF,CAAC;;;ACKD,OAAOC,QAAO;AAGP,IAAM,0BAAN,cAAsC,2BAI3C;AAAA,EACkB,OAAO;AAAA,EAEzB,MAAM,kBAAkB,QAEc;AACpC,UAAM,UAAU,eAAe,OAAO,OAAO;AAC7C,UAAM,SAAS,+BAA+B,MAAM,OAAO;AAE3D,WAAO;AAAA,EACT;AAAA,EAEmB,gBAAgB,MAAW;AAC5C,UAAM,qBAAqBC,GACxB,MAAM;AAAA,MACL;AAAA,MACA;AAAA,IACF,CAAC,EACA,UAAU,IAAI;AACjB,WAAO,mBAAmB,UAAU,mBAAmB,OAAO;AAAA,EAChE;AAAA,EAEA,MAAyB,gBAAgB,MAAW;AAClD,UAAM,qBAAqBA,GACxB,MAAM;AAAA,MACL;AAAA,MACA;AAAA,IACF,CAAC,EACA,UAAU,IAAI;AACjB,WAAO,mBAAmB,UAAU,mBAAmB,OAAO;AAAA,EAChE;AAAA,EAEA,MAAyB,aAAa,MAAW;AAC/C,UAAM,kBAAkBA,GACrB,MAAM;AAAA,MACL;AAAA,MACA;AAAA,IACF,CAAC,EACA,UAAU,IAAI;AACjB,WAAO,gBAAgB,UAAU,gBAAgB,OAAO;AAAA,EAC1D;AACF;;;ACrCA,eAAsB,YAAY,QAcmB;AACnD,QAAM,EAAE,SAAS,IAAI;AACrB,QAAM,sBACJ,OAAO,cAAc,YAAY;AACnC,QAAM,mBAAmB,OAAO,cAAc,SAAS;AACvD,QAAM,WAAyC,CAAC;AAEhD,MAAI,OAAO,cAAc;AACvB,aAAS,KAAK;AAAA,MACZ,MAAM;AAAA,MACN,SAAS,OAAO,aAAa;AAAA,IAC/B,CAAC;AAAA,EACH;AAEA,MAAI,SAAS,SAAS,aAAa;AACjC,UAAM,kBAAkB,gBAAgB,QAAQ;AAEhD,aAAS,KAAK;AAAA,MACZ,MAAM;AAAA,MACN,SAAS;AAAA,IACX,CAAC;AAED,UAAM,mBAAmB,MAAM,OAAO,SAAS,QAAQ;AAAA,MACrD,OAAO,OAAO,UAAU;AAAA,MACxB;AAAA,IACF,CAAC;AAED,UAAM,WAAW,MAAM,wCAAwC;AAAA,MAC7D;AAAA,QACE,MAAM,iBAAiB;AAAA,QACvB,WAAW,iBAAiB;AAAA,QAC5B,aAAa,iBAAiB;AAAA,QAC9B,YAAY,SAAS;AAAA,QACrB,WAAW,OAAO,UAAU;AAAA,QAC5B,UAAU,OAAO,SAAS;AAAA,QAE1B,iBAAiB,iBAAiB;AAAA,QAClC,kBAAkB,iBAAiB;AAAA,QACnC,WAAW,iBAAiB;AAAA,QAC5B,YAAY,iBAAiB;AAAA,MAC/B;AAAA,MACA;AAAA,IACF;AAEA,QAAI,OAAO,QAAQ,SAAS,OAAO;AACjC,YAAM,eAAe,MAAM,OAAO,OAAO,MAAM;AAAA,QAC7C,UAAU,SAAS;AAAA,QACnB,SAAS,SAAS,WAAW,CAAC;AAAA,QAC9B,gBAAgB,CAAC,SAAS,SAAS;AAAA,MACrC,CAAC;AAED,UAAI,iBAAiB,MAAM;AACzB,cAAM,QAAQ,MAAM,qCAAqC;AAAA,UACvD;AAAA,YACE,eAAe,cAAc;AAAA,YAC7B,OAAO,aAAa;AAAA,YACpB,YAAY,SAAS;AAAA,YACrB,kBAAkB,aAAa;AAAA,YAC/B,UAAU,SAAS;AAAA,UACrB;AAAA,UACA;AAAA,QACF;AAEA,eAAO,EAAE,UAAU,MAAM;AAAA,MAC3B;AAAA,IACF;AAEA,WAAO,EAAE,SAAS;AAAA,EACpB,WAAW,SAAS,SAAS,oBAAoB;AAC/C,UAAMC,YAAyC,CAAC;AAEhD,QAAI,OAAO,cAAc;AACvB,MAAAA,UAAS,KAAK;AAAA,QACZ,MAAM;AAAA,QACN,SAAS,OAAO,aAAa;AAAA,MAC/B,CAAC;AAAA,IACH;AAEA,IAAAA,UAAS,KAAK;AAAA,MACZ,MAAM;AAAA,MACN,SAAS,SAAS;AAAA,IACpB,CAAC;AAED,UAAM,mBAAmB,MAAM,OAAO,SAAS,QAAQ;AAAA,MACrD,OAAO,OAAO,UAAU;AAAA,MACxB,UAAAA;AAAA,IACF,CAAC;AAED,UAAM,WAAW,MAAM,mCAAmC;AAAA,MACxD;AAAA,QACE,MAAM,iBAAiB;AAAA,QACvB,WAAW,iBAAiB;AAAA,QAC5B,aAAa,iBAAiB;AAAA,QAC9B,YAAY,SAAS;AAAA,QACrB,WAAW,OAAO,UAAU;AAAA,QAC5B,UAAU,OAAO,SAAS;AAAA,QAE1B,iBAAiB,iBAAiB;AAAA,QAClC,kBAAkB,iBAAiB;AAAA,QACnC,WAAW,iBAAiB;AAAA,QAC5B,YAAY,iBAAiB;AAAA,MAC/B;AAAA,MACA;AAAA,IACF;AAEA,QAAI,OAAO,QAAQ,SAAS,cAAc,OAAO,UAAU,eAAe;AACxE,YAAM,eAAe,MAAM,OAAO,OAAO,MAAM;AAAA,QAC7C,MAAM,SAAS;AAAA,QACf,iBAAiB,SAAS;AAAA,QAC1B,iBAAiB,SAAS;AAAA,QAC1B,OAAO,OAAO,UAAU;AAAA,MAC1B,CAAC;AAED,UAAI,iBAAiB,MAAM;AACzB,cAAM,QAAQ,MAAM,gCAAgC;AAAA,UAClD;AAAA,YACE,eAAe,cAAc;AAAA,YAC7B,OAAO,aAAa;AAAA,YACpB,YAAY,SAAS;AAAA,YACrB,aAAa,aAAa;AAAA,YAC1B,UAAU,aAAa;AAAA,YAEvB,kBAAkB,aAAa;AAAA,YAC/B,mBAAmB,OAAO,UAAU;AAAA,YACpC,yBAAyB,aAAa;AAAA,YACtC,0BAA0B,aAAa;AAAA,YACvC,mBAAmB,aAAa;AAAA,YAChC,oBAAoB,aAAa;AAAA,UACnC;AAAA,UACA;AAAA,QACF;AAEA,eAAO,EAAE,UAAU,MAAM;AAAA,MAC3B;AAAA,IACF;AAEA,WAAO,EAAE,SAAS;AAAA,EACpB;AAEA,QAAM,IAAI,MAAM,4BAA4B;AAC9C;AAEA,SAAS,gBAAgB,UAA6C;AACpE,SAAO,aAAa,SAAS,QAAQ;AAAA;AAAA,EAAe,OAAO;AAAA,IACzD,SAAS,WAAW,CAAC;AAAA,EACvB,EACG,IAAI,CAAC,CAAC,KAAK,KAAK,MAAM,GAAG,GAAG,KAAK,KAAK,EAAE,EACxC,KAAK,IAAI,CAAC;AACf;;;AClMA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,qBAAAC;AAAA;;;ACGO,IAAM,2BAA2B,kBAAkB;AAAA,EACxD,YAAY;AAAA,EACZ,QAAQ;AAAA,IACN,GAAG;AAAA,EACL;AACF,CAAC;;;ACDD,SAAS,KAAAC,UAAS;AAGX,IAAM,8BAA8B,qBAAqB;AAAA,EAC9D,YAAY;AAAA,EACZ,MAAM;AAAA,EACN,eAAe;AAAA,EACf,QAAQ;AAAA,IACN,UAAUA,GAAE,OAAO;AAAA,IACnB,SAASA,GAAE,OAAOA,GAAE,OAAO,GAAGA,GAAE,OAAO,CAAC;AAAA,IACxC,QAAQA,GAAE,OAAO;AAAA,IACjB,WAAWA,GAAE,OAAO;AAAA,EACtB;AACF,CAAC;AAIM,IAAM,8BAA8B,qBAAqB;AAAA,EAC9D,YAAY;AAAA,EACZ,MAAM;AAAA,EACN,eAAe;AACjB,CAAC;AAIM,IAAM,2BAA2B,kBAAkB;AAAA,EACxD,YAAY;AAAA,EACZ,MAAM;AAAA,EACN,eAAe;AAAA,EACf,QAAQ;AAAA,IACN,kBAAkBA,GAAE,MAAMA,GAAE,OAAO,CAAC;AAAA,EACtC;AACF,CAAC;;;AChCD,OAAOC,QAAO;AAGd,SAAS,0BAA0B;AAGnC,IAAM,aAAaC,GAChB,OAAO;AAAA,EACN,aAAaA,GAAE,OAAO,OAAO;AAAA,EAC7B,UAAUA,GAAE,OAAO;AAAA,EACnB,SAASA,GAAE,MAAMA,GAAE,OAAO,CAAC;AAAA,EAC3B,QAAQA,GAAE,OAAO;AAAA,EACjB,cAAcA,GAAE,OAAO,OAAO;AAAA,EAC9B,aAAaA,GAAE,OAAO;AAAA,EACtB,UAAUA,GAAE,OAAO;AAAA,EACnB,KAAKA,GAAE,OAAO;AAChB,CAAC,EACA,MAAM;AAET,SAAS,QACP,MAKA;AACA,SAAO;AAAA,IACL,WAAW,CAAC;AAAA,IACZ,QAAQ,CAAC;AAAA,IACT,WAAW,KAAK;AAAA,MAAI,CAAC,SACnB,4BAA4B,IAAI;AAAA,QAC9B,IAAI,GAAG,KAAK,GAAG,IAAI,KAAK,QAAQ,IAAI,KAAK,WAAW;AAAA,QACpD,UAAU,KAAK;AAAA,QACf,WAAW,KAAK;AAAA,QAChB,SAAS,KAAK,QAAQ;AAAA,UACpB,CAAC,KAAK,QAAQ,UAAU;AACtB,gBAAI,OAAO,aAAa,KAAK,KAAK,CAAC,IAAI;AACvC,mBAAO;AAAA,UACT;AAAA,UACA,CAAC;AAAA,QACH;AAAA,QACA,QAAQ,KAAK,QAAQ,KAAK,YAAY;AAAA,QACtC,UAAU;AAAA,UACR,UAAU,KAAK;AAAA,UACf,KAAK,KAAK;AAAA,UACV,cAAc,KAAK;AAAA,QACrB;AAAA,MACF,CAAC;AAAA,IACH;AAAA,EACF;AACF;AAEO,IAAM,wBAAN,cAAoC,mBAAmB;AAAA,EAC1C,OAAO;AAAA,EAEhB,SAAS,QAMhB;AACA,UAAM,UACJ,OAAO,OAAO,YAAY,WACtB,OAAO,UACP,eAAe,OAAO,OAAO;AAEnC,UAAM,SAAS,WAAW,MAAM,KAAK,MAAM,OAAO,CAAC;AACnD,WAAO,QAAQ,MAAM;AAAA,EACvB;AAAA;AAAA,EAGS,kBAAkB,QAES;AAClC,UAAM,IAAI,MAAM,iBAAiB;AAAA,EACnC;AACF;AAEO,IAAM,2BAAN,cAAuC,mBAAmB;AAAA,EAC7C,OAAO;AAAA,EAEzB,MAAe,SAAS,QAQtB;AACA,UAAM,OAAO,MAAM,mBAAmB;AAAA,MACpC,MAAM,OAAO,QAAQ;AAAA,IACvB,CAAC;AACD,QAAI,CAAC,MAAM;AACT,YAAM,IAAI,MAAM,sBAAsB;AAAA,IACxC;AAEA,WAAO,QAAQ,WAAW,MAAM,IAAI,CAAC;AAAA,EACvC;AAAA;AAAA,EAGS,kBAAkB,QAES;AAClC,UAAM,IAAI,MAAM,iBAAiB;AAAA,EACnC;AACF;;;ACjGA,eAAsBC,aAAY,QAcmC;AACnE,QAAM,EAAE,SAAS,IAAI;AACrB,QAAM,sBACJ,OAAO,cAAc,YAAY;AACnC,QAAM,mBAAmB,OAAO,cAAc,SAAS;AACvD,QAAM,WAAyC,CAAC;AAEhD,MAAI,OAAO,cAAc;AACvB,aAAS,KAAK;AAAA,MACZ,MAAM;AAAA,MACN,SAAS,OAAO,aAAa;AAAA,IAC/B,CAAC;AAAA,EACH;AAEA,MAAI,SAAS,SAAS,oBAAoB;AACxC,UAAM,kBAAkBC,iBAAgB,QAAQ;AAEhD,aAAS,KAAK;AAAA,MACZ,MAAM;AAAA,MACN,SAAS;AAAA,IACX,CAAC;AAED,UAAM,mBAAmB,MAAM,OAAO,SAAS,QAAQ;AAAA,MACrD,OAAO,OAAO,UAAU;AAAA,MACxB;AAAA,IACF,CAAC;AAED,UAAM,WAAW,MAAM,4BAA4B;AAAA,MACjD;AAAA,QACE,MAAM,iBAAiB;AAAA,QACvB,WAAW,iBAAiB;AAAA,QAC5B,aAAa,iBAAiB;AAAA,QAC9B,YAAY,SAAS;AAAA,QACrB,WAAW,OAAO,UAAU;AAAA,QAC5B,UAAU,OAAO,SAAS;AAAA,QAE1B,iBAAiB,iBAAiB;AAAA,QAClC,kBAAkB,iBAAiB;AAAA,QACnC,WAAW,iBAAiB;AAAA,QAC5B,YAAY,iBAAiB;AAAA,MAC/B;AAAA,MACA;AAAA,IACF;AAEA,QAAI,OAAO,QAAQ,SAAS,OAAO;AACjC,YAAM,eAAe,MAAM,OAAO,OAAO,MAAM;AAAA,QAC7C,UAAU,SAAS;AAAA,QACnB,SAAS,SAAS,WAAW,CAAC;AAAA,QAC9B,gBAAgB,CAAC,SAAS,SAAS;AAAA,MACrC,CAAC;AAED,UAAI,iBAAiB,MAAM;AACzB,cAAM,QAAQ,MAAM,yBAAyB;AAAA,UAC3C;AAAA,YACE,eAAe,cAAc;AAAA,YAC7B,OAAO,aAAa;AAAA,YACpB,YAAY,SAAS;AAAA,YACrB,kBAAkB,aAAa;AAAA,YAC/B,UAAU,SAAS;AAAA,UACrB;AAAA,UACA;AAAA,QACF;AAEA,eAAO,EAAE,UAAU,MAAM;AAAA,MAC3B;AAAA,IACF;AAEA,WAAO,EAAE,SAAS;AAAA,EACpB,WAAW,SAAS,SAAS,oBAAoB;AAC/C,UAAMC,YAAyC,CAAC;AAEhD,QAAI,OAAO,cAAc;AACvB,MAAAA,UAAS,KAAK;AAAA,QACZ,MAAM;AAAA,QACN,SAAS,OAAO,aAAa;AAAA,MAC/B,CAAC;AAAA,IACH;AAEA,IAAAA,UAAS,KAAK;AAAA,MACZ,MAAM;AAAA,MACN,SAAS,SAAS;AAAA,IACpB,CAAC;AAED,UAAM,mBAAmB,MAAM,OAAO,SAAS,QAAQ;AAAA,MACrD,OAAO,OAAO,UAAU;AAAA,MACxB,UAAAA;AAAA,IACF,CAAC;AAED,UAAM,WAAW,MAAM,4BAA4B;AAAA,MACjD;AAAA,QACE,MAAM,iBAAiB;AAAA,QACvB,WAAW,iBAAiB;AAAA,QAC5B,aAAa,iBAAiB;AAAA,QAC9B,YAAY,SAAS;AAAA,QACrB,WAAW,OAAO,UAAU;AAAA,QAC5B,UAAU,OAAO,SAAS;AAAA,QAE1B,iBAAiB,iBAAiB;AAAA,QAClC,kBAAkB,iBAAiB;AAAA,QACnC,WAAW,iBAAiB;AAAA,QAC5B,YAAY,iBAAiB;AAAA,MAC/B;AAAA,MACA;AAAA,IACF;AAEA,QAAI,OAAO,QAAQ,SAAS,cAAc,OAAO,UAAU,eAAe;AACxE,YAAM,eAAe,MAAM,OAAO,OAAO,MAAM;AAAA,QAC7C,MAAM,SAAS;AAAA,QACf,iBAAiB,SAAS;AAAA,QAC1B,iBAAiB,SAAS;AAAA,QAC1B,OAAO,OAAO,UAAU;AAAA,MAC1B,CAAC;AAED,UAAI,iBAAiB,MAAM;AACzB,cAAM,QAAQ,MAAM,yBAAyB;AAAA,UAC3C;AAAA,YACE,eAAe,cAAc;AAAA,YAC7B,OAAO,aAAa;AAAA,YACpB,YAAY,SAAS;AAAA,YACrB,aAAa,aAAa;AAAA,YAC1B,UAAU,aAAa;AAAA,YACvB,kBAAkB,CAAC;AAAA,YAEnB,kBAAkB,aAAa;AAAA,YAC/B,mBAAmB,OAAO,UAAU;AAAA,YACpC,yBAAyB,aAAa;AAAA,YACtC,0BAA0B,aAAa;AAAA,YACvC,mBAAmB,aAAa;AAAA,YAChC,oBAAoB,aAAa;AAAA,UACnC;AAAA,UACA;AAAA,QACF;AAEA,eAAO,EAAE,UAAU,MAAM;AAAA,MAC3B;AAAA,IACF;AAEA,WAAO,EAAE,SAAS;AAAA,EACpB;AAEA,QAAM,IAAI,MAAM,4BAA4B;AAC9C;AAEA,SAASD,iBAAgB,UAAiC;AACxD,SAAO,aAAa,SAAS,QAAQ;AAAA;AAAA,EAAe,OAAO;AAAA,IACzD,SAAS,WAAW,CAAC;AAAA,EACvB,EACG,IAAI,CAAC,CAAC,KAAK,KAAK,MAAM,GAAG,GAAG,KAAK,KAAK,EAAE,EACxC,KAAK,IAAI,CAAC;AACf;;;AC9KO,IAAM,+BAA+B,0BAA0B;AAAA,EACpE,YAAY;AAAA,EACZ,MAAM;AAAA,EACN,eAAe;AAAA,EACf,QAAQ,CAAC;AACX,CAAC;;;ACXD;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,qBAAAE;AAAA;;;ACAA,SAAS,KAAAC,UAAS;;;ACGX,IAAM,wBAAwB,kBAAkB;AAAA,EACrD,YAAY;AAAA,EACZ,QAAQ;AAAA,IACN,GAAG;AAAA,EACL;AACF,CAAC;;;ACRM,IAAM,qBAAqB;AAAA,EAChC,QAAQ;AAAA,EACR,QAAQ;AAAA,EACR,SAAS;AAAA,EACT,QAAQ;AACV;AAKO,IAAM,iBAAiB;AAAA,EAC5B,mBAAmB;AAAA,EACnB,iBAAiB;AAAA,EACjB,iBAAiB;AACnB;;;AFHO,IAAM,oBAAoBC,GAAE,OAAO;AAAA,EACxC,aAAaA,GAAE,OAAO;AAAA,EACtB,UAAUA,GAAE,QAAQ,EAAE,SAAS;AAAA;AAAA;AAAA;AAAA;AAAA,EAK/B,UAAUA,GAAE,QAAQ,EAAE,SAAS;AAAA;AAAA;AAAA;AAAA,EAI/B,WAAWA,GAAE,KAAK,kBAAkB,EAAE,SAAS;AACjD,CAAC;AAEM,IAAM,uBAAuB,qBAAqB;AAAA,EACvD,YAAY;AAAA,EACZ,MAAM;AAAA,EACN,eAAe;AAAA,EACf,QAAQ;AAAA;AAAA;AAAA;AAAA,IAIN,oBAAoBA,GAAE,OAAO;AAAA;AAAA;AAAA;AAAA;AAAA,IAM7B,aAAaA,GAAE,OAAOA,GAAE,OAAO,GAAGA,GAAE,QAAQ,CAAC;AAAA;AAAA;AAAA;AAAA;AAAA,IAM7C,iBAAiBA,GAAE,OAAOA,GAAE,OAAO,GAAG,iBAAiB;AAAA;AAAA;AAAA;AAAA,IAKvD,UAAUA,GAAE,OAAO,EAAE,IAAI,EAAE,IAAI,CAAC,EAAE,IAAI,GAAG,EAAE,QAAQ,EAAE;AAAA,EACvD;AACF,CAAC;AAGM,IAAM,kCAAkCA,GAAE,OAAO;AAAA,EACtD,MAAMA,GAAE,KAAK,CAAC,UAAU,QAAQ,WAAW,CAAC;AAAA,EAC5C,SAASA,GAAE,OAAO;AACpB,CAAC;AAEM,IAAM,uBAAuB,qBAAqB;AAAA,EACvD,YAAY;AAAA,EACZ,MAAM;AAAA,EACN,eAAe;AAAA,EACf,QAAQ;AAAA;AAAA;AAAA;AAAA,IAIN,cAAcA,GAAE,MAAM,+BAA+B;AAAA,IACrD,WAAWA,GAAE,OAAO,EAAE,IAAI;AAAA,IAC1B,YAAYA,GAAE,KAAK,cAAc;AAAA;AAAA;AAAA;AAAA,IAKjC,WAAWA,GAAE,OAAOA,GAAE,OAAO,GAAGA,GAAE,QAAQ,CAAC,EAAE,SAAS;AAAA,EACxD;AACF,CAAC;AAGM,IAAM,0BAA0B,kBAAkB;AAAA,EACvD,YAAY;AAAA,EACZ,MAAM;AAAA,EACN,eAAe;AAAA,EACf,QAAQ;AAAA,IACN,cAAcA,GAAE,MAAMA,GAAE,OAAO,CAAC;AAAA,IAChC,aAAaA,GAAE,MAAMA,GAAE,OAAO,CAAC;AAAA,IAC/B,aAAaA,GAAE,MAAMA,GAAE,OAAO,CAAC;AAAA,IAC/B,gBAAgBA,GAAE,MAAMA,GAAE,OAAO,CAAC;AAAA,EACpC;AACF,CAAC;AAGM,IAAM,4BAA4B,kBAAkB;AAAA,EACzD,YAAY;AAAA,EACZ,MAAM;AAAA,EACN,eAAe;AAAA,EACf,QAAQ;AAAA,IACN,SAASA,GAAE,KAAK,CAAC,QAAQ,cAAc,MAAM,CAAC,EAAE,SAAS;AAAA,EAC3D;AACF,CAAC;;;AGhFD,SAAS,sBACP,iBACA;AACA,SAAO,OAAO,QAAQ,eAAe,EAClC,IAAI,CAAC,CAAC,KAAK,KAAK,MAAM;AACrB,UAAM,WAAW,MAAM,aAAa,QAAQ,aAAa;AACzD,WAAO,KAAK,GAAG,KAAK,QAAQ,MAAM,MAAM,WAAW;AAAA,EACrD,CAAC,EACA,KAAK,IAAI;AACd;AAEA,SAAS,qBAAqB,QAG3B;AACD,QAAM,YAAY,OAAO,aAAa,CAAC;AACvC,aAAW,CAAC,KAAK,KAAK,KAAK,OAAO,QAAQ,OAAO,eAAe,GAAG;AACjE,QAAI,MAAM,aAAa,MAAO;AAC9B,UAAM,QAAQ,UAAU,GAAG;AAC3B,QAAI,UAAU,UAAa,UAAU,QAAQ,UAAU,GAAI,QAAO;AAAA,EACpE;AACA,SAAO;AACT;AAEA,eAAsBC,aAAY,QAmBhC;AACA,QAAM,sBACJ,OAAO,cAAc,YAAY;AACnC,QAAM,mBAAmB,OAAO,cAAc,SAAS;AAEvD,QAAM,wBAAwB,OAAO,yBAAyB,OAAO;AACrE,QAAM,qBACJ,OAAO,UAAU,sBAAsB,OAAO,UAAU;AAE1D,QAAM,sBAAsB;AAAA,IAC1B,OAAO,SAAS;AAAA,EAClB;AAEA,QAAM,eAA6C,CAAC;AAEpD,MAAI,OAAO,cAAc;AACvB,iBAAa,KAAK;AAAA,MAChB,MAAM;AAAA,MACN,SAAS,OAAO,aAAa;AAAA,IAC/B,CAAC;AAAA,EACH;AAEA,eAAa,KAAK;AAAA,IAChB,MAAM;AAAA,IACN,SAAS;AAAA,MACP;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACF,EAAE,KAAK,IAAI;AAAA,EACb,CAAC;AAED,eAAa,KAAK;AAAA,IAChB,MAAM;AAAA,IACN,SAAS,OAAO,SAAS;AAAA,EAC3B,CAAC;AAED,MAAI;AACJ,MAAI;AAEJ,QAAM,YAAY,KAAK,IAAI;AAE3B,WAAS,OAAO,GAAG,OAAO,OAAO,SAAS,UAAU,QAAQ;AAC1D,UAAM,cAAc,MAAM,OAAO,SAAS,QAAQ;AAAA,MAChD,OAAO,OAAO,UAAU;AAAA,MACxB,aAAa,OAAO,UAAU;AAAA,MAC9B,UAAU;AAAA,IACZ,CAAC;AAED,iBAAa,KAAK;AAAA,MAChB,MAAM;AAAA,MACN,SAAS,YAAY;AAAA,IACvB,CAAC;AAED,gBAAY,oBAA6C,YAAY,IAAI;AACzE,QACE,aACA,qBAAqB;AAAA,MACnB;AAAA,MACA,iBAAiB,OAAO,SAAS;AAAA,IACnC,CAAC,GACD;AACA,mBAAa,eAAe;AAC5B;AAAA,IACF;AAEA,UAAM,uBAAuB,YAAY;AAEzC,UAAM,gBAAgB,MAAM,sBAAsB,QAAQ;AAAA,MACxD,OAAO;AAAA,MACP,aAAa,OAAO,UAAU;AAAA,MAC9B,UAAU;AAAA,QACR;AAAA,UACE,MAAM;AAAA,UACN,SAAS;AAAA,YACP;AAAA,YACA;AAAA,YACA;AAAA,YACA;AAAA,YACA;AAAA,YACA;AAAA,YACA,KAAK,UAAU,OAAO,SAAS,WAAW;AAAA,UAC5C,EAAE,KAAK,IAAI;AAAA,QACb;AAAA,QACA;AAAA,UACE,MAAM;AAAA,UACN,SAAS;AAAA,QACX;AAAA,MACF;AAAA,IACF,CAAC;AAED,iBAAa,KAAK;AAAA,MAChB,MAAM;AAAA,MACN,SAAS,cAAc;AAAA,IACzB,CAAC;AAAA,EACH;AAEA,MAAI,CAAC,YAAY;AACf,iBAAa,eAAe;AAE5B,UAAM,SAAS,MAAM,OAAO,SAAS,QAAQ;AAAA,MAC3C,OAAO,OAAO,UAAU;AAAA,MACxB,aAAa,OAAO,UAAU;AAAA,MAC9B,UAAU;AAAA,QACR,GAAG;AAAA,QACH;AAAA,UACE,MAAM;AAAA,UACN,SACE;AAAA,QACJ;AAAA,MACF;AAAA,IACF,CAAC;AAED,iBAAa,KAAK,EAAE,MAAM,aAAa,SAAS,OAAO,KAAK,CAAC;AAC7D,gBAAY,oBAA6C,OAAO,IAAI;AACpE,QAAI,WAAW;AACb,mBAAa,eAAe;AAAA,IAC9B;AAAA,EACF;AAEA,QAAM,cAAc,KAAK,IAAI;AAC7B,QAAM,gBAAgB,CAAC,GAAG,YAAY,EACnC,QAAQ,EACR,KAAK,CAAC,MAAM,EAAE,SAAS,WAAW;AAErC,QAAM,WAA2B,MAAM,qBAAqB;AAAA,IAC1D;AAAA,MACE,MACE,OAAO,eAAe,YAAY,WAAW,cAAc,UAAU;AAAA,MACvE;AAAA,MACA;AAAA,MACA,YAAY,OAAO,SAAS;AAAA,MAC5B,WAAW,OAAO,UAAU;AAAA,MAC5B,UAAU,OAAO,SAAS;AAAA,MAC1B,cAAc,aAAa,IAAI,CAAC,OAAO;AAAA,QACrC,MAAM,EAAE,SAAS,SAAS,SAAS;AAAA,QACnC,SAAS,OAAO,EAAE,OAAO;AAAA,MAC3B,EAAE;AAAA,MACF,WAAW,aAAa,OAAO,CAAC,MAAM,EAAE,SAAS,WAAW,EAAE;AAAA,MAC9D;AAAA,MACA;AAAA,IACF;AAAA,IACA;AAAA,EACF;AAEA,MAAI,OAAO,QAAQ,SAAS,eAAe;AACzC,UAAM,eAAe,MAAM,OAAO,OAAO,MAAM;AAAA,MAC7C,iBAAiB,OAAO,SAAS;AAAA,MACjC;AAAA,IACF,CAAC;AAED,UAAM,QAAQ,MAAM,wBAAwB;AAAA,MAC1C;AAAA,QACE,YAAY,SAAS;AAAA,QACrB,OAAO,aAAa;AAAA,QACpB,aAAa,aAAa;AAAA,QAC1B,UAAU,aAAa;AAAA,QACvB,eAAe,cAAc;AAAA,QAE7B,cAAc,aAAa;AAAA,QAC3B,aAAa,aAAa;AAAA,QAC1B,aAAa,aAAa;AAAA,QAC1B,gBAAgB,aAAa;AAAA,MAC/B;AAAA,MACA;AAAA,IACF;AACA,WAAO,EAAE,UAAU,MAAM;AAAA,EAC3B;AAEA,MAAI,OAAO,QAAQ,SAAS,cAAc,OAAO,UAAU,eAAe;AACxE,UAAM,eAAe,MAAM,OAAO,OAAO,MAAM;AAAA,MAC7C,MAAM;AAAA,MACN,iBAAiB,SAAS;AAAA,MAC1B,iBAAiB,KAAK;AAAA,QACpB,OAAO;AAAA,UACL,OAAO,QAAQ,OAAO,SAAS,eAAe,EAAE,IAAI,CAAC,CAAC,GAAG,CAAC,MAAM;AAAA,YAC9D;AAAA,YACA,EAAE;AAAA,UACJ,CAAC;AAAA,QACH;AAAA,MACF;AAAA,MACA,OAAO,OAAO,UAAU;AAAA,MACxB,MAAM;AAAA,QACJ,iBAAiB,OAAO,SAAS;AAAA,QACjC;AAAA,MACF;AAAA,IACF,CAAC;AAED,QAAI,iBAAiB,MAAM;AACzB,YAAM,QAAQ,MAAM,0BAA0B;AAAA,QAC5C;AAAA,UACE,YAAY,SAAS;AAAA,UACrB,OAAO,aAAa;AAAA,UACpB,aAAa,aAAa;AAAA,UAC1B,UAAU,aAAa;AAAA,UACvB,eAAe,cAAc;AAAA,UAC7B,SAAS,aAAa;AAAA,UAEtB,kBAAkB,aAAa;AAAA,UAC/B,mBAAmB,OAAO,UAAU;AAAA,UACpC,yBAAyB,aAAa;AAAA,UACtC,0BAA0B,aAAa;AAAA,UACvC,mBAAmB,aAAa;AAAA,UAChC,oBAAoB,aAAa;AAAA,QACnC;AAAA,QACA;AAAA,MACF;AACA,aAAO,EAAE,UAAU,MAAM;AAAA,IAC3B;AAAA,EACF;AAEA,SAAO,EAAE,SAAS;AACpB;;;ACpRA,SAAS,UAAU,OAAyB;AAC1C,SAAO,UAAU,UAAa,UAAU,QAAQ,UAAU;AAC5D;AAEA,SAAS,gBAAgB,OAAuB;AAC9C,SAAO,MAAM,KAAK;AACpB;AAEA,SAAS,YAAY,UAAmB,QAA0B;AAChE,MAAI,OAAO,aAAa,YAAY,OAAO,WAAW,UAAU;AAC9D,WAAO,gBAAgB,QAAQ,MAAM,gBAAgB,MAAM;AAAA,EAC7D;AACA,SAAO,gBAAgB,QAAQ,MAAM,gBAAgB,MAAM;AAC7D;AAEO,IAAM,mBAAN,cAA+B,eAAe;AAAA,EACjC,OAAO;AAAA,EAEzB,MAAe,MAAM,QAanB;AACA,UAAM,YAAY,OAAO,aAAa,CAAC;AACvC,UAAM,eAAe,OAAO,QAAQ,OAAO,eAAe,EACvD,OAAO,CAAC,CAAC,EAAE,KAAK,MAAM,MAAM,aAAa,KAAK,EAC9C,IAAI,CAAC,CAAC,GAAG,MAAM,GAAG;AAErB,UAAM,cAAwB,CAAC;AAC/B,UAAM,cAAwB,CAAC;AAC/B,UAAM,iBAA2B,CAAC;AAElC,eAAW,OAAO,cAAc;AAC9B,YAAM,QAAQ,UAAU,GAAG;AAC3B,UAAI,UAAU,KAAK,GAAG;AACpB,oBAAY,KAAK,GAAG;AACpB;AAAA,MACF;AACA,kBAAY,KAAK,GAAG;AAEpB,YAAM,WAAW,OAAO,gBAAgB,GAAG,GAAG;AAC9C,UAAI,aAAa,UAAa,CAAC,YAAY,UAAU,KAAK,GAAG;AAC3D,uBAAe,KAAK,GAAG;AAAA,MACzB;AAAA,IACF;AAEA,UAAM,gBAAgB,aAAa;AACnC,UAAM,eAAe,gBAAgB,YAAY,SAAS,eAAe;AACzE,UAAM,QAAQ,kBAAkB,IAAI,IAAI,eAAe;AAEvD,WAAO;AAAA,MACL,OAAO,KAAK,IAAI,GAAG,KAAK,IAAI,GAAG,KAAK,CAAC;AAAA,MACrC,aACE,YAAY,WAAW,KAAK,eAAe,WAAW,IAClD,kCACA;AAAA,MACN;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA,UAAU;AAAA,QACR;AAAA,QACA,cAAc,YAAY;AAAA,QAC1B,cAAc,YAAY;AAAA,QAC1B,iBAAiB,eAAe;AAAA,MAClC;AAAA,IACF;AAAA,EACF;AACF;","names":["z","z","z","z","z","messages","runTestCase","z","z","z","runTestCase","formatMCQPrompt","messages","runTestCase","z","z","runTestCase"]}

package/dist/catalogs/index.d.ts

@@ -0,0 +1,75 @@
+import { C as ClassConstructor, A as AbstractFileLoader, a as AbstractHttpLoader } from '../utilities-BrRH32rD.js';
+import { A as AbstractProvider } from '../provider-BDjGp2y-.js';
+import { A as AbstractScorer } from '../abstract-Dec9Sc5O.js';
+import { A as AbstractDataLoader, a as AbstractLoader, L as LoaderResult, R as Runner, b as RunnerParams } from '../data-BmN5WjZ4.js';
+import { BaseResponseV1, BaseScoreV1 } from '../schemas/index.js';
+import '../index-WiPjF2AL.js';
+import 'zod';
+import 'node:fs';
+import '../system-prompt--0FdPWqK.js';
+
+declare class BaseCatalog<T> {
+    protected items: Record<string, T>;
+    register(kind: string, item: T): void;
+    deregister(kind: string): void;
+    get<K extends T>(kind: string, throwIfNotFound?: boolean): K;
+}
+
+type ClassCatalogItem<T extends ClassConstructor<unknown>> = {
+    constructor: T;
+    instantiate: (...args: any[]) => InstanceType<T>;
+};
+declare class ClassCatalog<T extends ClassConstructor<unknown>> extends BaseCatalog<ClassCatalogItem<T>> {
+    register(kind: string, item: ClassCatalogItem<T>): void;
+    instantiate(kind: string, ...args: any[]): InstanceType<T>;
+}
+
+declare class ProviderCatalog extends ClassCatalog<ClassConstructor<AbstractProvider>> {
+}
+declare class ProviderInstanceCatalog extends BaseCatalog<AbstractProvider> {
+}
+
+declare class ScorerCatalog extends ClassCatalog<ClassConstructor<AbstractScorer>> {
+}
+declare class ScorerInstanceCatalog extends BaseCatalog<AbstractScorer> {
+}
+
+declare class LoaderCatalog<T extends AbstractDataLoader | AbstractFileLoader | AbstractHttpLoader | AbstractLoader = AbstractLoader> extends BaseCatalog<T> {
+    register(kind: string, loader: T): void;
+    /**
+     * Executes all the registered loaders with the given params until one succeeds.
+     */
+    tryLoad(params: Parameters<T["loadData"]>[0]): Promise<LoaderResult<{
+        id: string;
+        kind: string;
+        schemaVersion: number;
+        metadata?: Record<string, unknown> | undefined;
+    }, {
+        id: string;
+        kind: string;
+        schemaVersion: number;
+        startedAt: number;
+        completedAt: number;
+        testCaseId: string;
+        metadata?: Record<string, unknown> | undefined;
+    }, {
+        id: string;
+        kind: string;
+        schemaVersion: number;
+        value: number;
+        responseId: string;
+        scoringMethod: "ai" | "human" | "algo";
+        explanation?: string | undefined;
+        metadata?: Record<string, unknown> | undefined;
+    }, {
+        kind: string;
+        schemaVersion: number;
+        metadata?: Record<string, unknown> | undefined;
+    }>>;
+}
+
+declare class RunnerCatalog<TRunner extends Runner = Runner> extends BaseCatalog<TRunner> {
+    register<TResponse extends BaseResponseV1 = BaseResponseV1, TScore extends BaseScoreV1 = BaseScoreV1, TParams extends RunnerParams = RunnerParams>(kind: string, runner: Runner<TResponse, TScore, TParams>): void;
+}
+
+export { BaseCatalog, ClassCatalog, type ClassCatalogItem, LoaderCatalog, ProviderCatalog, ProviderInstanceCatalog, RunnerCatalog, ScorerCatalog, ScorerInstanceCatalog };

package/dist/catalogs/index.js

@@ -0,0 +1,88 @@
+import {
+  CatalogItemNotFoundError
+} from "../chunk-VBOM2YEG.js";
+import "../chunk-PZ5AY32C.js";
+
+// src/catalogs/base.ts
+var BaseCatalog = class {
+  items = {};
+  register(kind, item) {
+    this.items[kind] = item;
+  }
+  deregister(kind) {
+    delete this.items[kind];
+  }
+  get(kind, throwIfNotFound = true) {
+    const item = this.items[kind];
+    if (throwIfNotFound && !item) {
+      throw new CatalogItemNotFoundError(kind);
+    }
+    return item;
+  }
+};
+
+// src/catalogs/class.ts
+var ClassCatalog = class extends BaseCatalog {
+  register(kind, item) {
+    super.register(kind, item);
+  }
+  instantiate(kind, ...args) {
+    const item = this.get(kind);
+    if (!item) {
+      throw new CatalogItemNotFoundError(kind);
+    }
+    return item.instantiate(...args);
+  }
+};
+
+// src/catalogs/provider.ts
+var ProviderCatalog = class extends ClassCatalog {
+};
+var ProviderInstanceCatalog = class extends BaseCatalog {
+};
+
+// src/catalogs/scorer.ts
+var ScorerCatalog = class extends ClassCatalog {
+};
+var ScorerInstanceCatalog = class extends BaseCatalog {
+};
+
+// src/catalogs/loader.ts
+var LoaderCatalog = class extends BaseCatalog {
+  register(kind, loader) {
+    super.register(kind, loader);
+  }
+  /**
+   * Executes all the registered loaders with the given params until one succeeds.
+   */
+  async tryLoad(params) {
+    for (const [, loader] of Object.entries(this.items)) {
+      try {
+        return await loader.loadData(params);
+      } catch {
+        continue;
+      }
+    }
+    throw new CatalogItemNotFoundError({
+      message: "No compatible loader found"
+    });
+  }
+};
+
+// src/catalogs/runner.ts
+var RunnerCatalog = class extends BaseCatalog {
+  register(kind, runner) {
+    super.register(kind, runner);
+  }
+};
+export {
+  BaseCatalog,
+  ClassCatalog,
+  LoaderCatalog,
+  ProviderCatalog,
+  ProviderInstanceCatalog,
+  RunnerCatalog,
+  ScorerCatalog,
+  ScorerInstanceCatalog
+};
+//# sourceMappingURL=index.js.map

package/dist/catalogs/index.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/catalogs/base.ts","../../src/catalogs/class.ts","../../src/catalogs/provider.ts","../../src/catalogs/scorer.ts","../../src/catalogs/loader.ts","../../src/catalogs/runner.ts"],"sourcesContent":["import { CatalogItemNotFoundError } from \"@/errors/catalog\";\n\nexport class BaseCatalog<T> {\n  protected items: Record<string, T> = {};\n\n  register(kind: string, item: T) {\n    this.items[kind] = item;\n  }\n\n  deregister(kind: string) {\n    delete this.items[kind];\n  }\n\n  get<K extends T>(kind: string, throwIfNotFound: boolean = true) {\n    const item = this.items[kind];\n\n    if (throwIfNotFound && !item) {\n      throw new CatalogItemNotFoundError(kind);\n    }\n\n    return item as K;\n  }\n}\n","import { CatalogItemNotFoundError } from \"@/errors/catalog\";\nimport { ClassConstructor } from \"@/utilities\";\nimport { BaseCatalog } from \"./base\";\n\nexport type ClassCatalogItem<T extends ClassConstructor<unknown>> = {\n  constructor: T;\n  instantiate: (...args: any[]) => InstanceType<T>;\n};\n\nexport class ClassCatalog<\n  T extends ClassConstructor<unknown>,\n> extends BaseCatalog<ClassCatalogItem<T>> {\n  override register(kind: string, item: ClassCatalogItem<T>) {\n    super.register(kind, item);\n  }\n\n  instantiate(kind: string, ...args: any[]) {\n    const item = this.get(kind);\n\n    if (!item) {\n      throw new CatalogItemNotFoundError(kind);\n    }\n\n    return item.instantiate(...args);\n  }\n}\n","import { ClassCatalog } from \"./class\";\nimport { ClassConstructor } from \"@/utilities\";\nimport { AbstractProvider } from \"@/providers/abstract/provider\";\nimport { BaseCatalog } from \"./base\";\n\nexport class ProviderCatalog extends ClassCatalog<\n  ClassConstructor<AbstractProvider>\n> {}\nexport class ProviderInstanceCatalog extends BaseCatalog<AbstractProvider> {}\n","import { ClassCatalog } from \"./class\";\nimport { ClassConstructor } from \"@/utilities\";\nimport { AbstractScorer } from \"@/scorers/abstract\";\nimport { BaseCatalog } from \"./base\";\n\nexport class ScorerCatalog extends ClassCatalog<\n  ClassConstructor<AbstractScorer>\n> {}\nexport class ScorerInstanceCatalog extends BaseCatalog<AbstractScorer> {}\n","import { CatalogItemNotFoundError } from \"@/errors/catalog\";\nimport { BaseCatalog } from \"./base\";\nimport {\n  AbstractDataLoader,\n  AbstractFileLoader,\n  AbstractHttpLoader,\n  AbstractLoader,\n} from \"@/loaders\";\n\nexport class LoaderCatalog<\n  T extends\n    | AbstractDataLoader\n    | AbstractFileLoader\n    | AbstractHttpLoader\n    | AbstractLoader = AbstractLoader,\n> extends BaseCatalog<T> {\n  override register(kind: string, loader: T) {\n    super.register(kind, loader);\n  }\n\n  /**\n   * Executes all the registered loaders with the given params until one succeeds.\n   */\n  async tryLoad(params: Parameters<T[\"loadData\"]>[0]) {\n    for (const [, loader] of Object.entries(this.items)) {\n      try {\n        return await loader.loadData(params as any);\n      } catch {\n        continue;\n      }\n    }\n\n    throw new CatalogItemNotFoundError({\n      message: \"No compatible loader found\",\n    });\n  }\n}\n","import { Runner, RunnerParams } from \"@/types\";\nimport { BaseCatalog } from \"./base\";\nimport { BaseResponseV1, BaseScoreV1 } from \"@/schemas\";\n\nexport class RunnerCatalog<\n  TRunner extends Runner = Runner,\n> extends BaseCatalog<TRunner> {\n  override register<\n    TResponse extends BaseResponseV1 = BaseResponseV1,\n    TScore extends BaseScoreV1 = BaseScoreV1,\n    TParams extends RunnerParams = RunnerParams,\n  >(kind: string, runner: Runner<TResponse, TScore, TParams>) {\n    super.register(kind, runner as unknown as TRunner);\n  }\n}\n"],"mappings":";;;;;;AAEO,IAAM,cAAN,MAAqB;AAAA,EAChB,QAA2B,CAAC;AAAA,EAEtC,SAAS,MAAc,MAAS;AAC9B,SAAK,MAAM,IAAI,IAAI;AAAA,EACrB;AAAA,EAEA,WAAW,MAAc;AACvB,WAAO,KAAK,MAAM,IAAI;AAAA,EACxB;AAAA,EAEA,IAAiB,MAAc,kBAA2B,MAAM;AAC9D,UAAM,OAAO,KAAK,MAAM,IAAI;AAE5B,QAAI,mBAAmB,CAAC,MAAM;AAC5B,YAAM,IAAI,yBAAyB,IAAI;AAAA,IACzC;AAEA,WAAO;AAAA,EACT;AACF;;;ACbO,IAAM,eAAN,cAEG,YAAiC;AAAA,EAChC,SAAS,MAAc,MAA2B;AACzD,UAAM,SAAS,MAAM,IAAI;AAAA,EAC3B;AAAA,EAEA,YAAY,SAAiB,MAAa;AACxC,UAAM,OAAO,KAAK,IAAI,IAAI;AAE1B,QAAI,CAAC,MAAM;AACT,YAAM,IAAI,yBAAyB,IAAI;AAAA,IACzC;AAEA,WAAO,KAAK,YAAY,GAAG,IAAI;AAAA,EACjC;AACF;;;ACpBO,IAAM,kBAAN,cAA8B,aAEnC;AAAC;AACI,IAAM,0BAAN,cAAsC,YAA8B;AAAC;;;ACHrE,IAAM,gBAAN,cAA4B,aAEjC;AAAC;AACI,IAAM,wBAAN,cAAoC,YAA4B;AAAC;;;ACCjE,IAAM,gBAAN,cAMG,YAAe;AAAA,EACd,SAAS,MAAc,QAAW;AACzC,UAAM,SAAS,MAAM,MAAM;AAAA,EAC7B;AAAA;AAAA;AAAA;AAAA,EAKA,MAAM,QAAQ,QAAsC;AAClD,eAAW,CAAC,EAAE,MAAM,KAAK,OAAO,QAAQ,KAAK,KAAK,GAAG;AACnD,UAAI;AACF,eAAO,MAAM,OAAO,SAAS,MAAa;AAAA,MAC5C,QAAQ;AACN;AAAA,MACF;AAAA,IACF;AAEA,UAAM,IAAI,yBAAyB;AAAA,MACjC,SAAS;AAAA,IACX,CAAC;AAAA,EACH;AACF;;;AChCO,IAAM,gBAAN,cAEG,YAAqB;AAAA,EACpB,SAIP,MAAc,QAA4C;AAC1D,UAAM,SAAS,MAAM,MAA4B;AAAA,EACnD;AACF;","names":[]}

package/dist/chunk-22HU24QF.js

@@ -0,0 +1,8 @@
+// src/scorers/abstract.ts
+var AbstractScorer = class {
+};
+
+export {
+  AbstractScorer
+};
+//# sourceMappingURL=chunk-22HU24QF.js.map

package/dist/chunk-22HU24QF.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/scorers/abstract.ts"],"sourcesContent":["export abstract class AbstractScorer {\n  abstract readonly kind: string;\n\n  abstract score(params: any): Promise<BaseScorerResult | null>;\n}\n\nexport type BaseScorerResult = {\n  value: number;\n  explanation?: string;\n  metadata?: Record<string, unknown>;\n  [key: string]: unknown;\n};\n"],"mappings":";AAAO,IAAe,iBAAf,MAA8B;AAIrC;","names":[]}

package/dist/chunk-232PY7K3.js

@@ -0,0 +1,50 @@
+import {
+  ScoringMethod
+} from "./chunk-ZJWSK4VO.js";
+import {
+  IdSchema,
+  buildSchemaDefiner
+} from "./chunk-IUN2IUCS.js";
+
+// src/schemas/benchmark-spec.ts
+import z from "zod";
+var BaseBenchmarkSpecSchemaV1 = z.object({
+  kind: z.string(),
+  schemaVersion: z.number(),
+  metadata: z.record(z.string(), z.unknown()).optional()
+});
+var defineBenchmarkSpecSchema = buildSchemaDefiner();
+
+// src/schemas/test-case.ts
+import { z as z2 } from "zod";
+var BaseTestCaseSchemaV1 = z2.object({
+  id: IdSchema,
+  kind: z2.string(),
+  schemaVersion: z2.number(),
+  metadata: z2.record(z2.string(), z2.unknown()).optional()
+});
+var defineTestCaseSchema = buildSchemaDefiner();
+
+// src/schemas/score.ts
+import z3 from "zod";
+var BaseScoreSchemaV1 = z3.object({
+  id: IdSchema,
+  kind: z3.string(),
+  schemaVersion: z3.number(),
+  value: z3.number(),
+  responseId: IdSchema,
+  explanation: z3.string().optional(),
+  metadata: z3.record(z3.string(), z3.unknown()).optional(),
+  scoringMethod: z3.enum(ScoringMethod)
+});
+var defineScoreSchema = buildSchemaDefiner();
+
+export {
+  BaseBenchmarkSpecSchemaV1,
+  defineBenchmarkSpecSchema,
+  BaseTestCaseSchemaV1,
+  defineTestCaseSchema,
+  BaseScoreSchemaV1,
+  defineScoreSchema
+};
+//# sourceMappingURL=chunk-232PY7K3.js.map

package/dist/chunk-232PY7K3.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/schemas/benchmark-spec.ts","../src/schemas/test-case.ts","../src/schemas/score.ts"],"sourcesContent":["import z from \"zod\";\nimport { buildSchemaDefiner } from \"./schema-definer\";\n\nexport const BaseBenchmarkSpecSchemaV1 = z.object({\n  kind: z.string(),\n  schemaVersion: z.number(),\n  metadata: z.record(z.string(), z.unknown()).optional(),\n});\nexport type BaseBenchmarkSpecV1 = z.infer<typeof BaseBenchmarkSpecSchemaV1>;\n\nexport const defineBenchmarkSpecSchema =\n  buildSchemaDefiner<typeof BaseBenchmarkSpecSchemaV1.shape>();\n","import { IdSchema } from \"./id\";\nimport { z } from \"zod\";\nimport { buildSchemaDefiner } from \"./schema-definer\";\n\nexport const BaseTestCaseSchemaV1 = z.object({\n  id: IdSchema,\n  kind: z.string(),\n  schemaVersion: z.number(),\n  metadata: z.record(z.string(), z.unknown()).optional(),\n});\nexport type BaseTestCaseV1 = z.infer<typeof BaseTestCaseSchemaV1>;\n\nexport const defineTestCaseSchema =\n  buildSchemaDefiner<typeof BaseTestCaseSchemaV1.shape>();\n","import { IdSchema } from \"./id\";\nimport { ScoringMethod } from \"@/types\";\nimport { buildSchemaDefiner } from \"./schema-definer\";\nimport z from \"zod\";\n\nexport const BaseScoreSchemaV1 = z.object({\n  id: IdSchema,\n  kind: z.string(),\n  schemaVersion: z.number(),\n\n  value: z.number(),\n  responseId: IdSchema,\n  explanation: z.string().optional(),\n  metadata: z.record(z.string(), z.unknown()).optional(),\n  scoringMethod: z.enum(ScoringMethod),\n});\nexport type BaseScoreV1 = z.infer<typeof BaseScoreSchemaV1>;\n\nexport const defineScoreSchema =\n  buildSchemaDefiner<typeof BaseScoreSchemaV1.shape>();\n"],"mappings":";;;;;;;;;AAAA,OAAO,OAAO;AAGP,IAAM,4BAA4B,EAAE,OAAO;AAAA,EAChD,MAAM,EAAE,OAAO;AAAA,EACf,eAAe,EAAE,OAAO;AAAA,EACxB,UAAU,EAAE,OAAO,EAAE,OAAO,GAAG,EAAE,QAAQ,CAAC,EAAE,SAAS;AACvD,CAAC;AAGM,IAAM,4BACX,mBAA2D;;;ACV7D,SAAS,KAAAA,UAAS;AAGX,IAAM,uBAAuBC,GAAE,OAAO;AAAA,EAC3C,IAAI;AAAA,EACJ,MAAMA,GAAE,OAAO;AAAA,EACf,eAAeA,GAAE,OAAO;AAAA,EACxB,UAAUA,GAAE,OAAOA,GAAE,OAAO,GAAGA,GAAE,QAAQ,CAAC,EAAE,SAAS;AACvD,CAAC;AAGM,IAAM,uBACX,mBAAsD;;;ACVxD,OAAOC,QAAO;AAEP,IAAM,oBAAoBA,GAAE,OAAO;AAAA,EACxC,IAAI;AAAA,EACJ,MAAMA,GAAE,OAAO;AAAA,EACf,eAAeA,GAAE,OAAO;AAAA,EAExB,OAAOA,GAAE,OAAO;AAAA,EAChB,YAAY;AAAA,EACZ,aAAaA,GAAE,OAAO,EAAE,SAAS;AAAA,EACjC,UAAUA,GAAE,OAAOA,GAAE,OAAO,GAAGA,GAAE,QAAQ,CAAC,EAAE,SAAS;AAAA,EACrD,eAAeA,GAAE,KAAK,aAAa;AACrC,CAAC;AAGM,IAAM,oBACX,mBAAmD;","names":["z","z","z"]}

package/dist/chunk-7TREBPSJ.js

@@ -0,0 +1,26 @@
+import {
+  BaseResponseSchemaV1,
+  IdSchema,
+  defineResponseSchema
+} from "./chunk-IUN2IUCS.js";
+
+// src/schemas/llm/response.ts
+import z from "zod";
+var BaseLLMChatResponseSchemaV1 = defineResponseSchema({
+  baseSchema: BaseResponseSchemaV1,
+  fields: {
+    data: z.string(),
+    modelSlug: z.string(),
+    provider: z.string(),
+    systemPromptId: IdSchema.optional(),
+    inputTokensUsed: z.number().optional(),
+    outputTokensUsed: z.number().optional(),
+    inputCost: z.string().optional(),
+    outputCost: z.string().optional()
+  }
+});
+
+export {
+  BaseLLMChatResponseSchemaV1
+};
+//# sourceMappingURL=chunk-7TREBPSJ.js.map

package/dist/chunk-7TREBPSJ.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/schemas/llm/response.ts"],"sourcesContent":["import { IdSchema } from \"../id\";\nimport { BaseResponseSchemaV1, defineResponseSchema } from \"../response\";\nimport z from \"zod\";\n\n/**\n * Base response schema specialized for LLM chat responses.\n */\nexport const BaseLLMChatResponseSchemaV1 = defineResponseSchema({\n  baseSchema: BaseResponseSchemaV1,\n  fields: {\n    data: z.string(),\n    modelSlug: z.string(),\n    provider: z.string(),\n    systemPromptId: IdSchema.optional(),\n\n    inputTokensUsed: z.number().optional(),\n    outputTokensUsed: z.number().optional(),\n    inputCost: z.string().optional(),\n    outputCost: z.string().optional(),\n  },\n});\n"],"mappings":";;;;;;;AAEA,OAAO,OAAO;AAKP,IAAM,8BAA8B,qBAAqB;AAAA,EAC9D,YAAY;AAAA,EACZ,QAAQ;AAAA,IACN,MAAM,EAAE,OAAO;AAAA,IACf,WAAW,EAAE,OAAO;AAAA,IACpB,UAAU,EAAE,OAAO;AAAA,IACnB,gBAAgB,SAAS,SAAS;AAAA,IAElC,iBAAiB,EAAE,OAAO,EAAE,SAAS;AAAA,IACrC,kBAAkB,EAAE,OAAO,EAAE,SAAS;AAAA,IACtC,WAAW,EAAE,OAAO,EAAE,SAAS;AAAA,IAC/B,YAAY,EAAE,OAAO,EAAE,SAAS;AAAA,EAClC;AACF,CAAC;","names":[]}

package/dist/chunk-DUBKY73H.js

@@ -0,0 +1,128 @@
+// src/utils/string.ts
+function bufferToString(buffer) {
+  const decoder = new TextDecoder();
+  return decoder.decode(buffer);
+}
+function stringToBuffer(str) {
+  const encoder = new TextEncoder();
+  return encoder.encode(str);
+}
+
+// src/utils/sleep.ts
+async function sleep(ms, signal) {
+  return new Promise((resolve, reject) => {
+    if (signal?.aborted) {
+      return reject(new Error("Aborted"));
+    }
+    const timeout = setTimeout(() => {
+      cleanup();
+      resolve();
+    }, ms);
+    const onAbort = () => {
+      cleanup();
+      reject(new Error("Aborted"));
+    };
+    const cleanup = () => {
+      clearTimeout(timeout);
+      signal?.removeEventListener("abort", onAbort);
+    };
+    signal?.addEventListener("abort", onAbort);
+  });
+}
+
+// src/utils/llm.ts
+import { jsonrepair } from "jsonrepair";
+function parseResponseAsJSON(response) {
+  try {
+    return JSON.parse(jsonrepair(response));
+  } catch (e) {
+    if (process?.env?.PB_SDK_DEBUG) {
+      console.log("Original response", JSON.stringify(response));
+      console.error("Error parsing response as JSON", e);
+    }
+  }
+}
+
+// src/utils/rate-limiter.ts
+var RateLimiter = class {
+  maxWeight;
+  timeWindow;
+  timestamps = [];
+  constructor(options = {}) {
+    this.maxWeight = options.maxWeight ?? 25;
+    this.timeWindow = options.timeWindow ?? 1e3;
+  }
+  /**
+   * Checks if rate limiting is disabled
+   */
+  isDisabled() {
+    return this.maxWeight <= 0;
+  }
+  /**
+   * Disables rate limiting. Set `maxWeight` to re-enable it.
+   */
+  disable() {
+    this.maxWeight = 0;
+  }
+  /**
+   * Returns how many weight of calls are there in the current time window
+   */
+  getCurrentCalls() {
+    const now = Date.now();
+    this.timestamps = this.timestamps.filter(
+      (ts) => now - ts < this.timeWindow
+    );
+    return this.timestamps.length;
+  }
+  /**
+   * Executes the given function with rate limiting applied
+   */
+  async execute(func, { weight = 1, signal } = {}) {
+    if (this.maxWeight <= 0) {
+      return await func();
+    }
+    await this.waitForRateLimit(weight, signal);
+    return await func();
+  }
+  /**
+   * Waits until rate limit allows the specified weight of calls
+   */
+  async waitForRateLimit(weight, signal) {
+    const now = Date.now();
+    this.timestamps = this.timestamps.filter(
+      (ts) => now - ts < this.timeWindow
+    );
+    if (this.timestamps.length + weight <= this.maxWeight) {
+      for (let i = 0; i < weight; i++) {
+        this.timestamps.push(now);
+      }
+      return;
+    }
+    const earliest = this.timestamps[0];
+    const waitTime = this.timeWindow - (now - (earliest ?? 0));
+    await sleep(waitTime, signal);
+    return this.waitForRateLimit(weight, signal);
+  }
+  /**
+   * Resets the rate limited weight of calls
+   */
+  reset() {
+    this.timestamps = [];
+  }
+};
+
+// src/utils/id-generator.ts
+import { v7 as uuidv7 } from "uuid";
+var idGeneratorUUIDv7 = () => {
+  return uuidv7();
+};
+
+export {
+  bufferToString,
+  stringToBuffer,
+  sleep,
+  parseResponseAsJSON,
+  RateLimiter,
+  idGeneratorUUIDv7
+};
+//# sourceMappingURL=chunk-DUBKY73H.js.map

package/dist/chunk-DUBKY73H.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/utils/string.ts","../src/utils/sleep.ts","../src/utils/llm.ts","../src/utils/rate-limiter.ts","../src/utils/id-generator.ts"],"sourcesContent":["/**\n * Converts the given byte array to a string\n */\nexport function bufferToString(buffer: Uint8Array): string {\n  const decoder = new TextDecoder();\n  return decoder.decode(buffer);\n}\n\n/**\n * Converts the given string to a byte array\n */\nexport function stringToBuffer(str: string): Uint8Array {\n  const encoder = new TextEncoder();\n  return encoder.encode(str);\n}\n","export async function sleep(ms: number, signal?: AbortSignal) {\n  return new Promise<void>((resolve, reject) => {\n    if (signal?.aborted) {\n      return reject(new Error(\"Aborted\"));\n    }\n\n    const timeout = setTimeout(() => {\n      cleanup();\n      resolve();\n    }, ms);\n\n    const onAbort = () => {\n      cleanup();\n      reject(new Error(\"Aborted\"));\n    };\n\n    const cleanup = () => {\n      clearTimeout(timeout);\n      signal?.removeEventListener(\"abort\", onAbort);\n    };\n\n    signal?.addEventListener(\"abort\", onAbort);\n  });\n}\n","import { jsonrepair } from \"jsonrepair\";\n\n/**\n * Tries to repair and parse LLM response as a JSON object. LLM must\n * be configured to return a JSON object. This function only helps to\n * get rid out of some additional formatting (e.g. ```json) and repair\n * the JSON syntax (e.g missing comma, single quotes instead double).\n */\nexport function parseResponseAsJSON<T>(response: string) {\n  try {\n    return JSON.parse(jsonrepair(response)) as T;\n  } catch (e) {\n    if (process?.env?.PB_SDK_DEBUG) {\n      console.log(\"Original response\", JSON.stringify(response));\n      console.error(\"Error parsing response as JSON\", e);\n    }\n  }\n}\n","import { sleep } from \"./sleep\";\n\nexport interface RateLimiterOptions {\n  /**\n   * Maximum weight of calls allowed within a time window. Each call has a weight of 1 by default.\n   * Rate limiting is disabled when this is 0 or negative.\n   * @default 25\n   */\n  maxWeight?: number;\n\n  /**\n   * Time window in milliseconds for rate limiting\n   * @default 1000\n   */\n  timeWindow?: number;\n}\n\nexport interface RateLimiterCallOptions {\n  /**\n   * Weight of the call. Rate limiting is applied based on the total weight of the calls.\n   * @default 1\n   */\n  weight?: number;\n\n  /**\n   * Abort signal for cancellation\n   */\n  signal?: AbortSignal;\n}\n\n/**\n * Generic rate limiter. It can be used to limit async function calls\n * by a given number of calls within a time window.\n */\nexport class RateLimiter {\n  maxWeight: number;\n  timeWindow: number;\n\n  private timestamps: number[] = [];\n\n  constructor(options: RateLimiterOptions = {}) {\n    this.maxWeight = options.maxWeight ?? 25;\n    this.timeWindow = options.timeWindow ?? 1000;\n  }\n\n  /**\n   * Checks if rate limiting is disabled\n   */\n  isDisabled(): boolean {\n    return this.maxWeight <= 0;\n  }\n\n  /**\n   * Disables rate limiting. Set `maxWeight` to re-enable it.\n   */\n  disable() {\n    this.maxWeight = 0;\n  }\n\n  /**\n   * Returns how many weight of calls are there in the current time window\n   */\n  getCurrentCalls(): number {\n    const now = Date.now();\n    this.timestamps = this.timestamps.filter(\n      (ts) => now - ts < this.timeWindow\n    );\n    return this.timestamps.length;\n  }\n\n  /**\n   * Executes the given function with rate limiting applied\n   */\n  async execute<T = unknown>(\n    func: () => Promise<T>,\n    { weight = 1, signal }: RateLimiterCallOptions = {}\n  ): Promise<T> {\n    // If maxWeight is zero/negative, execute immediately (no rate limiting)\n    if (this.maxWeight <= 0) {\n      return await func();\n    }\n\n    await this.waitForRateLimit(weight, signal);\n    return await func();\n  }\n\n  /**\n   * Waits until rate limit allows the specified weight of calls\n   */\n  private async waitForRateLimit(\n    weight: number,\n    signal?: AbortSignal\n  ): Promise<void> {\n    const now = Date.now();\n\n    // Clear timestamps from the old time period\n    this.timestamps = this.timestamps.filter(\n      (ts) => now - ts < this.timeWindow\n    );\n\n    // Check if there are empty slots for the requests\n    if (this.timestamps.length + weight <= this.maxWeight) {\n      for (let i = 0; i < weight; i++) {\n        this.timestamps.push(now);\n      }\n      return;\n    }\n\n    // Calculate how much time to wait for the next time window\n    const earliest = this.timestamps[0];\n    const waitTime = this.timeWindow - (now - (earliest ?? 0));\n\n    await sleep(waitTime, signal);\n\n    // Retry after waiting\n    return this.waitForRateLimit(weight, signal);\n  }\n\n  /**\n   * Resets the rate limited weight of calls\n   */\n  reset(): void {\n    this.timestamps = [];\n  }\n}\n","import { IdGenerator } from \"@/types\";\nimport { v7 as uuidv7 } from \"uuid\";\n\nexport const idGeneratorUUIDv7: IdGenerator = () => {\n  return uuidv7();\n};\n"],"mappings":";AAGO,SAAS,eAAe,QAA4B;AACzD,QAAM,UAAU,IAAI,YAAY;AAChC,SAAO,QAAQ,OAAO,MAAM;AAC9B;AAKO,SAAS,eAAe,KAAyB;AACtD,QAAM,UAAU,IAAI,YAAY;AAChC,SAAO,QAAQ,OAAO,GAAG;AAC3B;;;ACdA,eAAsB,MAAM,IAAY,QAAsB;AAC5D,SAAO,IAAI,QAAc,CAAC,SAAS,WAAW;AAC5C,QAAI,QAAQ,SAAS;AACnB,aAAO,OAAO,IAAI,MAAM,SAAS,CAAC;AAAA,IACpC;AAEA,UAAM,UAAU,WAAW,MAAM;AAC/B,cAAQ;AACR,cAAQ;AAAA,IACV,GAAG,EAAE;AAEL,UAAM,UAAU,MAAM;AACpB,cAAQ;AACR,aAAO,IAAI,MAAM,SAAS,CAAC;AAAA,IAC7B;AAEA,UAAM,UAAU,MAAM;AACpB,mBAAa,OAAO;AACpB,cAAQ,oBAAoB,SAAS,OAAO;AAAA,IAC9C;AAEA,YAAQ,iBAAiB,SAAS,OAAO;AAAA,EAC3C,CAAC;AACH;;;ACvBA,SAAS,kBAAkB;AAQpB,SAAS,oBAAuB,UAAkB;AACvD,MAAI;AACF,WAAO,KAAK,MAAM,WAAW,QAAQ,CAAC;AAAA,EACxC,SAAS,GAAG;AACV,QAAI,SAAS,KAAK,cAAc;AAC9B,cAAQ,IAAI,qBAAqB,KAAK,UAAU,QAAQ,CAAC;AACzD,cAAQ,MAAM,kCAAkC,CAAC;AAAA,IACnD;AAAA,EACF;AACF;;;ACiBO,IAAM,cAAN,MAAkB;AAAA,EACvB;AAAA,EACA;AAAA,EAEQ,aAAuB,CAAC;AAAA,EAEhC,YAAY,UAA8B,CAAC,GAAG;AAC5C,SAAK,YAAY,QAAQ,aAAa;AACtC,SAAK,aAAa,QAAQ,cAAc;AAAA,EAC1C;AAAA;AAAA;AAAA;AAAA,EAKA,aAAsB;AACpB,WAAO,KAAK,aAAa;AAAA,EAC3B;AAAA;AAAA;AAAA;AAAA,EAKA,UAAU;AACR,SAAK,YAAY;AAAA,EACnB;AAAA;AAAA;AAAA;AAAA,EAKA,kBAA0B;AACxB,UAAM,MAAM,KAAK,IAAI;AACrB,SAAK,aAAa,KAAK,WAAW;AAAA,MAChC,CAAC,OAAO,MAAM,KAAK,KAAK;AAAA,IAC1B;AACA,WAAO,KAAK,WAAW;AAAA,EACzB;AAAA;AAAA;AAAA;AAAA,EAKA,MAAM,QACJ,MACA,EAAE,SAAS,GAAG,OAAO,IAA4B,CAAC,GACtC;AAEZ,QAAI,KAAK,aAAa,GAAG;AACvB,aAAO,MAAM,KAAK;AAAA,IACpB;AAEA,UAAM,KAAK,iBAAiB,QAAQ,MAAM;AAC1C,WAAO,MAAM,KAAK;AAAA,EACpB;AAAA;AAAA;AAAA;AAAA,EAKA,MAAc,iBACZ,QACA,QACe;AACf,UAAM,MAAM,KAAK,IAAI;AAGrB,SAAK,aAAa,KAAK,WAAW;AAAA,MAChC,CAAC,OAAO,MAAM,KAAK,KAAK;AAAA,IAC1B;AAGA,QAAI,KAAK,WAAW,SAAS,UAAU,KAAK,WAAW;AACrD,eAAS,IAAI,GAAG,IAAI,QAAQ,KAAK;AAC/B,aAAK,WAAW,KAAK,GAAG;AAAA,MAC1B;AACA;AAAA,IACF;AAGA,UAAM,WAAW,KAAK,WAAW,CAAC;AAClC,UAAM,WAAW,KAAK,cAAc,OAAO,YAAY;AAEvD,UAAM,MAAM,UAAU,MAAM;AAG5B,WAAO,KAAK,iBAAiB,QAAQ,MAAM;AAAA,EAC7C;AAAA;AAAA;AAAA;AAAA,EAKA,QAAc;AACZ,SAAK,aAAa,CAAC;AAAA,EACrB;AACF;;;AC3HA,SAAS,MAAM,cAAc;AAEtB,IAAM,oBAAiC,MAAM;AAClD,SAAO,OAAO;AAChB;","names":[]}

package/dist/chunk-GVF4YZF3.js

@@ -0,0 +1,15 @@
+// src/schemas/extensions/score/llm-as-a-judge-scorer.ts
+import z from "zod";
+var ExtensionLLMAsAJudgeScorerFieldsV1 = {
+  scorerAIProvider: z.string().optional(),
+  scorerAIModelSlug: z.string().optional(),
+  scorerAIInputTokensUsed: z.number().optional(),
+  scorerAIOutputTokensUsed: z.number().optional(),
+  scorerAIInputCost: z.string().optional(),
+  scorerAIOutputCost: z.string().optional()
+};
+
+export {
+  ExtensionLLMAsAJudgeScorerFieldsV1
+};
+//# sourceMappingURL=chunk-GVF4YZF3.js.map

package/dist/chunk-GVF4YZF3.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/schemas/extensions/score/llm-as-a-judge-scorer.ts"],"sourcesContent":["import z from \"zod\";\n\n/**\n * Provides a set of fields that holds information about the LLM model\n * that was used to judge the response.\n */\nexport const ExtensionLLMAsAJudgeScorerFieldsV1 = {\n  scorerAIProvider: z.string().optional(),\n  scorerAIModelSlug: z.string().optional(),\n  scorerAIInputTokensUsed: z.number().optional(),\n  scorerAIOutputTokensUsed: z.number().optional(),\n  scorerAIInputCost: z.string().optional(),\n  scorerAIOutputCost: z.string().optional(),\n};\n"],"mappings":";AAAA,OAAO,OAAO;AAMP,IAAM,qCAAqC;AAAA,EAChD,kBAAkB,EAAE,OAAO,EAAE,SAAS;AAAA,EACtC,mBAAmB,EAAE,OAAO,EAAE,SAAS;AAAA,EACvC,yBAAyB,EAAE,OAAO,EAAE,SAAS;AAAA,EAC7C,0BAA0B,EAAE,OAAO,EAAE,SAAS;AAAA,EAC9C,mBAAmB,EAAE,OAAO,EAAE,SAAS;AAAA,EACvC,oBAAoB,EAAE,OAAO,EAAE,SAAS;AAC1C;","names":[]}