peerbench 0.0.1 → 0.0.2-alpha-dev.0

package/dist/benchmarks/index.js

@@ -0,0 +1,458 @@
+import {
+  SimpleSystemPromptSchemaV1
+} from "../chunk-Q6GSOHOP.js";
+import {
+  defineRunner
+} from "../chunk-QY5MPNNB.js";
+import {
+  BaseResponseSchemaV1,
+  BaseScoreSchemaV1,
+  BaseTestCaseSchemaV1,
+  defineResponseSchema,
+  defineScoreSchema,
+  defineTestCaseSchema
+} from "../chunk-YY33MNMV.js";
+import "../chunk-OQE6TQXZ.js";
+import {
+  ScoringMethod
+} from "../chunk-HMQYGCKI.js";
+import {
+  JSONFileStorage
+} from "../chunk-ERALDEZY.js";
+import {
+  LLMAsAJudgeScorer,
+  MCQScorer
+} from "../chunk-ZEWI24CV.js";
+import {
+  AbstractLLMProvider
+} from "../chunk-R76XA2K6.js";
+import {
+  PEERBENCH_NAMESPACE
+} from "../chunk-UHHHSYVE.js";
+import {
+  idGeneratorUUIDv7
+} from "../chunk-4UBK6452.js";
+import {
+  ExtensionLLMAsAJudgeScoreFieldsV1,
+  ExtensionLLMResponseFieldsV1
+} from "../chunk-TRNCF2BG.js";
+import "../chunk-NUEOE3K5.js";
+import {
+  __export
+} from "../chunk-PZ5AY32C.js";
+
+// src/benchmarks/peerbench/index.ts
+var peerbench_exports = {};
+__export(peerbench_exports, {
+  MCQKind: () => MCQKind,
+  MCQResponseSchemaV1: () => MCQResponseSchemaV1,
+  MCQScoreSchemaV1: () => MCQScoreSchemaV1,
+  MCQTestCaseSchemaV1: () => MCQTestCaseSchemaV1,
+  MultiTurnKind: () => MultiTurnKind,
+  MultiTurnResponseSchemaV1: () => MultiTurnResponseSchemaV1,
+  MultiTurnScoreSchemaV1: () => MultiTurnScoreSchemaV1,
+  MultiTurnTestCaseSchemaV1: () => MultiTurnTestCaseSchemaV1,
+  PeerbenchJSONStorage: () => PeerbenchJSONStorage,
+  QAKind: () => QAKind,
+  QAResponseSchemaV1: () => QAResponseSchemaV1,
+  QAScoreSchemaV1: () => QAScoreSchemaV1,
+  QATestCaseSchemaV1: () => QATestCaseSchemaV1,
+  peerbenchRunner: () => peerbenchRunner
+});
+
+// src/benchmarks/peerbench/schema-sets/mcq.v1.ts
+import { z } from "zod";
+var MCQKind = `llm/mcq`;
+var MCQTestCaseSchemaV1 = defineTestCaseSchema({
+  baseSchema: BaseTestCaseSchemaV1,
+  namespace: PEERBENCH_NAMESPACE,
+  kind: MCQKind,
+  schemaVersion: 1,
+  fields: {
+    question: z.string(),
+    options: z.record(z.string(), z.string()),
+    correctAnswerKeys: z.string().array()
+  }
+});
+var MCQResponseSchemaV1 = defineResponseSchema({
+  baseSchema: BaseResponseSchemaV1,
+  namespace: PEERBENCH_NAMESPACE,
+  kind: MCQKind,
+  schemaVersion: 1,
+  fields: {
+    ...ExtensionLLMResponseFieldsV1
+  }
+});
+var MCQScoreSchemaV1 = defineScoreSchema({
+  baseSchema: BaseScoreSchemaV1,
+  namespace: PEERBENCH_NAMESPACE,
+  kind: MCQKind,
+  schemaVersion: 1,
+  fields: {
+    ...ExtensionLLMAsAJudgeScoreFieldsV1,
+    extractedAnswers: z.array(z.string())
+  }
+});
+
+// src/benchmarks/peerbench/schema-sets/multi-turn.v1.ts
+import { z as z2 } from "zod";
+var MultiTurnKind = `llm/multi-turn`;
+var MultiTurnTestCaseSchemaV1 = defineTestCaseSchema({
+  baseSchema: BaseTestCaseSchemaV1,
+  namespace: PEERBENCH_NAMESPACE,
+  kind: MultiTurnKind,
+  schemaVersion: 1,
+  fields: {
+    messages: z2.object({
+      role: z2.string(),
+      content: z2.string(),
+      goodAnswers: z2.string().array().optional(),
+      badAnswers: z2.string().array().optional()
+    }).array(),
+    maxTurns: z2.number().optional(),
+    expectedOutcome: z2.string().optional()
+  }
+});
+var MultiTurnResponseSchemaV1 = defineResponseSchema({
+  baseSchema: BaseResponseSchemaV1,
+  namespace: PEERBENCH_NAMESPACE,
+  kind: MultiTurnKind,
+  schemaVersion: 1,
+  fields: {
+    ...ExtensionLLMResponseFieldsV1,
+    replies: z2.object({
+      messageIndex: z2.number(),
+      startedAt: z2.number(),
+      completedAt: z2.number(),
+      data: z2.string(),
+      inputTokensUsed: z2.number().optional(),
+      outputTokensUsed: z2.number().optional(),
+      inputCost: z2.string().optional(),
+      outputCost: z2.string().optional()
+    }).array()
+  }
+});
+var MultiTurnScoreSchemaV1 = defineScoreSchema({
+  baseSchema: BaseScoreSchemaV1,
+  namespace: PEERBENCH_NAMESPACE,
+  kind: MultiTurnKind,
+  schemaVersion: 1,
+  fields: {
+    ...ExtensionLLMAsAJudgeScoreFieldsV1,
+    individualScores: z2.object({
+      replyIndex: z2.number(),
+      value: z2.number()
+    }).array()
+  }
+});
+
+// src/benchmarks/peerbench/schema-sets/qa.v1.ts
+import { z as z3 } from "zod";
+var QAKind = `llm/qa`;
+var QATestCaseSchemaV1 = defineTestCaseSchema({
+  baseSchema: BaseTestCaseSchemaV1,
+  namespace: PEERBENCH_NAMESPACE,
+  kind: QAKind,
+  schemaVersion: 1,
+  fields: {
+    question: z3.string(),
+    goodAnswers: z3.string().array(),
+    badAnswers: z3.string().array()
+  }
+});
+var QAResponseSchemaV1 = defineResponseSchema({
+  baseSchema: BaseResponseSchemaV1,
+  namespace: PEERBENCH_NAMESPACE,
+  kind: QAKind,
+  schemaVersion: 1,
+  fields: {
+    ...ExtensionLLMResponseFieldsV1
+  }
+});
+var QAScoreSchemaV1 = defineScoreSchema({
+  baseSchema: BaseScoreSchemaV1,
+  namespace: PEERBENCH_NAMESPACE,
+  kind: QAKind,
+  schemaVersion: 1,
+  fields: {
+    ...ExtensionLLMAsAJudgeScoreFieldsV1
+  }
+});
+
+// src/benchmarks/peerbench/runner.ts
+import Handlebars from "handlebars";
+import z4 from "zod";
+var peerbenchRunner = defineRunner(
+  {
+    schemaSets: [
+      {
+        testCase: MCQTestCaseSchemaV1,
+        response: MCQResponseSchemaV1,
+        score: MCQScoreSchemaV1
+      },
+      {
+        testCase: QATestCaseSchemaV1,
+        response: QAResponseSchemaV1,
+        score: QAScoreSchemaV1
+      }
+    ],
+    providers: [AbstractLLMProvider],
+    scorers: [LLMAsAJudgeScorer, MCQScorer],
+    runConfigSchema: {
+      model: z4.string(),
+      llmJudgeModel: z4.string().optional(),
+      llmJudgeSystemPrompt: SimpleSystemPromptSchemaV1.optional(),
+      systemPrompt: SimpleSystemPromptSchemaV1.optional(),
+      templateVariables: z4.record(z4.string(), z4.string()).optional()
+    }
+  },
+  async (params) => {
+    const { testCase, provider, scorer, runConfig } = params;
+    const messages = [];
+    if (runConfig.systemPrompt) {
+      messages.push({
+        role: "system",
+        content: runConfig.systemPrompt.content
+      });
+    }
+    if (testCase.kind === "llm/mcq.tc") {
+      messages.push({
+        role: "user",
+        content: formatMCQ(testCase)
+      });
+      templateMessages(messages, runConfig.templateVariables ?? {});
+      return runMCQ({
+        testCase,
+        messages,
+        provider,
+        scorer,
+        runConfig,
+        idGenerators: {
+          response: params.idGenerators?.response ?? idGeneratorUUIDv7,
+          score: params.idGenerators?.score ?? idGeneratorUUIDv7
+        }
+      });
+    }
+    if (testCase.kind === "llm/qa.tc") {
+      if (scorer && scorer?.kind !== `${PEERBENCH_NAMESPACE}/llm-as-a-judge`) {
+        throw new Error(
+          `QA test cases can only be scored with an LLM as a judge scorer, but ${scorer?.kind} was provided`
+        );
+      }
+      messages.push({
+        role: "user",
+        content: testCase.question
+      });
+      templateMessages(messages, runConfig.templateVariables ?? {});
+      return runQA({
+        testCase,
+        messages,
+        provider,
+        scorer,
+        runConfig,
+        idGenerators: {
+          response: params.idGenerators?.response ?? idGeneratorUUIDv7,
+          score: params.idGenerators?.score ?? idGeneratorUUIDv7
+        }
+      });
+    }
+    throw new Error("Unsupported test case kind");
+  }
+);
+async function runQA(params) {
+  const { messages, testCase, provider, scorer, runConfig } = params;
+  const providerResponse = await provider.forward({
+    model: runConfig.model,
+    messages
+  });
+  const response = await QAResponseSchemaV1.newWithId(
+    {
+      data: providerResponse.data,
+      startedAt: providerResponse.startedAt,
+      completedAt: providerResponse.completedAt,
+      testCaseId: testCase.id,
+      modelSlug: runConfig.model,
+      provider: provider.kind,
+      systemPromptId: runConfig.systemPrompt?.id,
+      inputTokensUsed: providerResponse.inputTokensUsed,
+      outputTokensUsed: providerResponse.outputTokensUsed,
+      inputCost: providerResponse.inputCost,
+      outputCost: providerResponse.outputCost
+    },
+    params.idGenerators?.response ?? idGeneratorUUIDv7
+  );
+  if (scorer?.kind === `${PEERBENCH_NAMESPACE}/llm-as-a-judge`) {
+    if (!runConfig.llmJudgeModel) {
+      throw new Error(
+        "LLM judge model is required when using LLM as a judge scorer"
+      );
+    }
+    const scorerResult = await scorer.score({
+      model: runConfig.llmJudgeModel,
+      response: response.data,
+      rubric: `Expected/Valid answers: ${testCase.goodAnswers.join("\n")}
+Invalid answers: ${testCase.badAnswers.join("\n")}`,
+      systemPrompt: runConfig.llmJudgeSystemPrompt?.content,
+      criteria: [
+        {
+          id: "correctness",
+          description: "Is the response matches with the expected/valid answers in terms of meaning?",
+          weight: 1
+        }
+      ]
+    });
+    if (scorerResult !== null) {
+      const score = await QAScoreSchemaV1.newWithId(
+        {
+          scoringMethod: ScoringMethod.ai,
+          value: scorerResult.value,
+          responseId: response.id,
+          explanation: scorerResult.explanation,
+          metadata: scorerResult.metadata,
+          scorerAIInputCost: scorerResult.inputCost,
+          scorerAIOutputCost: scorerResult.outputCost,
+          scorerAIInputTokensUsed: scorerResult.inputTokensUsed,
+          scorerAIOutputTokensUsed: scorerResult.outputTokensUsed,
+          scorerAIProvider: scorerResult.provider,
+          scorerAIModelSlug: runConfig.llmJudgeModel,
+          scorerAISystemPromptId: runConfig.llmJudgeSystemPrompt?.id
+        },
+        params.idGenerators?.score ?? idGeneratorUUIDv7
+      );
+      return { response, score };
+    }
+  }
+  return { response };
+}
+async function runMCQ(params) {
+  const { messages, testCase, provider, scorer, runConfig } = params;
+  const providerResponse = await provider.forward({
+    model: runConfig.model,
+    messages
+  });
+  const response = await MCQResponseSchemaV1.newWithId(
+    {
+      data: providerResponse.data,
+      startedAt: providerResponse.startedAt,
+      completedAt: providerResponse.completedAt,
+      testCaseId: testCase.id,
+      modelSlug: runConfig.model,
+      provider: provider.kind,
+      systemPromptId: runConfig.systemPrompt?.id,
+      inputTokensUsed: providerResponse.inputTokensUsed,
+      outputTokensUsed: providerResponse.outputTokensUsed,
+      inputCost: providerResponse.inputCost,
+      outputCost: providerResponse.outputCost
+    },
+    params.idGenerators?.response ?? idGeneratorUUIDv7
+  );
+  if (scorer?.kind === `${PEERBENCH_NAMESPACE}/mcq`) {
+    const scorerResult = await scorer.score({
+      response: response.data,
+      choices: testCase.options,
+      correctAnswers: testCase.correctAnswerKeys
+    });
+    if (scorerResult !== null) {
+      const score = await MCQScoreSchemaV1.newWithId(
+        {
+          scoringMethod: ScoringMethod.algo,
+          value: scorerResult.value,
+          responseId: response.id,
+          extractedAnswers: scorerResult.extractedAnswers,
+          explanation: scorerResult.explanation,
+          metadata: scorerResult.metadata
+        },
+        params.idGenerators?.score ?? idGeneratorUUIDv7
+      );
+      return { response, score };
+    }
+  }
+  if (scorer?.kind === `${PEERBENCH_NAMESPACE}/llm-as-a-judge`) {
+    if (!runConfig.llmJudgeModel) {
+      throw new Error(
+        "LLM judge model is required when using LLM as a judge scorer"
+      );
+    }
+    const scorerResult = await scorer.score({
+      model: runConfig.llmJudgeModel,
+      criteria: [
+        {
+          id: "correctness",
+          description: "Is the given answer key matches with one of the correct answer keys?",
+          weight: 1
+        }
+      ],
+      rubric: `Answer text itself or the key (A, B, C) is accepted
+Valid answer keys: ${testCase.correctAnswerKeys.map((key) => `- ${key}`).join("\n")}
+Valid Answer texts: ${testCase.correctAnswerKeys.map((key) => `- ${testCase.options?.[key] ?? ""}`).join("\n")}`,
+      fieldsToExtract: {
+        extractedAnswers: z4.string().array().describe(
+          "The extracted answer keys, valid or invalid (even if the answer text is provided rather than the key)"
+        )
+      },
+      response: response.data,
+      systemPrompt: runConfig.llmJudgeSystemPrompt?.content
+    });
+    if (scorerResult !== null) {
+      const score = await MCQScoreSchemaV1.newWithId(
+        {
+          scoringMethod: ScoringMethod.ai,
+          value: scorerResult.value,
+          extractedAnswers: scorerResult.extractedFields.extractedAnswers,
+          responseId: response.id,
+          explanation: scorerResult.explanation,
+          metadata: scorerResult.metadata,
+          scorerAIInputCost: scorerResult.inputCost,
+          scorerAIOutputCost: scorerResult.outputCost,
+          scorerAIInputTokensUsed: scorerResult.inputTokensUsed,
+          scorerAIOutputTokensUsed: scorerResult.outputTokensUsed,
+          scorerAIProvider: scorerResult.provider,
+          scorerAIModelSlug: runConfig.llmJudgeModel,
+          scorerAISystemPromptId: runConfig.llmJudgeSystemPrompt?.id
+        },
+        params.idGenerators?.score ?? idGeneratorUUIDv7
+      );
+      return { response, score };
+    }
+  }
+  return { response };
+}
+function formatMCQ(testCase) {
+  return `Question: ${testCase.question}
+Options:
+${Object.entries(
+    testCase.options ?? {}
+  ).map(([key, value]) => `${key}: ${value}`).join("\n")}`;
+}
+function templateMessages(messages, templateVariables) {
+  for (let i = 0; i < messages.length; i++) {
+    const template = Handlebars.compile(messages[i].content);
+    messages[i].content = template(templateVariables);
+  }
+}
+
+// src/benchmarks/peerbench/storages/json.ts
+import z5 from "zod";
+var PeerbenchJSONStorage = class extends JSONFileStorage {
+  constructor(config) {
+    super({
+      path: config.path,
+      chunkSize: config.chunkSize,
+      schema: z5.union([
+        MCQTestCaseSchemaV1,
+        MCQResponseSchemaV1,
+        MCQScoreSchemaV1,
+        QATestCaseSchemaV1,
+        QAResponseSchemaV1,
+        QAScoreSchemaV1,
+        MultiTurnTestCaseSchemaV1,
+        MultiTurnResponseSchemaV1,
+        MultiTurnScoreSchemaV1
+      ])
+    });
+  }
+};
+export {
+  peerbench_exports as peerbench
+};
+//# sourceMappingURL=index.js.map

package/dist/benchmarks/index.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/benchmarks/peerbench/index.ts","../../src/benchmarks/peerbench/schema-sets/mcq.v1.ts","../../src/benchmarks/peerbench/schema-sets/multi-turn.v1.ts","../../src/benchmarks/peerbench/schema-sets/qa.v1.ts","../../src/benchmarks/peerbench/runner.ts","../../src/benchmarks/peerbench/storages/json.ts"],"sourcesContent":["export * from \"./schema-sets/mcq.v1\";\nexport * from \"./schema-sets/multi-turn.v1\";\nexport * from \"./schema-sets/qa.v1\";\n\nexport * from \"./runner\";\n\nexport * from \"./storages/json\";\n","import { PEERBENCH_NAMESPACE } from \"@/constants\";\nimport {\n  BaseResponseSchemaV1,\n  BaseScoreSchemaV1,\n  BaseTestCaseSchemaV1,\n  defineResponseSchema,\n  defineScoreSchema,\n  defineTestCaseSchema,\n} from \"@/schemas\";\nimport { ExtensionLLMResponseFieldsV1 } from \"@/schemas/extensions/response/llm\";\nimport { ExtensionLLMAsAJudgeScoreFieldsV1 } from \"@/schemas/extensions/score/llm-as-a-judge-scorer\";\nimport { z } from \"zod\";\n\nexport const MCQKind = `llm/mcq` as const;\n\nexport const MCQTestCaseSchemaV1 = defineTestCaseSchema({\n  baseSchema: BaseTestCaseSchemaV1,\n  namespace: PEERBENCH_NAMESPACE,\n  kind: MCQKind,\n  schemaVersion: 1,\n  fields: {\n    question: z.string(),\n    options: z.record(z.string(), z.string()),\n    correctAnswerKeys: z.string().array(),\n  },\n});\nexport type MCQTestCaseV1 = z.infer<typeof MCQTestCaseSchemaV1>;\n\nexport const MCQResponseSchemaV1 = defineResponseSchema({\n  baseSchema: BaseResponseSchemaV1,\n  namespace: PEERBENCH_NAMESPACE,\n  kind: MCQKind,\n  schemaVersion: 1,\n  fields: {\n    ...ExtensionLLMResponseFieldsV1,\n  },\n});\nexport type MCQResponseV1 = z.infer<typeof MCQResponseSchemaV1>;\n\nexport const MCQScoreSchemaV1 = defineScoreSchema({\n  baseSchema: BaseScoreSchemaV1,\n  namespace: PEERBENCH_NAMESPACE,\n  kind: MCQKind,\n  schemaVersion: 1,\n  fields: {\n    ...ExtensionLLMAsAJudgeScoreFieldsV1,\n    extractedAnswers: z.array(z.string()),\n  },\n});\nexport type MCQScoreV1 = z.infer<typeof MCQScoreSchemaV1>;\n","import { PEERBENCH_NAMESPACE } from \"@/constants\";\nimport {\n  BaseResponseSchemaV1,\n  BaseScoreSchemaV1,\n  BaseTestCaseSchemaV1,\n  defineResponseSchema,\n  defineScoreSchema,\n  defineTestCaseSchema,\n} from \"@/schemas\";\nimport { ExtensionLLMResponseFieldsV1 } from \"@/schemas/extensions/response/llm\";\nimport { ExtensionLLMAsAJudgeScoreFieldsV1 } from \"@/schemas/extensions/score/llm-as-a-judge-scorer\";\nimport { z } from \"zod\";\n\nexport const MultiTurnKind = `llm/multi-turn` as const;\n\nexport const MultiTurnTestCaseSchemaV1 = defineTestCaseSchema({\n  baseSchema: BaseTestCaseSchemaV1,\n  namespace: PEERBENCH_NAMESPACE,\n  kind: MultiTurnKind,\n  schemaVersion: 1,\n  fields: {\n    messages: z\n      .object({\n        role: z.string(),\n        content: z.string(),\n        goodAnswers: z.string().array().optional(),\n        badAnswers: z.string().array().optional(),\n      })\n      .array(),\n\n    maxTurns: z.number().optional(),\n    expectedOutcome: z.string().optional(),\n  },\n});\nexport type MultiTurnTestCaseV1 = z.infer<typeof MultiTurnTestCaseSchemaV1>;\n\nexport const MultiTurnResponseSchemaV1 = defineResponseSchema({\n  baseSchema: BaseResponseSchemaV1,\n  namespace: PEERBENCH_NAMESPACE,\n  kind: MultiTurnKind,\n  schemaVersion: 1,\n  fields: {\n    ...ExtensionLLMResponseFieldsV1,\n    replies: z\n      .object({\n        messageIndex: z.number(),\n        startedAt: z.number(),\n        completedAt: z.number(),\n        data: z.string(),\n\n        inputTokensUsed: z.number().optional(),\n        outputTokensUsed: z.number().optional(),\n        inputCost: z.string().optional(),\n        outputCost: z.string().optional(),\n      })\n      .array(),\n  },\n});\nexport type MultiTurnResponseV1 = z.infer<typeof MultiTurnResponseSchemaV1>;\n\nexport const MultiTurnScoreSchemaV1 = defineScoreSchema({\n  baseSchema: BaseScoreSchemaV1,\n  namespace: PEERBENCH_NAMESPACE,\n  kind: MultiTurnKind,\n  schemaVersion: 1,\n  fields: {\n    ...ExtensionLLMAsAJudgeScoreFieldsV1,\n    individualScores: z\n      .object({\n        replyIndex: z.number(),\n        value: z.number(),\n      })\n      .array(),\n  },\n});\nexport type MultiTurnScoreV1 = z.infer<typeof MultiTurnScoreSchemaV1>;\n","import { PEERBENCH_NAMESPACE } from \"@/constants\";\nimport {\n  BaseResponseSchemaV1,\n  BaseScoreSchemaV1,\n  BaseTestCaseSchemaV1,\n  defineResponseSchema,\n  defineScoreSchema,\n  defineTestCaseSchema,\n} from \"@/schemas\";\nimport { ExtensionLLMResponseFieldsV1 } from \"@/schemas/extensions/response/llm\";\nimport { ExtensionLLMAsAJudgeScoreFieldsV1 } from \"@/schemas/extensions/score/llm-as-a-judge-scorer\";\nimport { z } from \"zod\";\n\nexport const QAKind = `llm/qa` as const;\n\nexport const QATestCaseSchemaV1 = defineTestCaseSchema({\n  baseSchema: BaseTestCaseSchemaV1,\n  namespace: PEERBENCH_NAMESPACE,\n  kind: QAKind,\n  schemaVersion: 1,\n  fields: {\n    question: z.string(),\n    goodAnswers: z.string().array(),\n    badAnswers: z.string().array(),\n  },\n});\nexport type QATestCaseV1 = z.infer<typeof QATestCaseSchemaV1>;\n\nexport const QAResponseSchemaV1 = defineResponseSchema({\n  baseSchema: BaseResponseSchemaV1,\n  namespace: PEERBENCH_NAMESPACE,\n  kind: QAKind,\n  schemaVersion: 1,\n  fields: {\n    ...ExtensionLLMResponseFieldsV1,\n  },\n});\nexport type QAResponseV1 = z.infer<typeof QAResponseSchemaV1>;\n\nexport const QAScoreSchemaV1 = defineScoreSchema({\n  baseSchema: BaseScoreSchemaV1,\n  namespace: PEERBENCH_NAMESPACE,\n  kind: QAKind,\n  schemaVersion: 1,\n  fields: {\n    ...ExtensionLLMAsAJudgeScoreFieldsV1,\n  },\n});\nexport type QAScoreV1 = z.infer<typeof QAScoreSchemaV1>;\n","import { defineRunner } from \"@/helpers/define-runner\";\nimport { AbstractLLMProvider } from \"@/providers\";\nimport {\n  SimpleSystemPromptSchemaV1,\n  SimpleSystemPromptV1,\n} from \"@/schemas/llm\";\nimport { LLMAsAJudgeScorer, MCQScorer } from \"@/scorers\";\nimport { IdGenerator, ScoringMethod } from \"@/types\";\nimport { idGeneratorUUIDv7 } from \"@/utils\";\nimport { ChatCompletionMessageParam } from \"openai/resources/index\";\nimport Handlebars from \"handlebars\";\nimport z from \"zod\";\nimport {\n  MCQResponseSchemaV1,\n  MCQScoreSchemaV1,\n  MCQTestCaseSchemaV1,\n  MCQTestCaseV1,\n} from \"./schema-sets/mcq.v1\";\nimport {\n  QAResponseSchemaV1,\n  QAScoreSchemaV1,\n  QATestCaseSchemaV1,\n  QATestCaseV1,\n} from \"./schema-sets/qa.v1\";\nimport { PEERBENCH_NAMESPACE } from \"@/constants\";\n\nexport const peerbenchRunner = defineRunner(\n  {\n    schemaSets: [\n      {\n        testCase: MCQTestCaseSchemaV1,\n        response: MCQResponseSchemaV1,\n        score: MCQScoreSchemaV1,\n      },\n      {\n        testCase: QATestCaseSchemaV1,\n        response: QAResponseSchemaV1,\n        score: QAScoreSchemaV1,\n      },\n    ],\n    providers: [AbstractLLMProvider],\n    scorers: [LLMAsAJudgeScorer, MCQScorer],\n\n    runConfigSchema: {\n      model: z.string(),\n      llmJudgeModel: z.string().optional(),\n      llmJudgeSystemPrompt: SimpleSystemPromptSchemaV1.optional(),\n      systemPrompt: SimpleSystemPromptSchemaV1.optional(),\n      templateVariables: z.record(z.string(), z.string()).optional(),\n    },\n  },\n  async (params) => {\n    const { testCase, provider, scorer, runConfig } = params;\n    const messages: ChatCompletionMessageParam[] = [];\n\n    if (runConfig.systemPrompt) {\n      messages.push({\n        role: \"system\",\n        content: runConfig.systemPrompt.content,\n      });\n    }\n\n    if (testCase.kind === \"llm/mcq.tc\") {\n      messages.push({\n        role: \"user\",\n        content: formatMCQ(testCase),\n      });\n      templateMessages(messages, runConfig.templateVariables ?? {});\n\n      return runMCQ({\n        testCase,\n        messages,\n        provider,\n        scorer,\n        runConfig,\n        idGenerators: {\n          response: params.idGenerators?.response ?? idGeneratorUUIDv7,\n          score: params.idGenerators?.score ?? idGeneratorUUIDv7,\n        },\n      });\n    }\n\n    if (testCase.kind === \"llm/qa.tc\") {\n      if (\n        scorer &&\n        scorer?.kind !== (`${PEERBENCH_NAMESPACE}/llm-as-a-judge` as const)\n      ) {\n        throw new Error(\n          `QA test cases can only be scored with an LLM as a judge scorer, but ${scorer?.kind} was provided`\n        );\n      }\n\n      messages.push({\n        role: \"user\",\n        content: testCase.question,\n      });\n      templateMessages(messages, runConfig.templateVariables ?? {});\n\n      return runQA({\n        testCase,\n        messages,\n        provider,\n        scorer,\n        runConfig,\n        idGenerators: {\n          response: params.idGenerators?.response ?? idGeneratorUUIDv7,\n          score: params.idGenerators?.score ?? idGeneratorUUIDv7,\n        },\n      });\n    }\n\n    throw new Error(\"Unsupported test case kind\");\n  }\n);\n\nasync function runQA(params: {\n  messages: ChatCompletionMessageParam[];\n  testCase: QATestCaseV1;\n  provider: AbstractLLMProvider;\n  scorer?: LLMAsAJudgeScorer;\n  runConfig: {\n    model: string;\n    llmJudgeModel?: string;\n    llmJudgeSystemPrompt?: SimpleSystemPromptV1;\n    systemPrompt?: SimpleSystemPromptV1;\n  };\n  idGenerators: {\n    response: IdGenerator;\n    score: IdGenerator;\n  };\n}) {\n  const { messages, testCase, provider, scorer, runConfig } = params;\n\n  const providerResponse = await provider.forward({\n    model: runConfig.model,\n    messages,\n  });\n\n  const response = await QAResponseSchemaV1.newWithId(\n    {\n      data: providerResponse.data,\n      startedAt: providerResponse.startedAt,\n      completedAt: providerResponse.completedAt,\n      testCaseId: testCase.id,\n      modelSlug: runConfig.model,\n      provider: provider.kind,\n      systemPromptId: runConfig.systemPrompt?.id,\n\n      inputTokensUsed: providerResponse.inputTokensUsed,\n      outputTokensUsed: providerResponse.outputTokensUsed,\n      inputCost: providerResponse.inputCost,\n      outputCost: providerResponse.outputCost,\n    },\n    params.idGenerators?.response ?? idGeneratorUUIDv7\n  );\n\n  if (scorer?.kind === (`${PEERBENCH_NAMESPACE}/llm-as-a-judge` as const)) {\n    if (!runConfig.llmJudgeModel) {\n      throw new Error(\n        \"LLM judge model is required when using LLM as a judge scorer\"\n      );\n    }\n\n    const scorerResult = await scorer.score({\n      model: runConfig.llmJudgeModel,\n      response: response.data,\n      rubric: `Expected/Valid answers: ${testCase.goodAnswers.join(\"\\n\")}\\nInvalid answers: ${testCase.badAnswers.join(\"\\n\")}`,\n      systemPrompt: runConfig.llmJudgeSystemPrompt?.content,\n      criteria: [\n        {\n          id: \"correctness\",\n          description:\n            \"Is the response matches with the expected/valid answers in terms of meaning?\",\n          weight: 1,\n        },\n      ],\n    });\n\n    if (scorerResult !== null) {\n      const score = await QAScoreSchemaV1.newWithId(\n        {\n          scoringMethod: ScoringMethod.ai,\n          value: scorerResult.value,\n          responseId: response.id,\n          explanation: scorerResult.explanation,\n          metadata: scorerResult.metadata,\n          scorerAIInputCost: scorerResult.inputCost,\n          scorerAIOutputCost: scorerResult.outputCost,\n          scorerAIInputTokensUsed: scorerResult.inputTokensUsed,\n          scorerAIOutputTokensUsed: scorerResult.outputTokensUsed,\n          scorerAIProvider: scorerResult.provider,\n          scorerAIModelSlug: runConfig.llmJudgeModel,\n          scorerAISystemPromptId: runConfig.llmJudgeSystemPrompt?.id,\n        },\n        params.idGenerators?.score ?? idGeneratorUUIDv7\n      );\n\n      return { response, score };\n    }\n  }\n\n  return { response };\n}\n\nasync function runMCQ(params: {\n  messages: ChatCompletionMessageParam[];\n  testCase: MCQTestCaseV1;\n  provider: AbstractLLMProvider;\n  scorer?: MCQScorer | LLMAsAJudgeScorer;\n  runConfig: {\n    model: string;\n    llmJudgeModel?: string;\n    llmJudgeSystemPrompt?: SimpleSystemPromptV1;\n    systemPrompt?: SimpleSystemPromptV1;\n  };\n  idGenerators: {\n    response: IdGenerator;\n    score: IdGenerator;\n  };\n}) {\n  const { messages, testCase, provider, scorer, runConfig } = params;\n\n  const providerResponse = await provider.forward({\n    model: runConfig.model,\n    messages,\n  });\n\n  const response = await MCQResponseSchemaV1.newWithId(\n    {\n      data: providerResponse.data,\n      startedAt: providerResponse.startedAt,\n      completedAt: providerResponse.completedAt,\n      testCaseId: testCase.id,\n      modelSlug: runConfig.model,\n      provider: provider.kind,\n      systemPromptId: runConfig.systemPrompt?.id,\n\n      inputTokensUsed: providerResponse.inputTokensUsed,\n      outputTokensUsed: providerResponse.outputTokensUsed,\n      inputCost: providerResponse.inputCost,\n      outputCost: providerResponse.outputCost,\n    },\n    params.idGenerators?.response ?? idGeneratorUUIDv7\n  );\n\n  if (scorer?.kind === (`${PEERBENCH_NAMESPACE}/mcq` as const)) {\n    const scorerResult = await scorer.score({\n      response: response.data,\n      choices: testCase.options,\n      correctAnswers: testCase.correctAnswerKeys,\n    });\n\n    if (scorerResult !== null) {\n      const score = await MCQScoreSchemaV1.newWithId(\n        {\n          scoringMethod: ScoringMethod.algo,\n          value: scorerResult.value,\n          responseId: response.id,\n          extractedAnswers: scorerResult.extractedAnswers,\n          explanation: scorerResult.explanation,\n          metadata: scorerResult.metadata,\n        },\n        params.idGenerators?.score ?? idGeneratorUUIDv7\n      );\n\n      return { response, score };\n    }\n  }\n\n  if (scorer?.kind === (`${PEERBENCH_NAMESPACE}/llm-as-a-judge` as const)) {\n    if (!runConfig.llmJudgeModel) {\n      throw new Error(\n        \"LLM judge model is required when using LLM as a judge scorer\"\n      );\n    }\n\n    const scorerResult = await scorer.score({\n      model: runConfig.llmJudgeModel,\n      criteria: [\n        {\n          id: \"correctness\",\n          description:\n            \"Is the given answer key matches with one of the correct answer keys?\",\n          weight: 1,\n        },\n      ],\n      rubric: `Answer text itself or the key (A, B, C) is accepted\nValid answer keys: ${testCase.correctAnswerKeys.map((key) => `- ${key}`).join(\"\\n\")}\nValid Answer texts: ${testCase.correctAnswerKeys.map((key) => `- ${testCase.options?.[key] ?? \"\"}`).join(\"\\n\")}`,\n      fieldsToExtract: {\n        extractedAnswers: z\n          .string()\n          .array()\n          .describe(\n            \"The extracted answer keys, valid or invalid (even if the answer text is provided rather than the key)\"\n          ),\n      },\n      response: response.data,\n      systemPrompt: runConfig.llmJudgeSystemPrompt?.content,\n    });\n\n    if (scorerResult !== null) {\n      const score = await MCQScoreSchemaV1.newWithId(\n        {\n          scoringMethod: ScoringMethod.ai,\n          value: scorerResult.value,\n          extractedAnswers: scorerResult.extractedFields.extractedAnswers,\n          responseId: response.id,\n          explanation: scorerResult.explanation,\n          metadata: scorerResult.metadata,\n          scorerAIInputCost: scorerResult.inputCost,\n          scorerAIOutputCost: scorerResult.outputCost,\n          scorerAIInputTokensUsed: scorerResult.inputTokensUsed,\n          scorerAIOutputTokensUsed: scorerResult.outputTokensUsed,\n          scorerAIProvider: scorerResult.provider,\n          scorerAIModelSlug: runConfig.llmJudgeModel,\n          scorerAISystemPromptId: runConfig.llmJudgeSystemPrompt?.id,\n        },\n        params.idGenerators?.score ?? idGeneratorUUIDv7\n      );\n\n      return { response, score };\n    }\n  }\n\n  return { response };\n}\n\nfunction formatMCQ(testCase: MCQTestCaseV1) {\n  return `Question: ${testCase.question}\\nOptions:\\n${Object.entries(\n    testCase.options ?? {}\n  )\n    .map(([key, value]) => `${key}: ${value}`)\n    .join(\"\\n\")}`;\n}\n\nfunction templateMessages(\n  messages: ChatCompletionMessageParam[],\n  templateVariables: Record<string, string>\n) {\n  for (let i = 0; i < messages.length; i++) {\n    const template = Handlebars.compile(messages[i]!.content);\n    messages[i]!.content = template(templateVariables);\n  }\n}\n","import { JSONFileStorage } from \"@/storages/json-file\";\nimport {\n  MCQResponseSchemaV1,\n  MCQResponseV1,\n  MCQScoreSchemaV1,\n  MCQScoreV1,\n  MCQTestCaseSchemaV1,\n  MCQTestCaseV1,\n} from \"../schema-sets/mcq.v1\";\nimport {\n  QAResponseSchemaV1,\n  QAResponseV1,\n  QAScoreSchemaV1,\n  QAScoreV1,\n  QATestCaseSchemaV1,\n  QATestCaseV1,\n} from \"../schema-sets/qa.v1\";\nimport {\n  MultiTurnResponseSchemaV1,\n  MultiTurnResponseV1,\n  MultiTurnScoreSchemaV1,\n  MultiTurnScoreV1,\n  MultiTurnTestCaseSchemaV1,\n  MultiTurnTestCaseV1,\n} from \"../schema-sets/multi-turn.v1\";\nimport z from \"zod\";\n\nexport class PeerbenchJSONStorage extends JSONFileStorage<\n  | MCQTestCaseV1\n  | MCQResponseV1\n  | MCQScoreV1\n  | QATestCaseV1\n  | QAResponseV1\n  | QAScoreV1\n  | MultiTurnTestCaseV1\n  | MultiTurnResponseV1\n  | MultiTurnScoreV1\n> {\n  constructor(config: { path: string; chunkSize?: number }) {\n    super({\n      path: config.path,\n      chunkSize: config.chunkSize,\n\n      schema: z.union([\n        MCQTestCaseSchemaV1,\n        MCQResponseSchemaV1,\n        MCQScoreSchemaV1,\n        QATestCaseSchemaV1,\n        QAResponseSchemaV1,\n        QAScoreSchemaV1,\n        MultiTurnTestCaseSchemaV1,\n        MultiTurnResponseSchemaV1,\n        MultiTurnScoreSchemaV1,\n      ]),\n    });\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;;;ACWA,SAAS,SAAS;AAEX,IAAM,UAAU;AAEhB,IAAM,sBAAsB,qBAAqB;AAAA,EACtD,YAAY;AAAA,EACZ,WAAW;AAAA,EACX,MAAM;AAAA,EACN,eAAe;AAAA,EACf,QAAQ;AAAA,IACN,UAAU,EAAE,OAAO;AAAA,IACnB,SAAS,EAAE,OAAO,EAAE,OAAO,GAAG,EAAE,OAAO,CAAC;AAAA,IACxC,mBAAmB,EAAE,OAAO,EAAE,MAAM;AAAA,EACtC;AACF,CAAC;AAGM,IAAM,sBAAsB,qBAAqB;AAAA,EACtD,YAAY;AAAA,EACZ,WAAW;AAAA,EACX,MAAM;AAAA,EACN,eAAe;AAAA,EACf,QAAQ;AAAA,IACN,GAAG;AAAA,EACL;AACF,CAAC;AAGM,IAAM,mBAAmB,kBAAkB;AAAA,EAChD,YAAY;AAAA,EACZ,WAAW;AAAA,EACX,MAAM;AAAA,EACN,eAAe;AAAA,EACf,QAAQ;AAAA,IACN,GAAG;AAAA,IACH,kBAAkB,EAAE,MAAM,EAAE,OAAO,CAAC;AAAA,EACtC;AACF,CAAC;;;ACrCD,SAAS,KAAAA,UAAS;AAEX,IAAM,gBAAgB;AAEtB,IAAM,4BAA4B,qBAAqB;AAAA,EAC5D,YAAY;AAAA,EACZ,WAAW;AAAA,EACX,MAAM;AAAA,EACN,eAAe;AAAA,EACf,QAAQ;AAAA,IACN,UAAUA,GACP,OAAO;AAAA,MACN,MAAMA,GAAE,OAAO;AAAA,MACf,SAASA,GAAE,OAAO;AAAA,MAClB,aAAaA,GAAE,OAAO,EAAE,MAAM,EAAE,SAAS;AAAA,MACzC,YAAYA,GAAE,OAAO,EAAE,MAAM,EAAE,SAAS;AAAA,IAC1C,CAAC,EACA,MAAM;AAAA,IAET,UAAUA,GAAE,OAAO,EAAE,SAAS;AAAA,IAC9B,iBAAiBA,GAAE,OAAO,EAAE,SAAS;AAAA,EACvC;AACF,CAAC;AAGM,IAAM,4BAA4B,qBAAqB;AAAA,EAC5D,YAAY;AAAA,EACZ,WAAW;AAAA,EACX,MAAM;AAAA,EACN,eAAe;AAAA,EACf,QAAQ;AAAA,IACN,GAAG;AAAA,IACH,SAASA,GACN,OAAO;AAAA,MACN,cAAcA,GAAE,OAAO;AAAA,MACvB,WAAWA,GAAE,OAAO;AAAA,MACpB,aAAaA,GAAE,OAAO;AAAA,MACtB,MAAMA,GAAE,OAAO;AAAA,MAEf,iBAAiBA,GAAE,OAAO,EAAE,SAAS;AAAA,MACrC,kBAAkBA,GAAE,OAAO,EAAE,SAAS;AAAA,MACtC,WAAWA,GAAE,OAAO,EAAE,SAAS;AAAA,MAC/B,YAAYA,GAAE,OAAO,EAAE,SAAS;AAAA,IAClC,CAAC,EACA,MAAM;AAAA,EACX;AACF,CAAC;AAGM,IAAM,yBAAyB,kBAAkB;AAAA,EACtD,YAAY;AAAA,EACZ,WAAW;AAAA,EACX,MAAM;AAAA,EACN,eAAe;AAAA,EACf,QAAQ;AAAA,IACN,GAAG;AAAA,IACH,kBAAkBA,GACf,OAAO;AAAA,MACN,YAAYA,GAAE,OAAO;AAAA,MACrB,OAAOA,GAAE,OAAO;AAAA,IAClB,CAAC,EACA,MAAM;AAAA,EACX;AACF,CAAC;;;AC/DD,SAAS,KAAAC,UAAS;AAEX,IAAM,SAAS;AAEf,IAAM,qBAAqB,qBAAqB;AAAA,EACrD,YAAY;AAAA,EACZ,WAAW;AAAA,EACX,MAAM;AAAA,EACN,eAAe;AAAA,EACf,QAAQ;AAAA,IACN,UAAUA,GAAE,OAAO;AAAA,IACnB,aAAaA,GAAE,OAAO,EAAE,MAAM;AAAA,IAC9B,YAAYA,GAAE,OAAO,EAAE,MAAM;AAAA,EAC/B;AACF,CAAC;AAGM,IAAM,qBAAqB,qBAAqB;AAAA,EACrD,YAAY;AAAA,EACZ,WAAW;AAAA,EACX,MAAM;AAAA,EACN,eAAe;AAAA,EACf,QAAQ;AAAA,IACN,GAAG;AAAA,EACL;AACF,CAAC;AAGM,IAAM,kBAAkB,kBAAkB;AAAA,EAC/C,YAAY;AAAA,EACZ,WAAW;AAAA,EACX,MAAM;AAAA,EACN,eAAe;AAAA,EACf,QAAQ;AAAA,IACN,GAAG;AAAA,EACL;AACF,CAAC;;;ACrCD,OAAO,gBAAgB;AACvB,OAAOC,QAAO;AAeP,IAAM,kBAAkB;AAAA,EAC7B;AAAA,IACE,YAAY;AAAA,MACV;AAAA,QACE,UAAU;AAAA,QACV,UAAU;AAAA,QACV,OAAO;AAAA,MACT;AAAA,MACA;AAAA,QACE,UAAU;AAAA,QACV,UAAU;AAAA,QACV,OAAO;AAAA,MACT;AAAA,IACF;AAAA,IACA,WAAW,CAAC,mBAAmB;AAAA,IAC/B,SAAS,CAAC,mBAAmB,SAAS;AAAA,IAEtC,iBAAiB;AAAA,MACf,OAAOC,GAAE,OAAO;AAAA,MAChB,eAAeA,GAAE,OAAO,EAAE,SAAS;AAAA,MACnC,sBAAsB,2BAA2B,SAAS;AAAA,MAC1D,cAAc,2BAA2B,SAAS;AAAA,MAClD,mBAAmBA,GAAE,OAAOA,GAAE,OAAO,GAAGA,GAAE,OAAO,CAAC,EAAE,SAAS;AAAA,IAC/D;AAAA,EACF;AAAA,EACA,OAAO,WAAW;AAChB,UAAM,EAAE,UAAU,UAAU,QAAQ,UAAU,IAAI;AAClD,UAAM,WAAyC,CAAC;AAEhD,QAAI,UAAU,cAAc;AAC1B,eAAS,KAAK;AAAA,QACZ,MAAM;AAAA,QACN,SAAS,UAAU,aAAa;AAAA,MAClC,CAAC;AAAA,IACH;AAEA,QAAI,SAAS,SAAS,cAAc;AAClC,eAAS,KAAK;AAAA,QACZ,MAAM;AAAA,QACN,SAAS,UAAU,QAAQ;AAAA,MAC7B,CAAC;AACD,uBAAiB,UAAU,UAAU,qBAAqB,CAAC,CAAC;AAE5D,aAAO,OAAO;AAAA,QACZ;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA,cAAc;AAAA,UACZ,UAAU,OAAO,cAAc,YAAY;AAAA,UAC3C,OAAO,OAAO,cAAc,SAAS;AAAA,QACvC;AAAA,MACF,CAAC;AAAA,IACH;AAEA,QAAI,SAAS,SAAS,aAAa;AACjC,UACE,UACA,QAAQ,SAAU,GAAG,mBAAmB,mBACxC;AACA,cAAM,IAAI;AAAA,UACR,uEAAuE,QAAQ,IAAI;AAAA,QACrF;AAAA,MACF;AAEA,eAAS,KAAK;AAAA,QACZ,MAAM;AAAA,QACN,SAAS,SAAS;AAAA,MACpB,CAAC;AACD,uBAAiB,UAAU,UAAU,qBAAqB,CAAC,CAAC;AAE5D,aAAO,MAAM;AAAA,QACX;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA,cAAc;AAAA,UACZ,UAAU,OAAO,cAAc,YAAY;AAAA,UAC3C,OAAO,OAAO,cAAc,SAAS;AAAA,QACvC;AAAA,MACF,CAAC;AAAA,IACH;AAEA,UAAM,IAAI,MAAM,4BAA4B;AAAA,EAC9C;AACF;AAEA,eAAe,MAAM,QAelB;AACD,QAAM,EAAE,UAAU,UAAU,UAAU,QAAQ,UAAU,IAAI;AAE5D,QAAM,mBAAmB,MAAM,SAAS,QAAQ;AAAA,IAC9C,OAAO,UAAU;AAAA,IACjB;AAAA,EACF,CAAC;AAED,QAAM,WAAW,MAAM,mBAAmB;AAAA,IACxC;AAAA,MACE,MAAM,iBAAiB;AAAA,MACvB,WAAW,iBAAiB;AAAA,MAC5B,aAAa,iBAAiB;AAAA,MAC9B,YAAY,SAAS;AAAA,MACrB,WAAW,UAAU;AAAA,MACrB,UAAU,SAAS;AAAA,MACnB,gBAAgB,UAAU,cAAc;AAAA,MAExC,iBAAiB,iBAAiB;AAAA,MAClC,kBAAkB,iBAAiB;AAAA,MACnC,WAAW,iBAAiB;AAAA,MAC5B,YAAY,iBAAiB;AAAA,IAC/B;AAAA,IACA,OAAO,cAAc,YAAY;AAAA,EACnC;AAEA,MAAI,QAAQ,SAAU,GAAG,mBAAmB,mBAA6B;AACvE,QAAI,CAAC,UAAU,eAAe;AAC5B,YAAM,IAAI;AAAA,QACR;AAAA,MACF;AAAA,IACF;AAEA,UAAM,eAAe,MAAM,OAAO,MAAM;AAAA,MACtC,OAAO,UAAU;AAAA,MACjB,UAAU,SAAS;AAAA,MACnB,QAAQ,2BAA2B,SAAS,YAAY,KAAK,IAAI,CAAC;AAAA,mBAAsB,SAAS,WAAW,KAAK,IAAI,CAAC;AAAA,MACtH,cAAc,UAAU,sBAAsB;AAAA,MAC9C,UAAU;AAAA,QACR;AAAA,UACE,IAAI;AAAA,UACJ,aACE;AAAA,UACF,QAAQ;AAAA,QACV;AAAA,MACF;AAAA,IACF,CAAC;AAED,QAAI,iBAAiB,MAAM;AACzB,YAAM,QAAQ,MAAM,gBAAgB;AAAA,QAClC;AAAA,UACE,eAAe,cAAc;AAAA,UAC7B,OAAO,aAAa;AAAA,UACpB,YAAY,SAAS;AAAA,UACrB,aAAa,aAAa;AAAA,UAC1B,UAAU,aAAa;AAAA,UACvB,mBAAmB,aAAa;AAAA,UAChC,oBAAoB,aAAa;AAAA,UACjC,yBAAyB,aAAa;AAAA,UACtC,0BAA0B,aAAa;AAAA,UACvC,kBAAkB,aAAa;AAAA,UAC/B,mBAAmB,UAAU;AAAA,UAC7B,wBAAwB,UAAU,sBAAsB;AAAA,QAC1D;AAAA,QACA,OAAO,cAAc,SAAS;AAAA,MAChC;AAEA,aAAO,EAAE,UAAU,MAAM;AAAA,IAC3B;AAAA,EACF;AAEA,SAAO,EAAE,SAAS;AACpB;AAEA,eAAe,OAAO,QAenB;AACD,QAAM,EAAE,UAAU,UAAU,UAAU,QAAQ,UAAU,IAAI;AAE5D,QAAM,mBAAmB,MAAM,SAAS,QAAQ;AAAA,IAC9C,OAAO,UAAU;AAAA,IACjB;AAAA,EACF,CAAC;AAED,QAAM,WAAW,MAAM,oBAAoB;AAAA,IACzC;AAAA,MACE,MAAM,iBAAiB;AAAA,MACvB,WAAW,iBAAiB;AAAA,MAC5B,aAAa,iBAAiB;AAAA,MAC9B,YAAY,SAAS;AAAA,MACrB,WAAW,UAAU;AAAA,MACrB,UAAU,SAAS;AAAA,MACnB,gBAAgB,UAAU,cAAc;AAAA,MAExC,iBAAiB,iBAAiB;AAAA,MAClC,kBAAkB,iBAAiB;AAAA,MACnC,WAAW,iBAAiB;AAAA,MAC5B,YAAY,iBAAiB;AAAA,IAC/B;AAAA,IACA,OAAO,cAAc,YAAY;AAAA,EACnC;AAEA,MAAI,QAAQ,SAAU,GAAG,mBAAmB,QAAkB;AAC5D,UAAM,eAAe,MAAM,OAAO,MAAM;AAAA,MACtC,UAAU,SAAS;AAAA,MACnB,SAAS,SAAS;AAAA,MAClB,gBAAgB,SAAS;AAAA,IAC3B,CAAC;AAED,QAAI,iBAAiB,MAAM;AACzB,YAAM,QAAQ,MAAM,iBAAiB;AAAA,QACnC;AAAA,UACE,eAAe,cAAc;AAAA,UAC7B,OAAO,aAAa;AAAA,UACpB,YAAY,SAAS;AAAA,UACrB,kBAAkB,aAAa;AAAA,UAC/B,aAAa,aAAa;AAAA,UAC1B,UAAU,aAAa;AAAA,QACzB;AAAA,QACA,OAAO,cAAc,SAAS;AAAA,MAChC;AAEA,aAAO,EAAE,UAAU,MAAM;AAAA,IAC3B;AAAA,EACF;AAEA,MAAI,QAAQ,SAAU,GAAG,mBAAmB,mBAA6B;AACvE,QAAI,CAAC,UAAU,eAAe;AAC5B,YAAM,IAAI;AAAA,QACR;AAAA,MACF;AAAA,IACF;AAEA,UAAM,eAAe,MAAM,OAAO,MAAM;AAAA,MACtC,OAAO,UAAU;AAAA,MACjB,UAAU;AAAA,QACR;AAAA,UACE,IAAI;AAAA,UACJ,aACE;AAAA,UACF,QAAQ;AAAA,QACV;AAAA,MACF;AAAA,MACA,QAAQ;AAAA,qBACO,SAAS,kBAAkB,IAAI,CAAC,QAAQ,KAAK,GAAG,EAAE,EAAE,KAAK,IAAI,CAAC;AAAA,sBAC7D,SAAS,kBAAkB,IAAI,CAAC,QAAQ,KAAK,SAAS,UAAU,GAAG,KAAK,EAAE,EAAE,EAAE,KAAK,IAAI,CAAC;AAAA,MACxG,iBAAiB;AAAA,QACf,kBAAkBA,GACf,OAAO,EACP,MAAM,EACN;AAAA,UACC;AAAA,QACF;AAAA,MACJ;AAAA,MACA,UAAU,SAAS;AAAA,MACnB,cAAc,UAAU,sBAAsB;AAAA,IAChD,CAAC;AAED,QAAI,iBAAiB,MAAM;AACzB,YAAM,QAAQ,MAAM,iBAAiB;AAAA,QACnC;AAAA,UACE,eAAe,cAAc;AAAA,UAC7B,OAAO,aAAa;AAAA,UACpB,kBAAkB,aAAa,gBAAgB;AAAA,UAC/C,YAAY,SAAS;AAAA,UACrB,aAAa,aAAa;AAAA,UAC1B,UAAU,aAAa;AAAA,UACvB,mBAAmB,aAAa;AAAA,UAChC,oBAAoB,aAAa;AAAA,UACjC,yBAAyB,aAAa;AAAA,UACtC,0BAA0B,aAAa;AAAA,UACvC,kBAAkB,aAAa;AAAA,UAC/B,mBAAmB,UAAU;AAAA,UAC7B,wBAAwB,UAAU,sBAAsB;AAAA,QAC1D;AAAA,QACA,OAAO,cAAc,SAAS;AAAA,MAChC;AAEA,aAAO,EAAE,UAAU,MAAM;AAAA,IAC3B;AAAA,EACF;AAEA,SAAO,EAAE,SAAS;AACpB;AAEA,SAAS,UAAU,UAAyB;AAC1C,SAAO,aAAa,SAAS,QAAQ;AAAA;AAAA,EAAe,OAAO;AAAA,IACzD,SAAS,WAAW,CAAC;AAAA,EACvB,EACG,IAAI,CAAC,CAAC,KAAK,KAAK,MAAM,GAAG,GAAG,KAAK,KAAK,EAAE,EACxC,KAAK,IAAI,CAAC;AACf;AAEA,SAAS,iBACP,UACA,mBACA;AACA,WAAS,IAAI,GAAG,IAAI,SAAS,QAAQ,KAAK;AACxC,UAAM,WAAW,WAAW,QAAQ,SAAS,CAAC,EAAG,OAAO;AACxD,aAAS,CAAC,EAAG,UAAU,SAAS,iBAAiB;AAAA,EACnD;AACF;;;AC/TA,OAAOC,QAAO;AAEP,IAAM,uBAAN,cAAmC,gBAUxC;AAAA,EACA,YAAY,QAA8C;AACxD,UAAM;AAAA,MACJ,MAAM,OAAO;AAAA,MACb,WAAW,OAAO;AAAA,MAElB,QAAQA,GAAE,MAAM;AAAA,QACd;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,MACF,CAAC;AAAA,IACH,CAAC;AAAA,EACH;AACF;","names":["z","z","z","z","z"]}

package/dist/chunk-4UBK6452.js

@@ -0,0 +1,128 @@
+// src/utils/sleep.ts
+async function sleep(ms, signal) {
+  return new Promise((resolve, reject) => {
+    if (signal?.aborted) {
+      return reject(new Error("Aborted"));
+    }
+    const timeout = setTimeout(() => {
+      cleanup();
+      resolve();
+    }, ms);
+    const onAbort = () => {
+      cleanup();
+      reject(new Error("Aborted"));
+    };
+    const cleanup = () => {
+      clearTimeout(timeout);
+      signal?.removeEventListener("abort", onAbort);
+    };
+    signal?.addEventListener("abort", onAbort);
+  });
+}
+
+// src/utils/llm.ts
+import { jsonrepair } from "jsonrepair";
+function parseResponseAsJSON(response) {
+  try {
+    return JSON.parse(jsonrepair(response));
+  } catch (e) {
+    if (process?.env?.PB_SDK_DEBUG) {
+      console.log("Original response", JSON.stringify(response));
+      console.error("Error parsing response as JSON", e);
+    }
+  }
+}
+
+// src/utils/rate-limiter.ts
+var RateLimiter = class {
+  maxWeight;
+  timeWindow;
+  timestamps = [];
+  constructor(options = {}) {
+    this.maxWeight = options.maxWeight ?? 25;
+    this.timeWindow = options.timeWindow ?? 1e3;
+  }
+  /**
+   * Checks if rate limiting is disabled
+   */
+  isDisabled() {
+    return this.maxWeight <= 0;
+  }
+  /**
+   * Disables rate limiting. Set `maxWeight` to re-enable it.
+   */
+  disable() {
+    this.maxWeight = 0;
+  }
+  /**
+   * Returns how many weight of calls are there in the current time window
+   */
+  getCurrentCalls() {
+    const now = Date.now();
+    this.timestamps = this.timestamps.filter(
+      (ts) => now - ts < this.timeWindow
+    );
+    return this.timestamps.length;
+  }
+  /**
+   * Executes the given function with rate limiting applied
+   */
+  async execute(func, { weight = 1, signal } = {}) {
+    if (this.maxWeight <= 0) {
+      return await func();
+    }
+    await this.waitForRateLimit(weight, signal);
+    return await func();
+  }
+  /**
+   * Waits until rate limit allows the specified weight of calls
+   */
+  async waitForRateLimit(weight, signal) {
+    const now = Date.now();
+    this.timestamps = this.timestamps.filter(
+      (ts) => now - ts < this.timeWindow
+    );
+    if (this.timestamps.length + weight <= this.maxWeight) {
+      for (let i = 0; i < weight; i++) {
+        this.timestamps.push(now);
+      }
+      return;
+    }
+    const earliest = this.timestamps[0];
+    const waitTime = this.timeWindow - (now - (earliest ?? 0));
+    await sleep(waitTime, signal);
+    return this.waitForRateLimit(weight, signal);
+  }
+  /**
+   * Resets the rate limited weight of calls
+   */
+  reset() {
+    this.timestamps = [];
+  }
+};
+
+// src/utils/string.ts
+function bufferToString(buffer, encoding = "utf-8") {
+  const decoder = new TextDecoder(encoding);
+  return decoder.decode(buffer);
+}
+function stringToBuffer(str) {
+  const encoder = new TextEncoder();
+  return encoder.encode(str);
+}
+
+// src/utils/id-generator.ts
+import { v7 as uuidv7 } from "uuid";
+var idGeneratorUUIDv7 = () => {
+  return uuidv7();
+};
+
+export {
+  sleep,
+  parseResponseAsJSON,
+  RateLimiter,
+  bufferToString,
+  stringToBuffer,
+  idGeneratorUUIDv7
+};
+//# sourceMappingURL=chunk-4UBK6452.js.map

package/dist/chunk-4UBK6452.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/utils/sleep.ts","../src/utils/llm.ts","../src/utils/rate-limiter.ts","../src/utils/string.ts","../src/utils/id-generator.ts"],"sourcesContent":["export async function sleep(ms: number, signal?: AbortSignal) {\n  return new Promise<void>((resolve, reject) => {\n    if (signal?.aborted) {\n      return reject(new Error(\"Aborted\"));\n    }\n\n    const timeout = setTimeout(() => {\n      cleanup();\n      resolve();\n    }, ms);\n\n    const onAbort = () => {\n      cleanup();\n      reject(new Error(\"Aborted\"));\n    };\n\n    const cleanup = () => {\n      clearTimeout(timeout);\n      signal?.removeEventListener(\"abort\", onAbort);\n    };\n\n    signal?.addEventListener(\"abort\", onAbort);\n  });\n}\n","import { jsonrepair } from \"jsonrepair\";\n\n/**\n * Tries to repair and parse LLM response as a JSON object. LLM must\n * be configured to return a JSON object. This function only helps to\n * get rid out of some additional formatting (e.g. ```json) and repair\n * the JSON syntax (e.g missing comma, single quotes instead double).\n */\nexport function parseResponseAsJSON<T>(response: string) {\n  try {\n    return JSON.parse(jsonrepair(response)) as T;\n  } catch (e) {\n    if (process?.env?.PB_SDK_DEBUG) {\n      console.log(\"Original response\", JSON.stringify(response));\n      console.error(\"Error parsing response as JSON\", e);\n    }\n  }\n}\n","import { sleep } from \"./sleep\";\n\nexport interface RateLimiterOptions {\n  /**\n   * Maximum weight of calls allowed within a time window. Each call has a weight of 1 by default.\n   * Rate limiting is disabled when this is 0 or negative.\n   * @default 25\n   */\n  maxWeight?: number;\n\n  /**\n   * Time window in milliseconds for rate limiting\n   * @default 1000\n   */\n  timeWindow?: number;\n}\n\nexport interface RateLimiterCallOptions {\n  /**\n   * Weight of the call. Rate limiting is applied based on the total weight of the calls.\n   * @default 1\n   */\n  weight?: number;\n\n  /**\n   * Abort signal for cancellation\n   */\n  signal?: AbortSignal;\n}\n\n/**\n * Generic rate limiter. It can be used to limit async function calls\n * by a given number of calls within a time window.\n */\nexport class RateLimiter {\n  maxWeight: number;\n  timeWindow: number;\n\n  private timestamps: number[] = [];\n\n  constructor(options: RateLimiterOptions = {}) {\n    this.maxWeight = options.maxWeight ?? 25;\n    this.timeWindow = options.timeWindow ?? 1000;\n  }\n\n  /**\n   * Checks if rate limiting is disabled\n   */\n  isDisabled(): boolean {\n    return this.maxWeight <= 0;\n  }\n\n  /**\n   * Disables rate limiting. Set `maxWeight` to re-enable it.\n   */\n  disable() {\n    this.maxWeight = 0;\n  }\n\n  /**\n   * Returns how many weight of calls are there in the current time window\n   */\n  getCurrentCalls(): number {\n    const now = Date.now();\n    this.timestamps = this.timestamps.filter(\n      (ts) => now - ts < this.timeWindow\n    );\n    return this.timestamps.length;\n  }\n\n  /**\n   * Executes the given function with rate limiting applied\n   */\n  async execute<T = unknown>(\n    func: () => Promise<T>,\n    { weight = 1, signal }: RateLimiterCallOptions = {}\n  ): Promise<T> {\n    // If maxWeight is zero/negative, execute immediately (no rate limiting)\n    if (this.maxWeight <= 0) {\n      return await func();\n    }\n\n    await this.waitForRateLimit(weight, signal);\n    return await func();\n  }\n\n  /**\n   * Waits until rate limit allows the specified weight of calls\n   */\n  private async waitForRateLimit(\n    weight: number,\n    signal?: AbortSignal\n  ): Promise<void> {\n    const now = Date.now();\n\n    // Clear timestamps from the old time period\n    this.timestamps = this.timestamps.filter(\n      (ts) => now - ts < this.timeWindow\n    );\n\n    // Check if there are empty slots for the requests\n    if (this.timestamps.length + weight <= this.maxWeight) {\n      for (let i = 0; i < weight; i++) {\n        this.timestamps.push(now);\n      }\n      return;\n    }\n\n    // Calculate how much time to wait for the next time window\n    const earliest = this.timestamps[0];\n    const waitTime = this.timeWindow - (now - (earliest ?? 0));\n\n    await sleep(waitTime, signal);\n\n    // Retry after waiting\n    return this.waitForRateLimit(weight, signal);\n  }\n\n  /**\n   * Resets the rate limited weight of calls\n   */\n  reset(): void {\n    this.timestamps = [];\n  }\n}\n","/**\n * Converts the given byte array to a string\n */\nexport function bufferToString(\n  buffer: Uint8Array,\n  encoding: BufferEncoding = \"utf-8\"\n): string {\n  const decoder = new TextDecoder(encoding);\n  return decoder.decode(buffer);\n}\n\n/**\n * Converts the given string to a byte array\n */\nexport function stringToBuffer(str: string): Uint8Array {\n  const encoder = new TextEncoder();\n  return encoder.encode(str);\n}\n","import { IdGenerator } from \"@/types\";\nimport { v7 as uuidv7 } from \"uuid\";\n\nexport const idGeneratorUUIDv7: IdGenerator = () => {\n  return uuidv7();\n};\n"],"mappings":";AAAA,eAAsB,MAAM,IAAY,QAAsB;AAC5D,SAAO,IAAI,QAAc,CAAC,SAAS,WAAW;AAC5C,QAAI,QAAQ,SAAS;AACnB,aAAO,OAAO,IAAI,MAAM,SAAS,CAAC;AAAA,IACpC;AAEA,UAAM,UAAU,WAAW,MAAM;AAC/B,cAAQ;AACR,cAAQ;AAAA,IACV,GAAG,EAAE;AAEL,UAAM,UAAU,MAAM;AACpB,cAAQ;AACR,aAAO,IAAI,MAAM,SAAS,CAAC;AAAA,IAC7B;AAEA,UAAM,UAAU,MAAM;AACpB,mBAAa,OAAO;AACpB,cAAQ,oBAAoB,SAAS,OAAO;AAAA,IAC9C;AAEA,YAAQ,iBAAiB,SAAS,OAAO;AAAA,EAC3C,CAAC;AACH;;;ACvBA,SAAS,kBAAkB;AAQpB,SAAS,oBAAuB,UAAkB;AACvD,MAAI;AACF,WAAO,KAAK,MAAM,WAAW,QAAQ,CAAC;AAAA,EACxC,SAAS,GAAG;AACV,QAAI,SAAS,KAAK,cAAc;AAC9B,cAAQ,IAAI,qBAAqB,KAAK,UAAU,QAAQ,CAAC;AACzD,cAAQ,MAAM,kCAAkC,CAAC;AAAA,IACnD;AAAA,EACF;AACF;;;ACiBO,IAAM,cAAN,MAAkB;AAAA,EACvB;AAAA,EACA;AAAA,EAEQ,aAAuB,CAAC;AAAA,EAEhC,YAAY,UAA8B,CAAC,GAAG;AAC5C,SAAK,YAAY,QAAQ,aAAa;AACtC,SAAK,aAAa,QAAQ,cAAc;AAAA,EAC1C;AAAA;AAAA;AAAA;AAAA,EAKA,aAAsB;AACpB,WAAO,KAAK,aAAa;AAAA,EAC3B;AAAA;AAAA;AAAA;AAAA,EAKA,UAAU;AACR,SAAK,YAAY;AAAA,EACnB;AAAA;AAAA;AAAA;AAAA,EAKA,kBAA0B;AACxB,UAAM,MAAM,KAAK,IAAI;AACrB,SAAK,aAAa,KAAK,WAAW;AAAA,MAChC,CAAC,OAAO,MAAM,KAAK,KAAK;AAAA,IAC1B;AACA,WAAO,KAAK,WAAW;AAAA,EACzB;AAAA;AAAA;AAAA;AAAA,EAKA,MAAM,QACJ,MACA,EAAE,SAAS,GAAG,OAAO,IAA4B,CAAC,GACtC;AAEZ,QAAI,KAAK,aAAa,GAAG;AACvB,aAAO,MAAM,KAAK;AAAA,IACpB;AAEA,UAAM,KAAK,iBAAiB,QAAQ,MAAM;AAC1C,WAAO,MAAM,KAAK;AAAA,EACpB;AAAA;AAAA;AAAA;AAAA,EAKA,MAAc,iBACZ,QACA,QACe;AACf,UAAM,MAAM,KAAK,IAAI;AAGrB,SAAK,aAAa,KAAK,WAAW;AAAA,MAChC,CAAC,OAAO,MAAM,KAAK,KAAK;AAAA,IAC1B;AAGA,QAAI,KAAK,WAAW,SAAS,UAAU,KAAK,WAAW;AACrD,eAAS,IAAI,GAAG,IAAI,QAAQ,KAAK;AAC/B,aAAK,WAAW,KAAK,GAAG;AAAA,MAC1B;AACA;AAAA,IACF;AAGA,UAAM,WAAW,KAAK,WAAW,CAAC;AAClC,UAAM,WAAW,KAAK,cAAc,OAAO,YAAY;AAEvD,UAAM,MAAM,UAAU,MAAM;AAG5B,WAAO,KAAK,iBAAiB,QAAQ,MAAM;AAAA,EAC7C;AAAA;AAAA;AAAA;AAAA,EAKA,QAAc;AACZ,SAAK,aAAa,CAAC;AAAA,EACrB;AACF;;;ACzHO,SAAS,eACd,QACA,WAA2B,SACnB;AACR,QAAM,UAAU,IAAI,YAAY,QAAQ;AACxC,SAAO,QAAQ,OAAO,MAAM;AAC9B;AAKO,SAAS,eAAe,KAAyB;AACtD,QAAM,UAAU,IAAI,YAAY;AAChC,SAAO,QAAQ,OAAO,GAAG;AAC3B;;;AChBA,SAAS,MAAM,cAAc;AAEtB,IAAM,oBAAiC,MAAM;AAClD,SAAO,OAAO;AAChB;","names":[]}