peerbench 0.0.11 → 0.0.12

package/dist/benchmarks/peerbench/schema-sets/multi-turn.v1.d.ts

@@ -6,7 +6,7 @@ export declare const MultiTurnTestCaseSchemaV1: z.ZodObject<Omit<{
     schemaVersion: z.ZodNumber;
     kind: z.ZodString;
     metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
-}, "kind" | "namespace" | "schemaVersion"> & {
+}, "kind" | "schemaVersion" | "namespace"> & {
     messages: z.ZodArray<z.ZodObject<{
         role: z.ZodString;
         content: z.ZodString;
@@ -34,7 +34,7 @@ export declare const MultiTurnTestCaseSchemaV1: z.ZodObject<Omit<{
         metadata?: Record<string, unknown> | undefined;
         maxTurns?: number | undefined;
         expectedOutcome?: string | undefined;
-    }, "kind" | "namespace" | "schemaVersion">) => {
+    }, "kind" | "schemaVersion" | "namespace">) => {
         id: string;
         messages: {
             role: string;
@@ -63,7 +63,7 @@ export declare const MultiTurnTestCaseSchemaV1: z.ZodObject<Omit<{
         metadata?: Record<string, unknown> | undefined;
         maxTurns?: number | undefined;
         expectedOutcome?: string | undefined;
-    }, "kind" | "id" | "namespace" | "schemaVersion">, generator: import("../../../index.js").IdGenerator): Promise<{
+    }, "id" | "kind" | "schemaVersion" | "namespace">, generator: import("../../../index.js").IdGenerator): Promise<{
         id: string;
         messages: {
             role: string;
@@ -89,7 +89,7 @@ export declare const MultiTurnResponseSchemaV1: z.ZodObject<Omit<{
     completedAt: z.ZodNumber;
     testCaseId: z.ZodString;
     metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
-}, "kind" | "namespace" | "schemaVersion"> & {
+}, "kind" | "schemaVersion" | "namespace"> & {
     replies: z.ZodArray<z.ZodObject<{
         messageIndex: z.ZodNumber;
         startedAt: z.ZodNumber;
@@ -140,7 +140,7 @@ export declare const MultiTurnResponseSchemaV1: z.ZodObject<Omit<{
         outputTokensUsed?: number | undefined;
         inputCost?: string | undefined;
         outputCost?: string | undefined;
-    }, "kind" | "namespace" | "schemaVersion">) => {
+    }, "kind" | "schemaVersion" | "namespace">) => {
         id: string;
         testCaseId: string;
         startedAt: number;
@@ -195,7 +195,7 @@ export declare const MultiTurnResponseSchemaV1: z.ZodObject<Omit<{
         outputTokensUsed?: number | undefined;
         inputCost?: string | undefined;
         outputCost?: string | undefined;
-    }, "kind" | "id" | "namespace" | "schemaVersion">, generator: import("../../../index.js").IdGenerator): Promise<{
+    }, "id" | "kind" | "schemaVersion" | "namespace">, generator: import("../../../index.js").IdGenerator): Promise<{
         id: string;
         testCaseId: string;
         startedAt: number;
@@ -239,7 +239,7 @@ export declare const MultiTurnScoreSchemaV1: z.ZodObject<Omit<{
         readonly human: "human";
         readonly algo: "algo";
     }>;
-}, "kind" | "namespace" | "schemaVersion"> & {
+}, "kind" | "schemaVersion" | "namespace"> & {
     individualScores: z.ZodArray<z.ZodObject<{
         replyIndex: z.ZodNumber;
         value: z.ZodNumber;
@@ -258,9 +258,9 @@ export declare const MultiTurnScoreSchemaV1: z.ZodObject<Omit<{
     schemaVersion: z.ZodLiteral<1>;
 }, z.core.$strip> & {
     new: (input: Omit<{
-        value: number;
         id: string;
         responseId: string;
+        value: number;
         scoringMethod: "ai" | "human" | "algo";
         individualScores: {
             replyIndex: number;
@@ -279,10 +279,10 @@ export declare const MultiTurnScoreSchemaV1: z.ZodObject<Omit<{
         scorerAIOutputTokensUsed?: number | undefined;
         scorerAIInputCost?: string | undefined;
         scorerAIOutputCost?: string | undefined;
-    }, "kind" | "namespace" | "schemaVersion">) => {
-        value: number;
+    }, "kind" | "schemaVersion" | "namespace">) => {
         id: string;
         responseId: string;
+        value: number;
         scoringMethod: "ai" | "human" | "algo";
         individualScores: {
             replyIndex: number;
@@ -303,9 +303,9 @@ export declare const MultiTurnScoreSchemaV1: z.ZodObject<Omit<{
         scorerAIOutputCost?: string | undefined;
     };
     newWithId(input: Omit<{
-        value: number;
         id: string;
         responseId: string;
+        value: number;
         scoringMethod: "ai" | "human" | "algo";
         individualScores: {
             replyIndex: number;
@@ -324,10 +324,10 @@ export declare const MultiTurnScoreSchemaV1: z.ZodObject<Omit<{
         scorerAIOutputTokensUsed?: number | undefined;
         scorerAIInputCost?: string | undefined;
         scorerAIOutputCost?: string | undefined;
-    }, "kind" | "id" | "namespace" | "schemaVersion">, generator: import("../../../index.js").IdGenerator): Promise<{
-        value: number;
+    }, "id" | "kind" | "schemaVersion" | "namespace">, generator: import("../../../index.js").IdGenerator): Promise<{
         id: string;
         responseId: string;
+        value: number;
         scoringMethod: "ai" | "human" | "algo";
         individualScores: {
             replyIndex: number;

package/dist/benchmarks/peerbench/schema-sets/qa.v1.d.ts

@@ -6,7 +6,7 @@ export declare const QATestCaseSchemaV1: z.ZodObject<Omit<{
     schemaVersion: z.ZodNumber;
     kind: z.ZodString;
     metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
-}, "kind" | "namespace" | "schemaVersion"> & {
+}, "kind" | "schemaVersion" | "namespace"> & {
     question: z.ZodString;
     goodAnswers: z.ZodArray<z.ZodString>;
     badAnswers: z.ZodArray<z.ZodString>;
@@ -24,7 +24,7 @@ export declare const QATestCaseSchemaV1: z.ZodObject<Omit<{
         kind: "llm/qa.tc";
         schemaVersion: 1;
         metadata?: Record<string, unknown> | undefined;
-    }, "kind" | "namespace" | "schemaVersion">) => {
+    }, "kind" | "schemaVersion" | "namespace">) => {
         id: string;
         question: string;
         goodAnswers: string[];
@@ -43,7 +43,7 @@ export declare const QATestCaseSchemaV1: z.ZodObject<Omit<{
         kind: "llm/qa.tc";
         schemaVersion: 1;
         metadata?: Record<string, unknown> | undefined;
-    }, "kind" | "id" | "namespace" | "schemaVersion">, generator: import("../../../index.js").IdGenerator): Promise<{
+    }, "id" | "kind" | "schemaVersion" | "namespace">, generator: import("../../../index.js").IdGenerator): Promise<{
         id: string;
         question: string;
         goodAnswers: string[];
@@ -64,7 +64,7 @@ export declare const QAResponseSchemaV1: z.ZodObject<Omit<{
     completedAt: z.ZodNumber;
     testCaseId: z.ZodString;
     metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
-}, "kind" | "namespace" | "schemaVersion"> & {
+}, "kind" | "schemaVersion" | "namespace"> & {
     data: z.ZodString;
     modelSlug: z.ZodString;
     provider: z.ZodString;
@@ -95,7 +95,7 @@ export declare const QAResponseSchemaV1: z.ZodObject<Omit<{
         outputTokensUsed?: number | undefined;
         inputCost?: string | undefined;
         outputCost?: string | undefined;
-    }, "kind" | "namespace" | "schemaVersion">) => {
+    }, "kind" | "schemaVersion" | "namespace">) => {
         id: string;
         testCaseId: string;
         startedAt: number;
@@ -130,7 +130,7 @@ export declare const QAResponseSchemaV1: z.ZodObject<Omit<{
         outputTokensUsed?: number | undefined;
         inputCost?: string | undefined;
         outputCost?: string | undefined;
-    }, "kind" | "id" | "namespace" | "schemaVersion">, generator: import("../../../index.js").IdGenerator): Promise<{
+    }, "id" | "kind" | "schemaVersion" | "namespace">, generator: import("../../../index.js").IdGenerator): Promise<{
         id: string;
         testCaseId: string;
         startedAt: number;
@@ -164,7 +164,7 @@ export declare const QAScoreSchemaV1: z.ZodObject<Omit<{
         readonly human: "human";
         readonly algo: "algo";
     }>;
-}, "kind" | "namespace" | "schemaVersion"> & {
+}, "kind" | "schemaVersion" | "namespace"> & {
     scorerAISystemPrompt: z.ZodOptional<z.ZodString>;
     scorerAISystemPromptId: z.ZodOptional<z.ZodString>;
     scorerAIProvider: z.ZodOptional<z.ZodString>;
@@ -179,9 +179,9 @@ export declare const QAScoreSchemaV1: z.ZodObject<Omit<{
     schemaVersion: z.ZodLiteral<1>;
 }, z.core.$strip> & {
     new: (input: Omit<{
-        value: number;
         id: string;
         responseId: string;
+        value: number;
         scoringMethod: "ai" | "human" | "algo";
         namespace: "peerbench.ai";
         kind: "llm/qa.sc";
@@ -196,10 +196,10 @@ export declare const QAScoreSchemaV1: z.ZodObject<Omit<{
         scorerAIOutputTokensUsed?: number | undefined;
         scorerAIInputCost?: string | undefined;
         scorerAIOutputCost?: string | undefined;
-    }, "kind" | "namespace" | "schemaVersion">) => {
-        value: number;
+    }, "kind" | "schemaVersion" | "namespace">) => {
         id: string;
         responseId: string;
+        value: number;
         scoringMethod: "ai" | "human" | "algo";
         namespace: "peerbench.ai";
         kind: "llm/qa.sc";
@@ -216,9 +216,9 @@ export declare const QAScoreSchemaV1: z.ZodObject<Omit<{
         scorerAIOutputCost?: string | undefined;
     };
     newWithId(input: Omit<{
-        value: number;
         id: string;
         responseId: string;
+        value: number;
         scoringMethod: "ai" | "human" | "algo";
         namespace: "peerbench.ai";
         kind: "llm/qa.sc";
@@ -233,10 +233,10 @@ export declare const QAScoreSchemaV1: z.ZodObject<Omit<{
         scorerAIOutputTokensUsed?: number | undefined;
         scorerAIInputCost?: string | undefined;
         scorerAIOutputCost?: string | undefined;
-    }, "kind" | "id" | "namespace" | "schemaVersion">, generator: import("../../../index.js").IdGenerator): Promise<{
-        value: number;
+    }, "id" | "kind" | "schemaVersion" | "namespace">, generator: import("../../../index.js").IdGenerator): Promise<{
         id: string;
         responseId: string;
+        value: number;
         scoringMethod: "ai" | "human" | "algo";
         namespace: "peerbench.ai";
         kind: "llm/qa.sc";

package/dist/{chunk-TRNCF2BG.js → chunk-HBGC6BDW.js}

@@ -32,4 +32,4 @@ export {
   ExtensionLLMResponseFieldsV1,
   ExtensionLLMAsAJudgeScoreFieldsV1
 };
-//# sourceMappingURL=chunk-TRNCF2BG.js.map
+//# sourceMappingURL=chunk-HBGC6BDW.js.map

package/dist/chunk-HBGC6BDW.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/schemas/extensions/response/llm.ts","../src/schemas/extensions/score/llm-as-a-judge-scorer.ts"],"sourcesContent":["import { IdSchema } from \"@/schemas/id\";\nimport z from \"zod\";\n\n/**\n * Provides a set of fields that holds information about the LLM and its response.\n */\nexport const ExtensionLLMResponseFieldsV1 = {\n  data: z.string(),\n  modelSlug: z.string(),\n  provider: z.string(),\n  systemPromptId: IdSchema.optional(),\n\n  inputTokensUsed: z.number().optional(),\n  outputTokensUsed: z.number().optional(),\n  inputCost: z.string().optional(),\n  outputCost: z.string().optional(),\n};\n\nexport function LLMResponseFieldsV1<\n  TRawDataField extends boolean = false,\n>(params?: {\n  /**\n   * If `true` then the `data` field (which is a `z.string()`)\n   * will be included in the returned fields\n   */\n  withRawDataField?: TRawDataField;\n}) {\n  const baseFields = {\n    modelSlug: z.string(),\n    provider: z.string(),\n    systemPromptId: IdSchema.optional(),\n\n    inputTokensUsed: z.number().optional(),\n    outputTokensUsed: z.number().optional(),\n    inputCost: z.string().optional(),\n    outputCost: z.string().optional(),\n  };\n\n  const fields = {\n    ...baseFields,\n    data: undefined as z.ZodString | undefined,\n  };\n\n  if (params?.withRawDataField) {\n    fields.data = z.string();\n  }\n\n  return fields as unknown as typeof baseFields &\n    (TRawDataField extends true ? { data: z.ZodString } : {});\n}\n","import z from \"zod\";\n\n/**\n * Provides a set of fields that holds information about the LLM model\n * that was used to judge the response.\n */\nexport const ExtensionLLMAsAJudgeScoreFieldsV1 = {\n  scorerAISystemPrompt: z.string().optional(),\n  scorerAISystemPromptId: z.string().optional(),\n  scorerAIProvider: z.string().optional(),\n  scorerAIModelSlug: z.string().optional(),\n  scorerAIInputTokensUsed: z.number().optional(),\n  scorerAIOutputTokensUsed: z.number().optional(),\n  scorerAIInputCost: z.string().optional(),\n  scorerAIOutputCost: z.string().optional(),\n};\n"],"mappings":";;;;;AACA,OAAO,OAAO;AAKP,IAAM,+BAA+B;AAAA,EAC1C,MAAM,EAAE,OAAO;AAAA,EACf,WAAW,EAAE,OAAO;AAAA,EACpB,UAAU,EAAE,OAAO;AAAA,EACnB,gBAAgB,SAAS,SAAS;AAAA,EAElC,iBAAiB,EAAE,OAAO,EAAE,SAAS;AAAA,EACrC,kBAAkB,EAAE,OAAO,EAAE,SAAS;AAAA,EACtC,WAAW,EAAE,OAAO,EAAE,SAAS;AAAA,EAC/B,YAAY,EAAE,OAAO,EAAE,SAAS;AAClC;;;AChBA,OAAOA,QAAO;AAMP,IAAM,oCAAoC;AAAA,EAC/C,sBAAsBA,GAAE,OAAO,EAAE,SAAS;AAAA,EAC1C,wBAAwBA,GAAE,OAAO,EAAE,SAAS;AAAA,EAC5C,kBAAkBA,GAAE,OAAO,EAAE,SAAS;AAAA,EACtC,mBAAmBA,GAAE,OAAO,EAAE,SAAS;AAAA,EACvC,yBAAyBA,GAAE,OAAO,EAAE,SAAS;AAAA,EAC7C,0BAA0BA,GAAE,OAAO,EAAE,SAAS;AAAA,EAC9C,mBAAmBA,GAAE,OAAO,EAAE,SAAS;AAAA,EACvC,oBAAoBA,GAAE,OAAO,EAAE,SAAS;AAC1C;","names":["z"]}

package/dist/dev.d.ts

@@ -0,0 +1,22 @@
+type PeerBenchTestCase<TKind extends string, TSchemaVersion extends number> = {
+    id: string;
+    kind: TKind;
+    schemaVersion: TSchemaVersion;
+    [key: string]: unknown;
+};
+type PeerBenchResponse<TKind extends string, TSchemaVersion extends number> = {
+    id: string;
+    kind: TKind;
+    schemaVersion: TSchemaVersion;
+    testCaseId: string;
+    [key: string]: unknown;
+};
+type PeerBenchScore<TKind extends string, TSchemaVersion extends number> = {
+    id: string;
+    kind: TKind;
+    schemaVersion: TSchemaVersion;
+    responseId: string;
+};
+declare function createTestCase<TKind extends string, TSchemaVersion extends number>(kind: TKind, schemaVersion: TSchemaVersion, tc: Omit<PeerBenchTestCase<TKind, TSchemaVersion>, "kind" | "schemaVersion">): PeerBenchTestCase<TKind, TSchemaVersion>;
+declare function createResponse<TKind extends string, TSchemaVersion extends number>(kind: TKind, schemaVersion: TSchemaVersion, rs: Omit<PeerBenchResponse<TKind, TSchemaVersion>, "kind" | "schemaVersion">): PeerBenchResponse<TKind, TSchemaVersion>;
+declare function createScore<TKind extends string, TSchemaVersion extends number>(kind: TKind, schemaVersion: TSchemaVersion, sc: Omit<PeerBenchScore<TKind, TSchemaVersion>, "kind" | "schemaVersion">): PeerBenchScore<TKind, TSchemaVersion>;

package/dist/providers/ai-sdk.d.ts

@@ -0,0 +1,24 @@
+import type { LanguageModelV3 } from "@ai-sdk/provider";
+import { RateLimiter } from "../utils";
+import { AbstractProvider } from "./abstract";
+import { type CallableLLM } from "./callables/llm";
+declare const AISdkProvider_base: (new () => AbstractProvider & {
+    readonly kind: "peerbench.ai/llm/ai-sdk";
+}) & {
+    readonly kind: "peerbench.ai/llm/ai-sdk";
+};
+export declare class AISdkProvider extends AISdkProvider_base {
+    private rateLimiter;
+    private maxRetries;
+    constructor(config?: AISdkProviderConfig);
+    model(config: AISdkModelConfig): CallableLLM<AISdkProvider>;
+}
+type AISdkProviderConfig = {
+    rateLimiter?: RateLimiter;
+    maxRetries?: number;
+};
+type AISdkModelConfig = {
+    aiSdkModel: LanguageModelV3;
+    stream?: boolean;
+};
+export {};

package/dist/providers/callables/llm.d.ts

@@ -34,5 +34,8 @@ export type LLMResponse = ProviderResponse<string> & {
      * Time taken to receive the first token.
      */
     timeToFirstToken?: number;
+    /**
+     * Additional metadata
+     */
     metadata?: Record<string, unknown>;
 };

package/dist/providers/index.d.ts

@@ -4,3 +4,4 @@ export * from "./abstract";
 export * from "./mastra";
 export * from "./openai";
 export * from "./openrouter";
+export * from "./ai-sdk";

package/dist/providers/index.js

@@ -264,7 +264,123 @@ var OpenRouterProvider = class extends AbstractProvider.withKind(
     await this.modelsCachePromise;
   }
 };
+
+// src/providers/ai-sdk.ts
+import { generateText, streamText } from "ai";
+function toModelMessages(messages) {
+  return messages.filter(
+    (m) => m.role === "system" || m.role === "user" || m.role === "assistant"
+  ).map((m) => ({
+    role: m.role,
+    content: typeof m.content === "string" ? m.content : JSON.stringify(m.content ?? "")
+  }));
+}
+var aiModuleCache = null;
+async function buildOutputOption(responseFormat) {
+  if (!responseFormat || responseFormat.type === "text") {
+    return void 0;
+  }
+  if (!aiModuleCache) {
+    aiModuleCache = await import("ai");
+  }
+  const Output = aiModuleCache["Output"];
+  const jsonSchema = aiModuleCache["jsonSchema"];
+  if (!Output || !jsonSchema) {
+    return void 0;
+  }
+  if (responseFormat.type === "json_schema") {
+    return Output.object({
+      schema: jsonSchema(responseFormat.json_schema.schema)
+    });
+  }
+  if (responseFormat.type === "json_object") {
+    return Output.json();
+  }
+  return void 0;
+}
+var AISdkProvider = class extends AbstractProvider.withKind(
+  `${PEERBENCH_NAMESPACE}/llm/ai-sdk`
+) {
+  rateLimiter;
+  maxRetries;
+  constructor(config) {
+    super();
+    this.maxRetries = config?.maxRetries ?? 2;
+    this.rateLimiter = config?.rateLimiter ?? new RateLimiter({
+      maxWeight: 20,
+      timeWindow: 3e3
+    });
+  }
+  model(config) {
+    const slug = `${config.aiSdkModel.provider}/${config.aiSdkModel.modelId}`;
+    return {
+      slug,
+      provider: this,
+      forward: async (args) => {
+        const messages = toModelMessages(args.messages);
+        const output = await buildOutputOption(args.responseFormat);
+        const baseOptions = {
+          model: config.aiSdkModel,
+          messages,
+          temperature: args.temperature,
+          maxOutputTokens: args.maxTokens,
+          abortSignal: args.abortSignal,
+          maxRetries: this.maxRetries
+        };
+        if (config.stream) {
+          let startedAt2;
+          const result2 = await this.rateLimiter.execute(
+            async () => {
+              startedAt2 = Date.now();
+              return streamText({
+                ...baseOptions,
+                ...output !== void 0 ? { output } : {}
+              });
+            },
+            { signal: args.abortSignal }
+          );
+          let text = "";
+          let firstTokenAt;
+          for await (const chunk of result2.textStream) {
+            if (firstTokenAt === void 0) {
+              firstTokenAt = Date.now();
+            }
+            text += chunk;
+          }
+          const usage = await result2.usage;
+          return {
+            data: text,
+            startedAt: startedAt2,
+            completedAt: Date.now(),
+            inputTokensUsed: usage.inputTokens,
+            outputTokensUsed: usage.outputTokens,
+            timeToFirstToken: firstTokenAt !== void 0 ? firstTokenAt - startedAt2 : void 0
+          };
+        }
+        let startedAt;
+        const result = await this.rateLimiter.execute(
+          async () => {
+            startedAt = Date.now();
+            return generateText({
+              ...baseOptions,
+              ...output !== void 0 ? { output } : {}
+            });
+          },
+          { signal: args.abortSignal }
+        );
+        return {
+          data: result.text,
+          startedAt,
+          completedAt: Date.now(),
+          inputTokensUsed: result.usage.inputTokens,
+          outputTokensUsed: result.usage.outputTokens
+        };
+      }
+    };
+  }
+};
 export {
+  AISdkProvider,
   AbstractProvider,
   MastraProvider,
   OpenAIProvider,

package/dist/providers/index.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/providers/abstract.ts","../../src/providers/mastra.ts","../../src/providers/openai.ts","../../src/providers/openrouter.ts"],"sourcesContent":["export abstract class AbstractProvider {\n  readonly kind: string;\n\n  constructor() {\n    this.kind = (this.constructor as any).kind;\n    if (!this.kind)\n      throw new Error(\n        `${this.constructor.name} must define \"static readonly kind\" as a constant property.`\n      );\n  }\n  static withKind<\n    TKind extends string,\n    TThis extends abstract new (...args: any[]) => AbstractProvider,\n  >(this: TThis, kind: TKind) {\n    const base = this as unknown as new (...args: any[]) => any;\n    const derived = class extends base {\n      static readonly kind: TKind = kind;\n      declare readonly kind: TKind;\n    };\n\n    return derived as unknown as (new () => InstanceType<TThis> & {\n      readonly kind: TKind;\n    }) & {\n      readonly kind: TKind;\n    };\n  }\n}\n\nexport type ProviderResponse<TData = unknown> = {\n  startedAt: number;\n  completedAt: number;\n  data: TData;\n};\n","import { PEERBENCH_NAMESPACE } from \"@/constants\";\nimport { AbstractProvider } from \"./abstract\";\nimport {\n  type CallableLLM,\n  type LLMResponse,\n  type CallableLLMForwardArgs,\n} from \"./callables/llm\";\nimport { MastraClient, type GetAgentResponse } from \"@mastra/client-js\";\nimport type { RequestContext } from \"@mastra/core/request-context\";\nimport { AgentMemoryOption } from \"@mastra/core/agent\";\nimport { ProviderOptions } from \"@mastra/core/dist/llm/model/provider-options\";\nimport { CoreMessage } from \"@mastra/core/llm\";\n\nexport class MastraProvider extends AbstractProvider.withKind(\n  `${PEERBENCH_NAMESPACE}/llm/mastra`\n) {\n  private readonly endpoint: string;\n  private readonly authToken?: string;\n  private client: MastraClient;\n\n  constructor(params: { endpoint: string; authToken?: string }) {\n    super();\n    this.endpoint = params.endpoint;\n    this.authToken = params.authToken;\n    this.client = new MastraClient({\n      baseUrl: this.endpoint,\n      headers: this.authToken\n        ? {\n          Authorization: `Bearer ${this.authToken}`,\n        }\n        : undefined,\n    });\n  }\n\n  async getAgentInfo(args: {\n    agentId: string;\n    requestContext?: RequestContext;\n  }) {\n    return await this.client\n      .getAgent(args.agentId)\n      .details(args.requestContext);\n  }\n\n  async getAgents(params?: {\n    requestContext?: RequestContext;\n    partial?: boolean;\n  }): Promise<Record<string, GetAgentResponse>> {\n    return this.client.listAgents(params?.requestContext, params?.partial);\n  }\n\n  agent(config: {\n    agentId: string;\n    memory?: AgentMemoryOption;\n    requestContext?: RequestContext;\n    providerOptions?: ProviderOptions;\n  }): CallableLLM<MastraProvider> {\n    return {\n      slug: config.agentId,\n      provider: this,\n      forward: async (\n        args: CallableLLMForwardArgs\n      ): Promise<LLMResponse> => {\n        const apiMessages = args.messages\n          .filter((m) => m.role === \"user\" || m.role === \"assistant\")\n          .map<CoreMessage>((m) => ({\n            role: m.role,\n            content: String(m.content ?? \"\"),\n          }));\n\n        const agent = this.client.getAgent(config.agentId);\n        const startedAt = Date.now();\n\n        const response = await agent.stream(\n          apiMessages,\n          {\n            providerOptions: config.providerOptions,\n            memory: config.memory,\n            requestContext: config.requestContext,\n          }\n        );\n\n        let text = \"\";\n        let firstTokenAt: number | undefined;\n        let inputTokensUsed: number | undefined;\n        let outputTokensUsed: number | undefined;\n        const metadata: Record<string, unknown> = {};\n\n        await response.processDataStream({\n          onChunk: async (chunk) => {\n            if (chunk.type === \"text-delta\") {\n              if (firstTokenAt === undefined) {\n                firstTokenAt = Date.now();\n              }\n              const payload = chunk.payload as { text?: string };\n              text += payload.text ?? \"\";\n            }\n\n            if (chunk.type === \"finish\") {\n              const payload = chunk.payload as {\n                output?: {\n                  usage?: {\n                    inputTokens?: number;\n                    outputTokens?: number;\n                  };\n                };\n              };\n              inputTokensUsed = payload.output?.usage?.inputTokens;\n              outputTokensUsed = payload.output?.usage?.outputTokens;\n            }\n          },\n        });\n\n        return {\n          data: text,\n          startedAt,\n          completedAt: Date.now(),\n          inputTokensUsed,\n          outputTokensUsed,\n          timeToFirstToken:\n            firstTokenAt !== undefined ? firstTokenAt - startedAt : undefined,\n          metadata:\n            Object.keys(metadata).length > 0 ? metadata : undefined,\n        };\n      },\n    };\n  }\n}\n","import { RateLimiter } from \"@/utils\";\nimport OpenAI, { APIError } from \"openai\";\nimport { AbstractProvider } from \"./abstract\";\nimport { PEERBENCH_NAMESPACE } from \"@/constants\";\nimport {\n  type CallableLLM,\n  type LLMResponse,\n  type CallableLLMForwardArgs,\n} from \"./callables/llm\";\n\n/**\n * Provider implementation that uses OpenAI SDK. It can be used with\n * any OpenAI compatible API.\n *\n * @example\n * ```ts\n * const provider = new OpenAIProvider({\n *   apiKey: \"sk-1234567890\",\n *   baseURL: \"https://openrouter.ai/api/v1\",\n * });\n * \n * const model = provider.model({ model: \"gpt-4o\" });\n * \n * const response = await model.forward({\n *   messages: [{ role: \"user\", content: \"Hello, how are you?\" }],\n * });\n * \n * console.log(response.data);\n * ```\n */\nexport class OpenAIProvider extends AbstractProvider.withKind(\n  `${PEERBENCH_NAMESPACE}/llm/openai`\n) {\n  private client: OpenAI;\n  private rateLimiter: RateLimiter;\n  private maxRetries: number;\n\n  constructor(config: {\n    apiKey: string;\n    baseURL: string;\n    maxRetries?: number;\n    timeout?: number;\n    rateLimiter?: RateLimiter;\n  }) {\n    super();\n    this.maxRetries = config.maxRetries ?? 3;\n    this.rateLimiter =\n      config.rateLimiter ??\n      new RateLimiter({\n        maxWeight: 20,\n        timeWindow: 3_000,\n      });\n\n    this.client = new OpenAI({\n      baseURL: config.baseURL,\n      apiKey: config.apiKey,\n      timeout: config.timeout,\n      dangerouslyAllowBrowser: true,\n    });\n  }\n\n  model(config: { model: string }): CallableLLM<OpenAIProvider> {\n    return {\n      slug: config.model,\n      provider: this,\n      forward: async (args: CallableLLMForwardArgs): Promise<LLMResponse> => {\n        let retryCount = this.maxRetries;\n        while (retryCount > 0) {\n          let startedAt: Date = new Date();\n\n          try {\n            const response = await this.rateLimiter.execute(\n              async () => {\n                startedAt = new Date();\n                return await this.client.chat.completions.create(\n                  {\n                    model: config.model,\n                    messages: args.messages,\n                    temperature: args.temperature,\n                    response_format: args.responseFormat,\n                  },\n                  { signal: args.abortSignal }\n                );\n              },\n              { signal: args.abortSignal }\n            );\n\n            if (\"error\" in response) {\n              const err = response.error as any;\n              throw new Error(\n                `${err.message} - Code ${err.code} - ${JSON.stringify(err)}`\n              );\n            }\n\n            if (!response?.choices?.[0]?.message?.content) {\n              throw new Error(\"No content returned from the model\");\n            }\n\n            return {\n              data: response.choices[0].message.content,\n              inputTokensUsed: response?.usage?.prompt_tokens,\n              outputTokensUsed: response?.usage?.completion_tokens,\n              startedAt: startedAt.getTime(),\n              completedAt: Date.now(),\n            };\n          } catch (err) {\n            if (err instanceof APIError && err.status === 401) {\n              throw new Error(`Invalid credentials provided`, { cause: err });\n            }\n\n            retryCount--;\n\n            if (err instanceof SyntaxError) {\n              console.debug(err);\n              continue;\n            }\n\n            if (retryCount !== 0) {\n              continue;\n            }\n\n            throw new Error(\n              `Failed to forward prompt to the model: ${err instanceof Error ? err.message : err}`,\n              { cause: err }\n            );\n          }\n        }\n\n        throw new Error(\n          `Failed to forward prompt to the model: Max retries reached`,\n          { cause: new Error(\"Max retries reached\") }\n        );\n      },\n    };\n  }\n}\n","import { AbstractProvider } from \"./abstract\";\nimport {\n  type CallableLLM,\n  type LLMResponse,\n  type CallableLLMForwardArgs,\n} from \"./callables/llm\";\nimport { RateLimiter } from \"@/utils\";\nimport { OpenAIProvider } from \"./openai\";\nimport { PEERBENCH_NAMESPACE } from \"@/constants\";\nimport Decimal from \"decimal.js\";\nimport axios from \"axios\";\n\nconst baseURL = \"https://openrouter.ai/api/v1\";\nconst MODELS_CACHE_TTL = 1000 * 60 * 60 * 24; // 24 hours\n\nexport class OpenRouterProvider extends AbstractProvider.withKind(\n  `${PEERBENCH_NAMESPACE}/llm/openrouter.ai`\n) {\n  private models: ModelsResponse | undefined = undefined;\n  private modelsCachePromise: Promise<ModelsResponse | undefined> =\n    Promise.resolve(undefined);\n  private modelsUpdatedAt = 0;\n  private openAIProvider: OpenAIProvider;\n\n  constructor(config: {\n    apiKey: string;\n    maxRetries?: number;\n    timeout?: number;\n    rateLimiter?: RateLimiter;\n  }) {\n    super();\n    this.openAIProvider = new OpenAIProvider({\n      baseURL,\n      apiKey: config.apiKey,\n      maxRetries: config.maxRetries,\n      timeout: config.timeout,\n      rateLimiter: config.rateLimiter,\n    });\n  }\n\n  model(config: { model: string }): CallableLLM<OpenRouterProvider> {\n    const openAICallable = this.openAIProvider.model({ model: config.model });\n    this.updateModelsCache().catch(() => { });\n\n    return {\n      slug: config.model,\n      provider: this,\n      forward: async (\n        args: CallableLLMForwardArgs\n      ): Promise<LLMResponse> => {\n        const response = await openAICallable.forward(args);\n\n        const modelInfo = this.models?.data.find(\n          (m) => m.id === config.model\n        );\n        let inputCost: string | undefined = undefined;\n        let outputCost: string | undefined = undefined;\n\n        if (modelInfo !== undefined) {\n          if (response.inputTokensUsed !== undefined) {\n            inputCost = new Decimal(modelInfo.pricing.prompt)\n              .mul(response.inputTokensUsed)\n              .toFixed(10);\n          }\n          if (response.outputTokensUsed !== undefined) {\n            outputCost = new Decimal(modelInfo.pricing.completion)\n              .mul(response.outputTokensUsed)\n              .toFixed(10);\n          }\n        }\n\n        return { ...response, inputCost, outputCost };\n      },\n    };\n  }\n\n  private async updateModelsCache() {\n    this.modelsCachePromise = this.modelsCachePromise\n      .then(async () => {\n        if (\n          this.models !== undefined &&\n          Date.now() - this.modelsUpdatedAt < MODELS_CACHE_TTL\n        ) {\n          return this.models;\n        }\n\n        return axios\n          .get<ModelsResponse>(`${baseURL}/models`)\n          .then((res) => res.data)\n          .then((data) => {\n            data = {\n              data: data.data.filter(\n                (m) =>\n                  m.architecture.input_modalities.includes(\"text\") &&\n                  m.architecture.output_modalities.includes(\"text\") &&\n                  ![\n                    \"morph/morph-v3-large\",\n                    \"morph/morph-v3-fast\",\n                    \"relace/relace-apply-3\",\n                  ].includes(m.id)\n              ),\n            };\n\n            this.models = data;\n            this.modelsUpdatedAt = Date.now();\n\n            return data;\n          });\n      })\n      .catch(() => undefined);\n\n    await this.modelsCachePromise;\n  }\n}\n\ntype PutModality = \"text\" | \"image\" | \"file\" | \"audio\";\ntype Modality = \"text->text\" | \"text+image->text\" | \"text+image->text+image\";\n\ntype ModelInfo = {\n  readonly id: string;\n  readonly canonical_slug: string;\n  readonly hugging_face_id: null | string;\n  readonly name: string;\n  readonly created: number;\n  readonly description: string;\n  readonly context_length: number;\n  readonly architecture: {\n    readonly modality: Modality;\n    readonly input_modalities: PutModality[];\n    readonly output_modalities: PutModality[];\n    readonly instruct_type: null | string;\n  };\n  readonly pricing: {\n    readonly prompt: string;\n    readonly completion: string;\n    readonly request?: string;\n    readonly image?: string;\n    readonly web_search?: string;\n    readonly internal_reasoning?: string;\n    readonly input_cache_read?: string;\n    readonly input_cache_write?: string;\n    readonly audio?: string;\n  };\n};\n\ntype ModelsResponse = {\n  data: ModelInfo[];\n};\n"],"mappings":";;;;;;;;;AAAO,IAAe,mBAAf,MAAgC;AAAA,EAC5B;AAAA,EAET,cAAc;AACZ,SAAK,OAAQ,KAAK,YAAoB;AACtC,QAAI,CAAC,KAAK;AACR,YAAM,IAAI;AAAA,QACR,GAAG,KAAK,YAAY,IAAI;AAAA,MAC1B;AAAA,EACJ;AAAA,EACA,OAAO,SAGQ,MAAa;AAC1B,UAAM,OAAO;AACb,UAAM,UAAU,cAAc,KAAK;AAAA,MACjC,OAAgB,OAAc;AAAA,IAEhC;AAEA,WAAO;AAAA,EAKT;AACF;;;ACnBA,SAAS,oBAA2C;AAM7C,IAAM,iBAAN,cAA6B,iBAAiB;AAAA,EACnD,GAAG,mBAAmB;AACxB,EAAE;AAAA,EACiB;AAAA,EACA;AAAA,EACT;AAAA,EAER,YAAY,QAAkD;AAC5D,UAAM;AACN,SAAK,WAAW,OAAO;AACvB,SAAK,YAAY,OAAO;AACxB,SAAK,SAAS,IAAI,aAAa;AAAA,MAC7B,SAAS,KAAK;AAAA,MACd,SAAS,KAAK,YACV;AAAA,QACA,eAAe,UAAU,KAAK,SAAS;AAAA,MACzC,IACE;AAAA,IACN,CAAC;AAAA,EACH;AAAA,EAEA,MAAM,aAAa,MAGhB;AACD,WAAO,MAAM,KAAK,OACf,SAAS,KAAK,OAAO,EACrB,QAAQ,KAAK,cAAc;AAAA,EAChC;AAAA,EAEA,MAAM,UAAU,QAG8B;AAC5C,WAAO,KAAK,OAAO,WAAW,QAAQ,gBAAgB,QAAQ,OAAO;AAAA,EACvE;AAAA,EAEA,MAAM,QAK0B;AAC9B,WAAO;AAAA,MACL,MAAM,OAAO;AAAA,MACb,UAAU;AAAA,MACV,SAAS,OACP,SACyB;AACzB,cAAM,cAAc,KAAK,SACtB,OAAO,CAAC,MAAM,EAAE,SAAS,UAAU,EAAE,SAAS,WAAW,EACzD,IAAiB,CAAC,OAAO;AAAA,UACxB,MAAM,EAAE;AAAA,UACR,SAAS,OAAO,EAAE,WAAW,EAAE;AAAA,QACjC,EAAE;AAEJ,cAAM,QAAQ,KAAK,OAAO,SAAS,OAAO,OAAO;AACjD,cAAM,YAAY,KAAK,IAAI;AAE3B,cAAM,WAAW,MAAM,MAAM;AAAA,UAC3B;AAAA,UACA;AAAA,YACE,iBAAiB,OAAO;AAAA,YACxB,QAAQ,OAAO;AAAA,YACf,gBAAgB,OAAO;AAAA,UACzB;AAAA,QACF;AAEA,YAAI,OAAO;AACX,YAAI;AACJ,YAAI;AACJ,YAAI;AACJ,cAAM,WAAoC,CAAC;AAE3C,cAAM,SAAS,kBAAkB;AAAA,UAC/B,SAAS,OAAO,UAAU;AACxB,gBAAI,MAAM,SAAS,cAAc;AAC/B,kBAAI,iBAAiB,QAAW;AAC9B,+BAAe,KAAK,IAAI;AAAA,cAC1B;AACA,oBAAM,UAAU,MAAM;AACtB,sBAAQ,QAAQ,QAAQ;AAAA,YAC1B;AAEA,gBAAI,MAAM,SAAS,UAAU;AAC3B,oBAAM,UAAU,MAAM;AAQtB,gCAAkB,QAAQ,QAAQ,OAAO;AACzC,iCAAmB,QAAQ,QAAQ,OAAO;AAAA,YAC5C;AAAA,UACF;AAAA,QACF,CAAC;AAED,eAAO;AAAA,UACL,MAAM;AAAA,UACN;AAAA,UACA,aAAa,KAAK,IAAI;AAAA,UACtB;AAAA,UACA;AAAA,UACA,kBACE,iBAAiB,SAAY,eAAe,YAAY;AAAA,UAC1D,UACE,OAAO,KAAK,QAAQ,EAAE,SAAS,IAAI,WAAW;AAAA,QAClD;AAAA,MACF;AAAA,IACF;AAAA,EACF;AACF;;;AC7HA,OAAO,UAAU,gBAAgB;AA6B1B,IAAM,iBAAN,cAA6B,iBAAiB;AAAA,EACnD,GAAG,mBAAmB;AACxB,EAAE;AAAA,EACQ;AAAA,EACA;AAAA,EACA;AAAA,EAER,YAAY,QAMT;AACD,UAAM;AACN,SAAK,aAAa,OAAO,cAAc;AACvC,SAAK,cACH,OAAO,eACP,IAAI,YAAY;AAAA,MACd,WAAW;AAAA,MACX,YAAY;AAAA,IACd,CAAC;AAEH,SAAK,SAAS,IAAI,OAAO;AAAA,MACvB,SAAS,OAAO;AAAA,MAChB,QAAQ,OAAO;AAAA,MACf,SAAS,OAAO;AAAA,MAChB,yBAAyB;AAAA,IAC3B,CAAC;AAAA,EACH;AAAA,EAEA,MAAM,QAAwD;AAC5D,WAAO;AAAA,MACL,MAAM,OAAO;AAAA,MACb,UAAU;AAAA,MACV,SAAS,OAAO,SAAuD;AACrE,YAAI,aAAa,KAAK;AACtB,eAAO,aAAa,GAAG;AACrB,cAAI,YAAkB,oBAAI,KAAK;AAE/B,cAAI;AACF,kBAAM,WAAW,MAAM,KAAK,YAAY;AAAA,cACtC,YAAY;AACV,4BAAY,oBAAI,KAAK;AACrB,uBAAO,MAAM,KAAK,OAAO,KAAK,YAAY;AAAA,kBACxC;AAAA,oBACE,OAAO,OAAO;AAAA,oBACd,UAAU,KAAK;AAAA,oBACf,aAAa,KAAK;AAAA,oBAClB,iBAAiB,KAAK;AAAA,kBACxB;AAAA,kBACA,EAAE,QAAQ,KAAK,YAAY;AAAA,gBAC7B;AAAA,cACF;AAAA,cACA,EAAE,QAAQ,KAAK,YAAY;AAAA,YAC7B;AAEA,gBAAI,WAAW,UAAU;AACvB,oBAAM,MAAM,SAAS;AACrB,oBAAM,IAAI;AAAA,gBACR,GAAG,IAAI,OAAO,WAAW,IAAI,IAAI,MAAM,KAAK,UAAU,GAAG,CAAC;AAAA,cAC5D;AAAA,YACF;AAEA,gBAAI,CAAC,UAAU,UAAU,CAAC,GAAG,SAAS,SAAS;AAC7C,oBAAM,IAAI,MAAM,oCAAoC;AAAA,YACtD;AAEA,mBAAO;AAAA,cACL,MAAM,SAAS,QAAQ,CAAC,EAAE,QAAQ;AAAA,cAClC,iBAAiB,UAAU,OAAO;AAAA,cAClC,kBAAkB,UAAU,OAAO;AAAA,cACnC,WAAW,UAAU,QAAQ;AAAA,cAC7B,aAAa,KAAK,IAAI;AAAA,YACxB;AAAA,UACF,SAAS,KAAK;AACZ,gBAAI,eAAe,YAAY,IAAI,WAAW,KAAK;AACjD,oBAAM,IAAI,MAAM,gCAAgC,EAAE,OAAO,IAAI,CAAC;AAAA,YAChE;AAEA;AAEA,gBAAI,eAAe,aAAa;AAC9B,sBAAQ,MAAM,GAAG;AACjB;AAAA,YACF;AAEA,gBAAI,eAAe,GAAG;AACpB;AAAA,YACF;AAEA,kBAAM,IAAI;AAAA,cACR,0CAA0C,eAAe,QAAQ,IAAI,UAAU,GAAG;AAAA,cAClF,EAAE,OAAO,IAAI;AAAA,YACf;AAAA,UACF;AAAA,QACF;AAEA,cAAM,IAAI;AAAA,UACR;AAAA,UACA,EAAE,OAAO,IAAI,MAAM,qBAAqB,EAAE;AAAA,QAC5C;AAAA,MACF;AAAA,IACF;AAAA,EACF;AACF;;;AC9HA,OAAO,aAAa;AACpB,OAAO,WAAW;AAElB,IAAM,UAAU;AAChB,IAAM,mBAAmB,MAAO,KAAK,KAAK;AAEnC,IAAM,qBAAN,cAAiC,iBAAiB;AAAA,EACvD,GAAG,mBAAmB;AACxB,EAAE;AAAA,EACQ,SAAqC;AAAA,EACrC,qBACN,QAAQ,QAAQ,MAAS;AAAA,EACnB,kBAAkB;AAAA,EAClB;AAAA,EAER,YAAY,QAKT;AACD,UAAM;AACN,SAAK,iBAAiB,IAAI,eAAe;AAAA,MACvC;AAAA,MACA,QAAQ,OAAO;AAAA,MACf,YAAY,OAAO;AAAA,MACnB,SAAS,OAAO;AAAA,MAChB,aAAa,OAAO;AAAA,IACtB,CAAC;AAAA,EACH;AAAA,EAEA,MAAM,QAA4D;AAChE,UAAM,iBAAiB,KAAK,eAAe,MAAM,EAAE,OAAO,OAAO,MAAM,CAAC;AACxE,SAAK,kBAAkB,EAAE,MAAM,MAAM;AAAA,IAAE,CAAC;AAExC,WAAO;AAAA,MACL,MAAM,OAAO;AAAA,MACb,UAAU;AAAA,MACV,SAAS,OACP,SACyB;AACzB,cAAM,WAAW,MAAM,eAAe,QAAQ,IAAI;AAElD,cAAM,YAAY,KAAK,QAAQ,KAAK;AAAA,UAClC,CAAC,MAAM,EAAE,OAAO,OAAO;AAAA,QACzB;AACA,YAAI,YAAgC;AACpC,YAAI,aAAiC;AAErC,YAAI,cAAc,QAAW;AAC3B,cAAI,SAAS,oBAAoB,QAAW;AAC1C,wBAAY,IAAI,QAAQ,UAAU,QAAQ,MAAM,EAC7C,IAAI,SAAS,eAAe,EAC5B,QAAQ,EAAE;AAAA,UACf;AACA,cAAI,SAAS,qBAAqB,QAAW;AAC3C,yBAAa,IAAI,QAAQ,UAAU,QAAQ,UAAU,EAClD,IAAI,SAAS,gBAAgB,EAC7B,QAAQ,EAAE;AAAA,UACf;AAAA,QACF;AAEA,eAAO,EAAE,GAAG,UAAU,WAAW,WAAW;AAAA,MAC9C;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAc,oBAAoB;AAChC,SAAK,qBAAqB,KAAK,mBAC5B,KAAK,YAAY;AAChB,UACE,KAAK,WAAW,UAChB,KAAK,IAAI,IAAI,KAAK,kBAAkB,kBACpC;AACA,eAAO,KAAK;AAAA,MACd;AAEA,aAAO,MACJ,IAAoB,GAAG,OAAO,SAAS,EACvC,KAAK,CAAC,QAAQ,IAAI,IAAI,EACtB,KAAK,CAAC,SAAS;AACd,eAAO;AAAA,UACL,MAAM,KAAK,KAAK;AAAA,YACd,CAAC,MACC,EAAE,aAAa,iBAAiB,SAAS,MAAM,KAC/C,EAAE,aAAa,kBAAkB,SAAS,MAAM,KAChD,CAAC;AAAA,cACC;AAAA,cACA;AAAA,cACA;AAAA,YACF,EAAE,SAAS,EAAE,EAAE;AAAA,UACnB;AAAA,QACF;AAEA,aAAK,SAAS;AACd,aAAK,kBAAkB,KAAK,IAAI;AAEhC,eAAO;AAAA,MACT,CAAC;AAAA,IACL,CAAC,EACA,MAAM,MAAM,MAAS;AAExB,UAAM,KAAK;AAAA,EACb;AACF;","names":[]}
+{"version":3,"sources":["../../src/providers/abstract.ts","../../src/providers/mastra.ts","../../src/providers/openai.ts","../../src/providers/openrouter.ts","../../src/providers/ai-sdk.ts"],"sourcesContent":["export abstract class AbstractProvider {\n  readonly kind: string;\n\n  constructor() {\n    this.kind = (this.constructor as any).kind;\n    if (!this.kind)\n      throw new Error(\n        `${this.constructor.name} must define \"static readonly kind\" as a constant property.`\n      );\n  }\n  static withKind<\n    TKind extends string,\n    TThis extends abstract new (...args: any[]) => AbstractProvider,\n  >(this: TThis, kind: TKind) {\n    const base = this as unknown as new (...args: any[]) => any;\n    const derived = class extends base {\n      static readonly kind: TKind = kind;\n      declare readonly kind: TKind;\n    };\n\n    return derived as unknown as (new () => InstanceType<TThis> & {\n      readonly kind: TKind;\n    }) & {\n      readonly kind: TKind;\n    };\n  }\n}\n\nexport type ProviderResponse<TData = unknown> = {\n  startedAt: number;\n  completedAt: number;\n  data: TData;\n};\n","import { PEERBENCH_NAMESPACE } from \"@/constants\";\nimport { AbstractProvider } from \"./abstract\";\nimport {\n  type CallableLLM,\n  type LLMResponse,\n  type CallableLLMForwardArgs,\n} from \"./callables/llm\";\nimport { MastraClient, type GetAgentResponse } from \"@mastra/client-js\";\nimport type { RequestContext } from \"@mastra/core/request-context\";\nimport { AgentMemoryOption } from \"@mastra/core/agent\";\nimport { ProviderOptions } from \"@mastra/core/dist/llm/model/provider-options\";\nimport { CoreMessage } from \"@mastra/core/llm\";\n\nexport class MastraProvider extends AbstractProvider.withKind(\n  `${PEERBENCH_NAMESPACE}/llm/mastra`\n) {\n  private readonly endpoint: string;\n  private readonly authToken?: string;\n  private client: MastraClient;\n\n  constructor(params: { endpoint: string; authToken?: string }) {\n    super();\n    this.endpoint = params.endpoint;\n    this.authToken = params.authToken;\n    this.client = new MastraClient({\n      baseUrl: this.endpoint,\n      headers: this.authToken\n        ? {\n          Authorization: `Bearer ${this.authToken}`,\n        }\n        : undefined,\n    });\n  }\n\n  async getAgentInfo(args: {\n    agentId: string;\n    requestContext?: RequestContext;\n  }) {\n    return await this.client\n      .getAgent(args.agentId)\n      .details(args.requestContext);\n  }\n\n  async getAgents(params?: {\n    requestContext?: RequestContext;\n    partial?: boolean;\n  }): Promise<Record<string, GetAgentResponse>> {\n    return this.client.listAgents(params?.requestContext, params?.partial);\n  }\n\n  agent(config: {\n    agentId: string;\n    memory?: AgentMemoryOption;\n    requestContext?: RequestContext;\n    providerOptions?: ProviderOptions;\n  }): CallableLLM<MastraProvider> {\n    return {\n      slug: config.agentId,\n      provider: this,\n      forward: async (\n        args: CallableLLMForwardArgs\n      ): Promise<LLMResponse> => {\n        const apiMessages = args.messages\n          .filter((m) => m.role === \"user\" || m.role === \"assistant\")\n          .map<CoreMessage>((m) => ({\n            role: m.role,\n            content: String(m.content ?? \"\"),\n          }));\n\n        const agent = this.client.getAgent(config.agentId);\n        const startedAt = Date.now();\n\n        const response = await agent.stream(\n          apiMessages,\n          {\n            providerOptions: config.providerOptions,\n            memory: config.memory,\n            requestContext: config.requestContext,\n          }\n        );\n\n        let text = \"\";\n        let firstTokenAt: number | undefined;\n        let inputTokensUsed: number | undefined;\n        let outputTokensUsed: number | undefined;\n        const metadata: Record<string, unknown> = {};\n\n        await response.processDataStream({\n          onChunk: async (chunk) => {\n            if (chunk.type === \"text-delta\") {\n              if (firstTokenAt === undefined) {\n                firstTokenAt = Date.now();\n              }\n              const payload = chunk.payload as { text?: string };\n              text += payload.text ?? \"\";\n            }\n\n            if (chunk.type === \"finish\") {\n              const payload = chunk.payload as {\n                output?: {\n                  usage?: {\n                    inputTokens?: number;\n                    outputTokens?: number;\n                  };\n                };\n              };\n              inputTokensUsed = payload.output?.usage?.inputTokens;\n              outputTokensUsed = payload.output?.usage?.outputTokens;\n            }\n          },\n        });\n\n        return {\n          data: text,\n          startedAt,\n          completedAt: Date.now(),\n          inputTokensUsed,\n          outputTokensUsed,\n          timeToFirstToken:\n            firstTokenAt !== undefined ? firstTokenAt - startedAt : undefined,\n          metadata:\n            Object.keys(metadata).length > 0 ? metadata : undefined,\n        };\n      },\n    };\n  }\n}\n","import { RateLimiter } from \"@/utils\";\nimport OpenAI, { APIError } from \"openai\";\nimport { AbstractProvider } from \"./abstract\";\nimport { PEERBENCH_NAMESPACE } from \"@/constants\";\nimport {\n  type CallableLLM,\n  type LLMResponse,\n  type CallableLLMForwardArgs,\n} from \"./callables/llm\";\n\n/**\n * Provider implementation that uses OpenAI SDK. It can be used with\n * any OpenAI compatible API.\n *\n * @example\n * ```ts\n * const provider = new OpenAIProvider({\n *   apiKey: \"sk-1234567890\",\n *   baseURL: \"https://openrouter.ai/api/v1\",\n * });\n * \n * const model = provider.model({ model: \"gpt-4o\" });\n * \n * const response = await model.forward({\n *   messages: [{ role: \"user\", content: \"Hello, how are you?\" }],\n * });\n * \n * console.log(response.data);\n * ```\n */\nexport class OpenAIProvider extends AbstractProvider.withKind(\n  `${PEERBENCH_NAMESPACE}/llm/openai`\n) {\n  private client: OpenAI;\n  private rateLimiter: RateLimiter;\n  private maxRetries: number;\n\n  constructor(config: {\n    apiKey: string;\n    baseURL: string;\n    maxRetries?: number;\n    timeout?: number;\n    rateLimiter?: RateLimiter;\n  }) {\n    super();\n    this.maxRetries = config.maxRetries ?? 3;\n    this.rateLimiter =\n      config.rateLimiter ??\n      new RateLimiter({\n        maxWeight: 20,\n        timeWindow: 3_000,\n      });\n\n    this.client = new OpenAI({\n      baseURL: config.baseURL,\n      apiKey: config.apiKey,\n      timeout: config.timeout,\n      dangerouslyAllowBrowser: true,\n    });\n  }\n\n  model(config: { model: string }): CallableLLM<OpenAIProvider> {\n    return {\n      slug: config.model,\n      provider: this,\n      forward: async (args: CallableLLMForwardArgs): Promise<LLMResponse> => {\n        let retryCount = this.maxRetries;\n        while (retryCount > 0) {\n          let startedAt: Date = new Date();\n\n          try {\n            const response = await this.rateLimiter.execute(\n              async () => {\n                startedAt = new Date();\n                return await this.client.chat.completions.create(\n                  {\n                    model: config.model,\n                    messages: args.messages,\n                    temperature: args.temperature,\n                    response_format: args.responseFormat,\n                  },\n                  { signal: args.abortSignal }\n                );\n              },\n              { signal: args.abortSignal }\n            );\n\n            if (\"error\" in response) {\n              const err = response.error as any;\n              throw new Error(\n                `${err.message} - Code ${err.code} - ${JSON.stringify(err)}`\n              );\n            }\n\n            if (!response?.choices?.[0]?.message?.content) {\n              throw new Error(\"No content returned from the model\");\n            }\n\n            return {\n              data: response.choices[0].message.content,\n              inputTokensUsed: response?.usage?.prompt_tokens,\n              outputTokensUsed: response?.usage?.completion_tokens,\n              startedAt: startedAt.getTime(),\n              completedAt: Date.now(),\n            };\n          } catch (err) {\n            if (err instanceof APIError && err.status === 401) {\n              throw new Error(`Invalid credentials provided`, { cause: err });\n            }\n\n            retryCount--;\n\n            if (err instanceof SyntaxError) {\n              console.debug(err);\n              continue;\n            }\n\n            if (retryCount !== 0) {\n              continue;\n            }\n\n            throw new Error(\n              `Failed to forward prompt to the model: ${err instanceof Error ? err.message : err}`,\n              { cause: err }\n            );\n          }\n        }\n\n        throw new Error(\n          `Failed to forward prompt to the model: Max retries reached`,\n          { cause: new Error(\"Max retries reached\") }\n        );\n      },\n    };\n  }\n}\n","import { AbstractProvider } from \"./abstract\";\nimport {\n  type CallableLLM,\n  type LLMResponse,\n  type CallableLLMForwardArgs,\n} from \"./callables/llm\";\nimport { RateLimiter } from \"@/utils\";\nimport { OpenAIProvider } from \"./openai\";\nimport { PEERBENCH_NAMESPACE } from \"@/constants\";\nimport Decimal from \"decimal.js\";\nimport axios from \"axios\";\n\nconst baseURL = \"https://openrouter.ai/api/v1\";\nconst MODELS_CACHE_TTL = 1000 * 60 * 60 * 24; // 24 hours\n\nexport class OpenRouterProvider extends AbstractProvider.withKind(\n  `${PEERBENCH_NAMESPACE}/llm/openrouter.ai`\n) {\n  private models: ModelsResponse | undefined = undefined;\n  private modelsCachePromise: Promise<ModelsResponse | undefined> =\n    Promise.resolve(undefined);\n  private modelsUpdatedAt = 0;\n  private openAIProvider: OpenAIProvider;\n\n  constructor(config: {\n    apiKey: string;\n    maxRetries?: number;\n    timeout?: number;\n    rateLimiter?: RateLimiter;\n  }) {\n    super();\n    this.openAIProvider = new OpenAIProvider({\n      baseURL,\n      apiKey: config.apiKey,\n      maxRetries: config.maxRetries,\n      timeout: config.timeout,\n      rateLimiter: config.rateLimiter,\n    });\n  }\n\n  model(config: { model: string }): CallableLLM<OpenRouterProvider> {\n    const openAICallable = this.openAIProvider.model({ model: config.model });\n    this.updateModelsCache().catch(() => { });\n\n    return {\n      slug: config.model,\n      provider: this,\n      forward: async (\n        args: CallableLLMForwardArgs\n      ): Promise<LLMResponse> => {\n        const response = await openAICallable.forward(args);\n\n        const modelInfo = this.models?.data.find(\n          (m) => m.id === config.model\n        );\n        let inputCost: string | undefined = undefined;\n        let outputCost: string | undefined = undefined;\n\n        if (modelInfo !== undefined) {\n          if (response.inputTokensUsed !== undefined) {\n            inputCost = new Decimal(modelInfo.pricing.prompt)\n              .mul(response.inputTokensUsed)\n              .toFixed(10);\n          }\n          if (response.outputTokensUsed !== undefined) {\n            outputCost = new Decimal(modelInfo.pricing.completion)\n              .mul(response.outputTokensUsed)\n              .toFixed(10);\n          }\n        }\n\n        return { ...response, inputCost, outputCost };\n      },\n    };\n  }\n\n  private async updateModelsCache() {\n    this.modelsCachePromise = this.modelsCachePromise\n      .then(async () => {\n        if (\n          this.models !== undefined &&\n          Date.now() - this.modelsUpdatedAt < MODELS_CACHE_TTL\n        ) {\n          return this.models;\n        }\n\n        return axios\n          .get<ModelsResponse>(`${baseURL}/models`)\n          .then((res) => res.data)\n          .then((data) => {\n            data = {\n              data: data.data.filter(\n                (m) =>\n                  m.architecture.input_modalities.includes(\"text\") &&\n                  m.architecture.output_modalities.includes(\"text\") &&\n                  ![\n                    \"morph/morph-v3-large\",\n                    \"morph/morph-v3-fast\",\n                    \"relace/relace-apply-3\",\n                  ].includes(m.id)\n              ),\n            };\n\n            this.models = data;\n            this.modelsUpdatedAt = Date.now();\n\n            return data;\n          });\n      })\n      .catch(() => undefined);\n\n    await this.modelsCachePromise;\n  }\n}\n\ntype PutModality = \"text\" | \"image\" | \"file\" | \"audio\";\ntype Modality = \"text->text\" | \"text+image->text\" | \"text+image->text+image\";\n\ntype ModelInfo = {\n  readonly id: string;\n  readonly canonical_slug: string;\n  readonly hugging_face_id: null | string;\n  readonly name: string;\n  readonly created: number;\n  readonly description: string;\n  readonly context_length: number;\n  readonly architecture: {\n    readonly modality: Modality;\n    readonly input_modalities: PutModality[];\n    readonly output_modalities: PutModality[];\n    readonly instruct_type: null | string;\n  };\n  readonly pricing: {\n    readonly prompt: string;\n    readonly completion: string;\n    readonly request?: string;\n    readonly image?: string;\n    readonly web_search?: string;\n    readonly internal_reasoning?: string;\n    readonly input_cache_read?: string;\n    readonly input_cache_write?: string;\n    readonly audio?: string;\n  };\n};\n\ntype ModelsResponse = {\n  data: ModelInfo[];\n};\n","import type { LanguageModelV3 } from \"@ai-sdk/provider\";\nimport { generateText, streamText } from \"ai\";\nimport { RateLimiter } from \"@/utils\";\nimport { AbstractProvider } from \"./abstract\";\nimport { PEERBENCH_NAMESPACE } from \"@/constants\";\nimport {\n  type CallableLLM,\n  type LLMResponse,\n  type CallableLLMForwardArgs,\n} from \"./callables/llm\";\n\nfunction toModelMessages(messages: CallableLLMForwardArgs[\"messages\"]) {\n  return messages\n    .filter(\n      (m) => m.role === \"system\" || m.role === \"user\" || m.role === \"assistant\"\n    )\n    .map((m) => ({\n      role: m.role as \"system\" | \"user\" | \"assistant\",\n      content:\n        typeof m.content === \"string\"\n          ? m.content\n          : JSON.stringify(m.content ?? \"\"),\n    }));\n}\n\nlet aiModuleCache: Record<string, unknown> | null = null;\n\nasync function buildOutputOption(\n  responseFormat: CallableLLMForwardArgs[\"responseFormat\"]\n): Promise<unknown> {\n  if (!responseFormat || responseFormat.type === \"text\") {\n    return undefined;\n  }\n\n  if (!aiModuleCache) {\n    aiModuleCache = (await import(\"ai\")) as unknown as Record<string, unknown>;\n  }\n\n  const Output = aiModuleCache[\"Output\"] as\n    | {\n        object: (opts: { schema: unknown }) => unknown;\n        json: () => unknown;\n      }\n    | undefined;\n  const jsonSchema = aiModuleCache[\"jsonSchema\"] as\n    | ((schema: unknown) => unknown)\n    | undefined;\n\n  if (!Output || !jsonSchema) {\n    return undefined;\n  }\n\n  if (responseFormat.type === \"json_schema\") {\n    return Output.object({\n      schema: jsonSchema(responseFormat.json_schema.schema),\n    });\n  }\n\n  if (responseFormat.type === \"json_object\") {\n    return Output.json();\n  }\n\n  return undefined;\n}\n\nexport class AISdkProvider extends AbstractProvider.withKind(\n  `${PEERBENCH_NAMESPACE}/llm/ai-sdk`\n) {\n  private rateLimiter: RateLimiter;\n  private maxRetries: number;\n\n  constructor(config?: AISdkProviderConfig) {\n    super();\n    this.maxRetries = config?.maxRetries ?? 2;\n    this.rateLimiter =\n      config?.rateLimiter ??\n      new RateLimiter({\n        maxWeight: 20,\n        timeWindow: 3_000,\n      });\n  }\n\n  model(config: AISdkModelConfig): CallableLLM<AISdkProvider> {\n    const slug = `${config.aiSdkModel.provider}/${config.aiSdkModel.modelId}`;\n\n    return {\n      slug,\n      provider: this,\n      forward: async (args: CallableLLMForwardArgs): Promise<LLMResponse> => {\n        const messages = toModelMessages(args.messages);\n        const output = await buildOutputOption(args.responseFormat);\n\n        const baseOptions = {\n          model: config.aiSdkModel,\n          messages,\n          temperature: args.temperature,\n          maxOutputTokens: args.maxTokens,\n          abortSignal: args.abortSignal,\n          maxRetries: this.maxRetries,\n        };\n\n        if (config.stream) {\n          let startedAt!: number;\n          const result = await this.rateLimiter.execute(\n            async () => {\n              startedAt = Date.now();\n              return streamText({\n                ...baseOptions,\n                ...(output !== undefined ? { output } : {}),\n              } as any);\n            },\n            { signal: args.abortSignal }\n          );\n\n          let text = \"\";\n          let firstTokenAt: number | undefined;\n\n          for await (const chunk of result.textStream) {\n            if (firstTokenAt === undefined) {\n              firstTokenAt = Date.now();\n            }\n            text += chunk;\n          }\n\n          const usage = await result.usage;\n\n          return {\n            data: text,\n            startedAt,\n            completedAt: Date.now(),\n            inputTokensUsed: usage.inputTokens,\n            outputTokensUsed: usage.outputTokens,\n            timeToFirstToken:\n              firstTokenAt !== undefined ? firstTokenAt - startedAt : undefined,\n          };\n        }\n\n        let startedAt!: number;\n        const result = await this.rateLimiter.execute(\n          async () => {\n            startedAt = Date.now();\n            return generateText({\n              ...baseOptions,\n              ...(output !== undefined ? { output } : {}),\n            } as any);\n          },\n          { signal: args.abortSignal }\n        );\n\n        return {\n          data: result.text,\n          startedAt,\n          completedAt: Date.now(),\n          inputTokensUsed: result.usage.inputTokens,\n          outputTokensUsed: result.usage.outputTokens,\n        };\n      },\n    };\n  }\n}\n\ntype AISdkProviderConfig = {\n  rateLimiter?: RateLimiter;\n  maxRetries?: number;\n};\n\ntype AISdkModelConfig = {\n  aiSdkModel: LanguageModelV3;\n  stream?: boolean;\n};\n"],"mappings":";;;;;;;;;AAAO,IAAe,mBAAf,MAAgC;AAAA,EAC5B;AAAA,EAET,cAAc;AACZ,SAAK,OAAQ,KAAK,YAAoB;AACtC,QAAI,CAAC,KAAK;AACR,YAAM,IAAI;AAAA,QACR,GAAG,KAAK,YAAY,IAAI;AAAA,MAC1B;AAAA,EACJ;AAAA,EACA,OAAO,SAGQ,MAAa;AAC1B,UAAM,OAAO;AACb,UAAM,UAAU,cAAc,KAAK;AAAA,MACjC,OAAgB,OAAc;AAAA,IAEhC;AAEA,WAAO;AAAA,EAKT;AACF;;;ACnBA,SAAS,oBAA2C;AAM7C,IAAM,iBAAN,cAA6B,iBAAiB;AAAA,EACnD,GAAG,mBAAmB;AACxB,EAAE;AAAA,EACiB;AAAA,EACA;AAAA,EACT;AAAA,EAER,YAAY,QAAkD;AAC5D,UAAM;AACN,SAAK,WAAW,OAAO;AACvB,SAAK,YAAY,OAAO;AACxB,SAAK,SAAS,IAAI,aAAa;AAAA,MAC7B,SAAS,KAAK;AAAA,MACd,SAAS,KAAK,YACV;AAAA,QACA,eAAe,UAAU,KAAK,SAAS;AAAA,MACzC,IACE;AAAA,IACN,CAAC;AAAA,EACH;AAAA,EAEA,MAAM,aAAa,MAGhB;AACD,WAAO,MAAM,KAAK,OACf,SAAS,KAAK,OAAO,EACrB,QAAQ,KAAK,cAAc;AAAA,EAChC;AAAA,EAEA,MAAM,UAAU,QAG8B;AAC5C,WAAO,KAAK,OAAO,WAAW,QAAQ,gBAAgB,QAAQ,OAAO;AAAA,EACvE;AAAA,EAEA,MAAM,QAK0B;AAC9B,WAAO;AAAA,MACL,MAAM,OAAO;AAAA,MACb,UAAU;AAAA,MACV,SAAS,OACP,SACyB;AACzB,cAAM,cAAc,KAAK,SACtB,OAAO,CAAC,MAAM,EAAE,SAAS,UAAU,EAAE,SAAS,WAAW,EACzD,IAAiB,CAAC,OAAO;AAAA,UACxB,MAAM,EAAE;AAAA,UACR,SAAS,OAAO,EAAE,WAAW,EAAE;AAAA,QACjC,EAAE;AAEJ,cAAM,QAAQ,KAAK,OAAO,SAAS,OAAO,OAAO;AACjD,cAAM,YAAY,KAAK,IAAI;AAE3B,cAAM,WAAW,MAAM,MAAM;AAAA,UAC3B;AAAA,UACA;AAAA,YACE,iBAAiB,OAAO;AAAA,YACxB,QAAQ,OAAO;AAAA,YACf,gBAAgB,OAAO;AAAA,UACzB;AAAA,QACF;AAEA,YAAI,OAAO;AACX,YAAI;AACJ,YAAI;AACJ,YAAI;AACJ,cAAM,WAAoC,CAAC;AAE3C,cAAM,SAAS,kBAAkB;AAAA,UAC/B,SAAS,OAAO,UAAU;AACxB,gBAAI,MAAM,SAAS,cAAc;AAC/B,kBAAI,iBAAiB,QAAW;AAC9B,+BAAe,KAAK,IAAI;AAAA,cAC1B;AACA,oBAAM,UAAU,MAAM;AACtB,sBAAQ,QAAQ,QAAQ;AAAA,YAC1B;AAEA,gBAAI,MAAM,SAAS,UAAU;AAC3B,oBAAM,UAAU,MAAM;AAQtB,gCAAkB,QAAQ,QAAQ,OAAO;AACzC,iCAAmB,QAAQ,QAAQ,OAAO;AAAA,YAC5C;AAAA,UACF;AAAA,QACF,CAAC;AAED,eAAO;AAAA,UACL,MAAM;AAAA,UACN;AAAA,UACA,aAAa,KAAK,IAAI;AAAA,UACtB;AAAA,UACA;AAAA,UACA,kBACE,iBAAiB,SAAY,eAAe,YAAY;AAAA,UAC1D,UACE,OAAO,KAAK,QAAQ,EAAE,SAAS,IAAI,WAAW;AAAA,QAClD;AAAA,MACF;AAAA,IACF;AAAA,EACF;AACF;;;AC7HA,OAAO,UAAU,gBAAgB;AA6B1B,IAAM,iBAAN,cAA6B,iBAAiB;AAAA,EACnD,GAAG,mBAAmB;AACxB,EAAE;AAAA,EACQ;AAAA,EACA;AAAA,EACA;AAAA,EAER,YAAY,QAMT;AACD,UAAM;AACN,SAAK,aAAa,OAAO,cAAc;AACvC,SAAK,cACH,OAAO,eACP,IAAI,YAAY;AAAA,MACd,WAAW;AAAA,MACX,YAAY;AAAA,IACd,CAAC;AAEH,SAAK,SAAS,IAAI,OAAO;AAAA,MACvB,SAAS,OAAO;AAAA,MAChB,QAAQ,OAAO;AAAA,MACf,SAAS,OAAO;AAAA,MAChB,yBAAyB;AAAA,IAC3B,CAAC;AAAA,EACH;AAAA,EAEA,MAAM,QAAwD;AAC5D,WAAO;AAAA,MACL,MAAM,OAAO;AAAA,MACb,UAAU;AAAA,MACV,SAAS,OAAO,SAAuD;AACrE,YAAI,aAAa,KAAK;AACtB,eAAO,aAAa,GAAG;AACrB,cAAI,YAAkB,oBAAI,KAAK;AAE/B,cAAI;AACF,kBAAM,WAAW,MAAM,KAAK,YAAY;AAAA,cACtC,YAAY;AACV,4BAAY,oBAAI,KAAK;AACrB,uBAAO,MAAM,KAAK,OAAO,KAAK,YAAY;AAAA,kBACxC;AAAA,oBACE,OAAO,OAAO;AAAA,oBACd,UAAU,KAAK;AAAA,oBACf,aAAa,KAAK;AAAA,oBAClB,iBAAiB,KAAK;AAAA,kBACxB;AAAA,kBACA,EAAE,QAAQ,KAAK,YAAY;AAAA,gBAC7B;AAAA,cACF;AAAA,cACA,EAAE,QAAQ,KAAK,YAAY;AAAA,YAC7B;AAEA,gBAAI,WAAW,UAAU;AACvB,oBAAM,MAAM,SAAS;AACrB,oBAAM,IAAI;AAAA,gBACR,GAAG,IAAI,OAAO,WAAW,IAAI,IAAI,MAAM,KAAK,UAAU,GAAG,CAAC;AAAA,cAC5D;AAAA,YACF;AAEA,gBAAI,CAAC,UAAU,UAAU,CAAC,GAAG,SAAS,SAAS;AAC7C,oBAAM,IAAI,MAAM,oCAAoC;AAAA,YACtD;AAEA,mBAAO;AAAA,cACL,MAAM,SAAS,QAAQ,CAAC,EAAE,QAAQ;AAAA,cAClC,iBAAiB,UAAU,OAAO;AAAA,cAClC,kBAAkB,UAAU,OAAO;AAAA,cACnC,WAAW,UAAU,QAAQ;AAAA,cAC7B,aAAa,KAAK,IAAI;AAAA,YACxB;AAAA,UACF,SAAS,KAAK;AACZ,gBAAI,eAAe,YAAY,IAAI,WAAW,KAAK;AACjD,oBAAM,IAAI,MAAM,gCAAgC,EAAE,OAAO,IAAI,CAAC;AAAA,YAChE;AAEA;AAEA,gBAAI,eAAe,aAAa;AAC9B,sBAAQ,MAAM,GAAG;AACjB;AAAA,YACF;AAEA,gBAAI,eAAe,GAAG;AACpB;AAAA,YACF;AAEA,kBAAM,IAAI;AAAA,cACR,0CAA0C,eAAe,QAAQ,IAAI,UAAU,GAAG;AAAA,cAClF,EAAE,OAAO,IAAI;AAAA,YACf;AAAA,UACF;AAAA,QACF;AAEA,cAAM,IAAI;AAAA,UACR;AAAA,UACA,EAAE,OAAO,IAAI,MAAM,qBAAqB,EAAE;AAAA,QAC5C;AAAA,MACF;AAAA,IACF;AAAA,EACF;AACF;;;AC9HA,OAAO,aAAa;AACpB,OAAO,WAAW;AAElB,IAAM,UAAU;AAChB,IAAM,mBAAmB,MAAO,KAAK,KAAK;AAEnC,IAAM,qBAAN,cAAiC,iBAAiB;AAAA,EACvD,GAAG,mBAAmB;AACxB,EAAE;AAAA,EACQ,SAAqC;AAAA,EACrC,qBACN,QAAQ,QAAQ,MAAS;AAAA,EACnB,kBAAkB;AAAA,EAClB;AAAA,EAER,YAAY,QAKT;AACD,UAAM;AACN,SAAK,iBAAiB,IAAI,eAAe;AAAA,MACvC;AAAA,MACA,QAAQ,OAAO;AAAA,MACf,YAAY,OAAO;AAAA,MACnB,SAAS,OAAO;AAAA,MAChB,aAAa,OAAO;AAAA,IACtB,CAAC;AAAA,EACH;AAAA,EAEA,MAAM,QAA4D;AAChE,UAAM,iBAAiB,KAAK,eAAe,MAAM,EAAE,OAAO,OAAO,MAAM,CAAC;AACxE,SAAK,kBAAkB,EAAE,MAAM,MAAM;AAAA,IAAE,CAAC;AAExC,WAAO;AAAA,MACL,MAAM,OAAO;AAAA,MACb,UAAU;AAAA,MACV,SAAS,OACP,SACyB;AACzB,cAAM,WAAW,MAAM,eAAe,QAAQ,IAAI;AAElD,cAAM,YAAY,KAAK,QAAQ,KAAK;AAAA,UAClC,CAAC,MAAM,EAAE,OAAO,OAAO;AAAA,QACzB;AACA,YAAI,YAAgC;AACpC,YAAI,aAAiC;AAErC,YAAI,cAAc,QAAW;AAC3B,cAAI,SAAS,oBAAoB,QAAW;AAC1C,wBAAY,IAAI,QAAQ,UAAU,QAAQ,MAAM,EAC7C,IAAI,SAAS,eAAe,EAC5B,QAAQ,EAAE;AAAA,UACf;AACA,cAAI,SAAS,qBAAqB,QAAW;AAC3C,yBAAa,IAAI,QAAQ,UAAU,QAAQ,UAAU,EAClD,IAAI,SAAS,gBAAgB,EAC7B,QAAQ,EAAE;AAAA,UACf;AAAA,QACF;AAEA,eAAO,EAAE,GAAG,UAAU,WAAW,WAAW;AAAA,MAC9C;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAc,oBAAoB;AAChC,SAAK,qBAAqB,KAAK,mBAC5B,KAAK,YAAY;AAChB,UACE,KAAK,WAAW,UAChB,KAAK,IAAI,IAAI,KAAK,kBAAkB,kBACpC;AACA,eAAO,KAAK;AAAA,MACd;AAEA,aAAO,MACJ,IAAoB,GAAG,OAAO,SAAS,EACvC,KAAK,CAAC,QAAQ,IAAI,IAAI,EACtB,KAAK,CAAC,SAAS;AACd,eAAO;AAAA,UACL,MAAM,KAAK,KAAK;AAAA,YACd,CAAC,MACC,EAAE,aAAa,iBAAiB,SAAS,MAAM,KAC/C,EAAE,aAAa,kBAAkB,SAAS,MAAM,KAChD,CAAC;AAAA,cACC;AAAA,cACA;AAAA,cACA;AAAA,YACF,EAAE,SAAS,EAAE,EAAE;AAAA,UACnB;AAAA,QACF;AAEA,aAAK,SAAS;AACd,aAAK,kBAAkB,KAAK,IAAI;AAEhC,eAAO;AAAA,MACT,CAAC;AAAA,IACL,CAAC,EACA,MAAM,MAAM,MAAS;AAExB,UAAM,KAAK;AAAA,EACb;AACF;;;AChHA,SAAS,cAAc,kBAAkB;AAUzC,SAAS,gBAAgB,UAA8C;AACrE,SAAO,SACJ;AAAA,IACC,CAAC,MAAM,EAAE,SAAS,YAAY,EAAE,SAAS,UAAU,EAAE,SAAS;AAAA,EAChE,EACC,IAAI,CAAC,OAAO;AAAA,IACX,MAAM,EAAE;AAAA,IACR,SACE,OAAO,EAAE,YAAY,WACjB,EAAE,UACF,KAAK,UAAU,EAAE,WAAW,EAAE;AAAA,EACtC,EAAE;AACN;AAEA,IAAI,gBAAgD;AAEpD,eAAe,kBACb,gBACkB;AAClB,MAAI,CAAC,kBAAkB,eAAe,SAAS,QAAQ;AACrD,WAAO;AAAA,EACT;AAEA,MAAI,CAAC,eAAe;AAClB,oBAAiB,MAAM,OAAO,IAAI;AAAA,EACpC;AAEA,QAAM,SAAS,cAAc,QAAQ;AAMrC,QAAM,aAAa,cAAc,YAAY;AAI7C,MAAI,CAAC,UAAU,CAAC,YAAY;AAC1B,WAAO;AAAA,EACT;AAEA,MAAI,eAAe,SAAS,eAAe;AACzC,WAAO,OAAO,OAAO;AAAA,MACnB,QAAQ,WAAW,eAAe,YAAY,MAAM;AAAA,IACtD,CAAC;AAAA,EACH;AAEA,MAAI,eAAe,SAAS,eAAe;AACzC,WAAO,OAAO,KAAK;AAAA,EACrB;AAEA,SAAO;AACT;AAEO,IAAM,gBAAN,cAA4B,iBAAiB;AAAA,EAClD,GAAG,mBAAmB;AACxB,EAAE;AAAA,EACQ;AAAA,EACA;AAAA,EAER,YAAY,QAA8B;AACxC,UAAM;AACN,SAAK,aAAa,QAAQ,cAAc;AACxC,SAAK,cACH,QAAQ,eACR,IAAI,YAAY;AAAA,MACd,WAAW;AAAA,MACX,YAAY;AAAA,IACd,CAAC;AAAA,EACL;AAAA,EAEA,MAAM,QAAsD;AAC1D,UAAM,OAAO,GAAG,OAAO,WAAW,QAAQ,IAAI,OAAO,WAAW,OAAO;AAEvE,WAAO;AAAA,MACL;AAAA,MACA,UAAU;AAAA,MACV,SAAS,OAAO,SAAuD;AACrE,cAAM,WAAW,gBAAgB,KAAK,QAAQ;AAC9C,cAAM,SAAS,MAAM,kBAAkB,KAAK,cAAc;AAE1D,cAAM,cAAc;AAAA,UAClB,OAAO,OAAO;AAAA,UACd;AAAA,UACA,aAAa,KAAK;AAAA,UAClB,iBAAiB,KAAK;AAAA,UACtB,aAAa,KAAK;AAAA,UAClB,YAAY,KAAK;AAAA,QACnB;AAEA,YAAI,OAAO,QAAQ;AACjB,cAAIA;AACJ,gBAAMC,UAAS,MAAM,KAAK,YAAY;AAAA,YACpC,YAAY;AACV,cAAAD,aAAY,KAAK,IAAI;AACrB,qBAAO,WAAW;AAAA,gBAChB,GAAG;AAAA,gBACH,GAAI,WAAW,SAAY,EAAE,OAAO,IAAI,CAAC;AAAA,cAC3C,CAAQ;AAAA,YACV;AAAA,YACA,EAAE,QAAQ,KAAK,YAAY;AAAA,UAC7B;AAEA,cAAI,OAAO;AACX,cAAI;AAEJ,2BAAiB,SAASC,QAAO,YAAY;AAC3C,gBAAI,iBAAiB,QAAW;AAC9B,6BAAe,KAAK,IAAI;AAAA,YAC1B;AACA,oBAAQ;AAAA,UACV;AAEA,gBAAM,QAAQ,MAAMA,QAAO;AAE3B,iBAAO;AAAA,YACL,MAAM;AAAA,YACN,WAAAD;AAAA,YACA,aAAa,KAAK,IAAI;AAAA,YACtB,iBAAiB,MAAM;AAAA,YACvB,kBAAkB,MAAM;AAAA,YACxB,kBACE,iBAAiB,SAAY,eAAeA,aAAY;AAAA,UAC5D;AAAA,QACF;AAEA,YAAI;AACJ,cAAM,SAAS,MAAM,KAAK,YAAY;AAAA,UACpC,YAAY;AACV,wBAAY,KAAK,IAAI;AACrB,mBAAO,aAAa;AAAA,cAClB,GAAG;AAAA,cACH,GAAI,WAAW,SAAY,EAAE,OAAO,IAAI,CAAC;AAAA,YAC3C,CAAQ;AAAA,UACV;AAAA,UACA,EAAE,QAAQ,KAAK,YAAY;AAAA,QAC7B;AAEA,eAAO;AAAA,UACL,MAAM,OAAO;AAAA,UACb;AAAA,UACA,aAAa,KAAK,IAAI;AAAA,UACtB,iBAAiB,OAAO,MAAM;AAAA,UAC9B,kBAAkB,OAAO,MAAM;AAAA,QACjC;AAAA,MACF;AAAA,IACF;AAAA,EACF;AACF;","names":["startedAt","result"]}

package/dist/schemas/extensions/index.js

@@ -1,7 +1,7 @@
 import {
   ExtensionLLMAsAJudgeScoreFieldsV1,
   ExtensionLLMResponseFieldsV1
-} from "../../chunk-TRNCF2BG.js";
+} from "../../chunk-HBGC6BDW.js";
 import "../../chunk-NUEOE3K5.js";
 import "../../chunk-PZ5AY32C.js";
 

package/dist/schemas/extensions/response/llm.d.ts

@@ -12,3 +12,20 @@ export declare const ExtensionLLMResponseFieldsV1: {
     inputCost: z.ZodOptional<z.ZodString>;
     outputCost: z.ZodOptional<z.ZodString>;
 };
+export declare function LLMResponseFieldsV1<TRawDataField extends boolean = false>(params?: {
+    /**
+     * If `true` then the `data` field (which is a `z.string()`)
+     * will be included in the returned fields
+     */
+    withRawDataField?: TRawDataField;
+}): {
+    modelSlug: z.ZodString;
+    provider: z.ZodString;
+    systemPromptId: z.ZodOptional<z.ZodString>;
+    inputTokensUsed: z.ZodOptional<z.ZodNumber>;
+    outputTokensUsed: z.ZodOptional<z.ZodNumber>;
+    inputCost: z.ZodOptional<z.ZodString>;
+    outputCost: z.ZodOptional<z.ZodString>;
+} & (TRawDataField extends true ? {
+    data: z.ZodString;
+} : {});

package/dist/schemas/llm/simple-system-prompt.d.ts

@@ -6,7 +6,7 @@ export declare const SimpleSystemPromptSchemaV1: z.ZodObject<Omit<{
     schemaVersion: z.ZodNumber;
     version: z.ZodNumber;
     metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
-}, "kind" | "namespace" | "schemaVersion"> & {
+}, "kind" | "schemaVersion" | "namespace"> & {
     content: z.ZodString;
 } & {
     namespace: z.ZodLiteral<"peerbench.ai">;
@@ -21,7 +21,7 @@ export declare const SimpleSystemPromptSchemaV1: z.ZodObject<Omit<{
         kind: `${string}/simple.sys-prompt`;
         schemaVersion: 1;
         metadata?: Record<string, unknown> | undefined;
-    }, "kind" | "namespace" | "schemaVersion">) => {
+    }, "kind" | "schemaVersion" | "namespace">) => {
         id: string;
         version: number;
         content: string;
@@ -38,7 +38,7 @@ export declare const SimpleSystemPromptSchemaV1: z.ZodObject<Omit<{
         kind: `${string}/simple.sys-prompt`;
         schemaVersion: 1;
         metadata?: Record<string, unknown> | undefined;
-    }, "kind" | "id" | "namespace" | "schemaVersion">, generator: import("../../index.js").IdGenerator): Promise<{
+    }, "id" | "kind" | "schemaVersion" | "namespace">, generator: import("../../index.js").IdGenerator): Promise<{
         id: string;
         version: number;
         content: string;

package/dist/schemas/llm/system-prompt.d.ts

@@ -25,33 +25,33 @@ export declare const defineSystemPromptSchema: <TBaseSchema extends import("../.
     fields?: TFields | undefined;
 }) => (TBaseSchema extends z.ZodObject<infer U extends Readonly<{
     [k: string]: z.core.$ZodType<unknown, unknown, z.core.$ZodTypeInternals<unknown, unknown>>;
-}>, z.core.$strip> ? z.ZodObject<Omit<U, "kind" | "namespace" | "schemaVersion"> & TFields & {
+}>, z.core.$strip> ? z.ZodObject<Omit<U, "kind" | "schemaVersion" | "namespace"> & TFields & {
     namespace: TNamespace extends undefined ? U["namespace"] : z.ZodLiteral<TNamespace>;
     kind: TKind extends undefined ? U["kind"] : z.ZodLiteral<`${TKind}.sys-prompt`>;
     schemaVersion: TSchemaVersion extends undefined ? U["schemaVersion"] : z.ZodLiteral<TSchemaVersion>;
 }, z.core.$strip> : never) & {
     new: (input: Omit<z.core.output<TBaseSchema extends z.ZodObject<infer U extends Readonly<{
         [k: string]: z.core.$ZodType<unknown, unknown, z.core.$ZodTypeInternals<unknown, unknown>>;
-    }>, z.core.$strip> ? z.ZodObject<Omit<U, "kind" | "namespace" | "schemaVersion"> & TFields & {
+    }>, z.core.$strip> ? z.ZodObject<Omit<U, "kind" | "schemaVersion" | "namespace"> & TFields & {
         namespace: TNamespace extends undefined ? U["namespace"] : z.ZodLiteral<TNamespace>;
         kind: TKind extends undefined ? U["kind"] : z.ZodLiteral<`${TKind}.sys-prompt`>;
         schemaVersion: TSchemaVersion extends undefined ? U["schemaVersion"] : z.ZodLiteral<TSchemaVersion>;
-    }, z.core.$strip> : never>, "kind" | "namespace" | "schemaVersion">) => z.core.output<TBaseSchema extends z.ZodObject<infer U extends Readonly<{
+    }, z.core.$strip> : never>, "kind" | "schemaVersion" | "namespace">) => z.core.output<TBaseSchema extends z.ZodObject<infer U extends Readonly<{
         [k: string]: z.core.$ZodType<unknown, unknown, z.core.$ZodTypeInternals<unknown, unknown>>;
-    }>, z.core.$strip> ? z.ZodObject<Omit<U, "kind" | "namespace" | "schemaVersion"> & TFields & {
+    }>, z.core.$strip> ? z.ZodObject<Omit<U, "kind" | "schemaVersion" | "namespace"> & TFields & {
         namespace: TNamespace extends undefined ? U["namespace"] : z.ZodLiteral<TNamespace>;
         kind: TKind extends undefined ? U["kind"] : z.ZodLiteral<`${TKind}.sys-prompt`>;
         schemaVersion: TSchemaVersion extends undefined ? U["schemaVersion"] : z.ZodLiteral<TSchemaVersion>;
     }, z.core.$strip> : never>;
     newWithId(input: Omit<z.core.output<TBaseSchema extends z.ZodObject<infer U extends Readonly<{
         [k: string]: z.core.$ZodType<unknown, unknown, z.core.$ZodTypeInternals<unknown, unknown>>;
-    }>, z.core.$strip> ? z.ZodObject<Omit<U, "kind" | "namespace" | "schemaVersion"> & TFields & {
+    }>, z.core.$strip> ? z.ZodObject<Omit<U, "kind" | "schemaVersion" | "namespace"> & TFields & {
         namespace: TNamespace extends undefined ? U["namespace"] : z.ZodLiteral<TNamespace>;
         kind: TKind extends undefined ? U["kind"] : z.ZodLiteral<`${TKind}.sys-prompt`>;
         schemaVersion: TSchemaVersion extends undefined ? U["schemaVersion"] : z.ZodLiteral<TSchemaVersion>;
-    }, z.core.$strip> : never>, "kind" | "id" | "namespace" | "schemaVersion">, generator: import("../../index.js").IdGenerator): Promise<z.core.output<TBaseSchema extends z.ZodObject<infer U extends Readonly<{
+    }, z.core.$strip> : never>, "id" | "kind" | "schemaVersion" | "namespace">, generator: import("../../index.js").IdGenerator): Promise<z.core.output<TBaseSchema extends z.ZodObject<infer U extends Readonly<{
         [k: string]: z.core.$ZodType<unknown, unknown, z.core.$ZodTypeInternals<unknown, unknown>>;
-    }>, z.core.$strip> ? z.ZodObject<Omit<U, "kind" | "namespace" | "schemaVersion"> & TFields & {
+    }>, z.core.$strip> ? z.ZodObject<Omit<U, "kind" | "schemaVersion" | "namespace"> & TFields & {
         namespace: TNamespace extends undefined ? U["namespace"] : z.ZodLiteral<TNamespace>;
         kind: TKind extends undefined ? U["kind"] : z.ZodLiteral<`${TKind}.sys-prompt`>;
         schemaVersion: TSchemaVersion extends undefined ? U["schemaVersion"] : z.ZodLiteral<TSchemaVersion>;