peerbench 0.0.9 → 0.0.11

package/dist/chunk-6WDCU5BP.js

@@ -0,0 +1,9 @@
+// src/helpers/define-runner.ts
+function defineRunner(fn) {
+  return fn;
+}
+
+export {
+  defineRunner
+};
+//# sourceMappingURL=chunk-6WDCU5BP.js.map

package/dist/chunk-6WDCU5BP.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/helpers/define-runner.ts"],"sourcesContent":["import { RunnerParams, RunnerResult } from \"@/types\";\n\nexport function defineRunner<TParams extends RunnerParams, TResult extends RunnerResult>(\n  fn: (params: TParams) => Promise<TResult>\n) {\n  return fn;\n}\n"],"mappings":";AAEO,SAAS,aACd,IACA;AACA,SAAO;AACT;","names":[]}

package/dist/{chunk-YY33MNMV.js → chunk-7KMGLEYP.js}

@@ -3,7 +3,7 @@ import {
 } from "./chunk-OQE6TQXZ.js";
 import {
   ScoringMethod
-} from "./chunk-HMQYGCKI.js";
+} from "./chunk-ZJWSK4VO.js";
 import {
   IdSchema
 } from "./chunk-NUEOE3K5.js";
@@ -62,4 +62,4 @@ export {
   BaseScoreSchemaV1,
   defineScoreSchema
 };
-//# sourceMappingURL=chunk-YY33MNMV.js.map
+//# sourceMappingURL=chunk-7KMGLEYP.js.map

package/dist/{chunk-HMQYGCKI.js → chunk-ZJWSK4VO.js}

@@ -8,4 +8,4 @@ var ScoringMethod = {
 export {
   ScoringMethod
 };
-//# sourceMappingURL=chunk-HMQYGCKI.js.map
+//# sourceMappingURL=chunk-ZJWSK4VO.js.map

package/dist/chunk-ZJWSK4VO.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/types/index.ts"],"sourcesContent":["export * from \"./runner\";\n\nimport { IdSchema } from \"@/schemas/id\";\nimport z from \"zod\";\n\nexport type Id = z.infer<typeof IdSchema>;\n\nexport type IdGenerator<TInput = unknown> = (input: TInput) => MaybePromise<Id>;\n\nexport type MaybePromise<T> = T | Promise<T>;\n\nexport const ScoringMethod = {\n  ai: \"ai\",\n  human: \"human\",\n  algo: \"algo\",\n} as const;\nexport type ScoringMethod = (typeof ScoringMethod)[keyof typeof ScoringMethod];\n"],"mappings":";AAWO,IAAM,gBAAgB;AAAA,EAC3B,IAAI;AAAA,EACJ,OAAO;AAAA,EACP,MAAM;AACR;","names":[]}

package/dist/helpers/define-runner.d.ts

@@ -1,45 +1,2 @@
-import { IdGenerator, InferRunConfig, ProviderCtor, Runner, ScorerCtor } from "../types";
-import z from "zod";
-export declare function defineRunner<const TProviders extends ProviderCtor[], const TScorers extends ScorerCtor[], const TSchemaSets extends SchemaSetDefinition[], const TRunConfigSchema extends z.ZodRawShape = {}>(config: {
-    schemaSets: TSchemaSets;
-    providers: TProviders;
-    scorers: TScorers;
-    runConfigSchema?: TRunConfigSchema;
-    /**
-     * @default true
-     */
-    parseRunConfig?: boolean;
-    defaults?: {
-        scorer?: InstanceType<TScorers[number]>;
-        responseIdGenerator?: IdGenerator;
-        scoreIdGenerator?: IdGenerator;
-    };
-}, fn: Runner<TSchemaSets[number]["testCase"], TSchemaSets[number]["response"], TSchemaSets[number]["score"], InstanceType<TProviders[number]>, InstanceType<TScorers[number]>, InferRunConfig<TRunConfigSchema>>): ((params: Parameters<typeof fn>[0]) => Promise<{
-    response: z.core.output<TSchemaSets[number]["response"]>;
-    score?: z.core.output<TSchemaSets[number]["score"]> | undefined;
-}>) & {
-    /**
-     * The configuration that was used to define the runner.
-     */
-    config: {
-        runConfigSchema: z.ZodObject<{ -readonly [P in keyof TRunConfigSchema]: TRunConfigSchema[P]; }, z.core.$strip>;
-        schemaSets: TSchemaSets;
-        providers: TProviders;
-        scorers: TScorers;
-        /**
-         * @default true
-         */
-        parseRunConfig?: boolean;
-        defaults?: {
-            scorer?: InstanceType<TScorers[number]>;
-            responseIdGenerator?: IdGenerator;
-            scoreIdGenerator?: IdGenerator;
-        };
-    };
-};
-type SchemaSetDefinition<TTestCase extends z.ZodObject = z.ZodObject, TResponse extends z.ZodObject = z.ZodObject, TScore extends z.ZodObject = z.ZodObject> = {
-    testCase: TTestCase;
-    response: TResponse;
-    score: TScore;
-};
-export {};
+import { RunnerParams, RunnerResult } from "../types";
+export declare function defineRunner<TParams extends RunnerParams, TResult extends RunnerResult>(fn: (params: TParams) => Promise<TResult>): (params: TParams) => Promise<TResult>;

package/dist/index.js

@@ -1,9 +1,9 @@
 import {
   defineRunner
-} from "./chunk-RTEAK4II.js";
+} from "./chunk-6WDCU5BP.js";
 import {
   ScoringMethod
-} from "./chunk-HMQYGCKI.js";
+} from "./chunk-ZJWSK4VO.js";
 import {
   CATEGORIES,
   PEERBENCH_NAMESPACE

package/dist/providers/callables/callable.d.ts

@@ -0,0 +1,4 @@
+import { AbstractProvider } from "../abstract";
+export interface Callable<TProvider = AbstractProvider> {
+    readonly provider: TProvider;
+}

package/dist/providers/callables/llm.d.ts

@@ -0,0 +1,38 @@
+import { ChatCompletionMessageParam } from "openai/resources/chat/completions";
+import { ResponseFormatJSONObject, ResponseFormatJSONSchema, ResponseFormatText } from "openai/resources/shared";
+import { AbstractProvider, ProviderResponse } from "../abstract";
+import { Callable } from "./callable";
+export interface CallableLLM<TProvider extends AbstractProvider = AbstractProvider> extends Callable<TProvider> {
+    slug: string;
+    forward(args: CallableLLMForwardArgs): Promise<LLMResponse>;
+}
+export type CallableLLMForwardArgs = {
+    messages: ChatCompletionMessageParam[];
+    abortSignal?: AbortSignal;
+    maxTokens?: number;
+    temperature?: number;
+    responseFormat?: ResponseFormatText | ResponseFormatJSONSchema | ResponseFormatJSONObject;
+};
+export type LLMResponse = ProviderResponse<string> & {
+    /**
+     * Number of input tokens used.
+     */
+    inputTokensUsed?: number;
+    /**
+     * Number of output tokens used.
+     */
+    outputTokensUsed?: number;
+    /**
+     * Cost of the input tokens.
+     */
+    inputCost?: string;
+    /**
+     * Cost of the output tokens.
+     */
+    outputCost?: string;
+    /**
+     * Time taken to receive the first token.
+     */
+    timeToFirstToken?: number;
+    metadata?: Record<string, unknown>;
+};

package/dist/providers/example/echo.d.ts

@@ -1,12 +1,13 @@
-import { AbstractLLMProvider, ChatResponse, LLMProviderForwardArgs } from "../abstract/llm";
-/**
- * Example provider implementation for local testing and as a reference.
- *
- * - Extends `AbstractLLMProvider`
- * - Implements `forward({ messages, model, ... })`
- * - Does not perform any network calls
- */
-export declare class ExampleEchoLLMProvider extends AbstractLLMProvider {
-    readonly kind = "example.echo";
-    forward(args: LLMProviderForwardArgs): Promise<ChatResponse>;
+import { AbstractProvider } from "../abstract";
+import { type CallableLLM } from "../callables/llm";
+declare const ExampleEchoLLMProvider_base: (new () => AbstractProvider & {
+    readonly kind: "example.echo";
+}) & {
+    readonly kind: "example.echo";
+};
+export declare class ExampleEchoLLMProvider extends ExampleEchoLLMProvider_base {
+    model(config?: {
+        model?: string;
+    }): CallableLLM<ExampleEchoLLMProvider>;
 }
+export {};

package/dist/providers/example/restapi.d.ts

@@ -1,25 +1,18 @@
-import { AbstractLLMProvider, ChatResponse, LLMProviderForwardArgs } from "../abstract/llm";
-/**
- * Example "custom REST API LLM agent provider".
- *
- * Sometimes you don't call a model API directly. You call your own REST API, and *it* talks to the model.
- * That REST API can hide secrets, run tools, do retrieval, apply guardrails, and whatever else your
- * product needs.
- *
- * In the SDK we still want a clean abstraction, so we model that REST API as an `AbstractLLMProvider`.
- * The runner (or host app) still passes `messages + model`, and the provider still returns one final string.
- *
- * If you’re implementing your own provider, this is the only part that matters: translate
- * `LLMProviderForwardArgs` into your HTTP request, then translate your HTTP response back into
- * `ChatResponse`.
- */
-export declare class ExampleRestApiLLMAgentProvider extends AbstractLLMProvider {
-    readonly kind = "example.restapi.agent";
+import { AbstractProvider } from "../abstract";
+import { type CallableLLM } from "../callables/llm";
+declare const ExampleRestApiLLMAgentProvider_base: (new () => AbstractProvider & {
+    readonly kind: "example.rest-api.agent";
+}) & {
+    readonly kind: "example.rest-api.agent";
+};
+export declare class ExampleRestApiLLMAgentProvider extends ExampleRestApiLLMAgentProvider_base {
     private readonly baseUrl;
     private readonly apiKey?;
     private readonly headers?;
     constructor(config: ExampleRestApiAgentProviderConfig);
-    forward(args: LLMProviderForwardArgs): Promise<ChatResponse>;
+    model(config?: {
+        model?: string;
+    }): CallableLLM<ExampleRestApiLLMAgentProvider>;
 }
 type ExampleRestApiAgentProviderConfig = {
     /**

package/dist/providers/index.d.ts

@@ -1,5 +1,6 @@
-export * from "./abstract/llm";
-export * from "./abstract/provider";
+export * from "./callables/callable";
+export * from "./callables/llm";
+export * from "./abstract";
 export * from "./mastra";
 export * from "./openai";
 export * from "./openrouter";

package/dist/providers/index.js

@@ -1,15 +1,270 @@
 import {
-  AbstractLLMProvider,
-  AbstractProvider,
-  MastraProvider,
-  OpenAIProvider,
-  OpenRouterProvider
-} from "../chunk-SMLNDQFX.js";
-import "../chunk-UHHHSYVE.js";
-import "../chunk-4UBK6452.js";
+  PEERBENCH_NAMESPACE
+} from "../chunk-UHHHSYVE.js";
+import {
+  RateLimiter
+} from "../chunk-4UBK6452.js";
 import "../chunk-PZ5AY32C.js";
+
+// src/providers/abstract.ts
+var AbstractProvider = class {
+  kind;
+  constructor() {
+    this.kind = this.constructor.kind;
+    if (!this.kind)
+      throw new Error(
+        `${this.constructor.name} must define "static readonly kind" as a constant property.`
+      );
+  }
+  static withKind(kind) {
+    const base = this;
+    const derived = class extends base {
+      static kind = kind;
+    };
+    return derived;
+  }
+};
+
+// src/providers/mastra.ts
+import { MastraClient } from "@mastra/client-js";
+var MastraProvider = class extends AbstractProvider.withKind(
+  `${PEERBENCH_NAMESPACE}/llm/mastra`
+) {
+  endpoint;
+  authToken;
+  client;
+  constructor(params) {
+    super();
+    this.endpoint = params.endpoint;
+    this.authToken = params.authToken;
+    this.client = new MastraClient({
+      baseUrl: this.endpoint,
+      headers: this.authToken ? {
+        Authorization: `Bearer ${this.authToken}`
+      } : void 0
+    });
+  }
+  async getAgentInfo(args) {
+    return await this.client.getAgent(args.agentId).details(args.requestContext);
+  }
+  async getAgents(params) {
+    return this.client.listAgents(params?.requestContext, params?.partial);
+  }
+  agent(config) {
+    return {
+      slug: config.agentId,
+      provider: this,
+      forward: async (args) => {
+        const apiMessages = args.messages.filter((m) => m.role === "user" || m.role === "assistant").map((m) => ({
+          role: m.role,
+          content: String(m.content ?? "")
+        }));
+        const agent = this.client.getAgent(config.agentId);
+        const startedAt = Date.now();
+        const response = await agent.stream(
+          apiMessages,
+          {
+            providerOptions: config.providerOptions,
+            memory: config.memory,
+            requestContext: config.requestContext
+          }
+        );
+        let text = "";
+        let firstTokenAt;
+        let inputTokensUsed;
+        let outputTokensUsed;
+        const metadata = {};
+        await response.processDataStream({
+          onChunk: async (chunk) => {
+            if (chunk.type === "text-delta") {
+              if (firstTokenAt === void 0) {
+                firstTokenAt = Date.now();
+              }
+              const payload = chunk.payload;
+              text += payload.text ?? "";
+            }
+            if (chunk.type === "finish") {
+              const payload = chunk.payload;
+              inputTokensUsed = payload.output?.usage?.inputTokens;
+              outputTokensUsed = payload.output?.usage?.outputTokens;
+            }
+          }
+        });
+        return {
+          data: text,
+          startedAt,
+          completedAt: Date.now(),
+          inputTokensUsed,
+          outputTokensUsed,
+          timeToFirstToken: firstTokenAt !== void 0 ? firstTokenAt - startedAt : void 0,
+          metadata: Object.keys(metadata).length > 0 ? metadata : void 0
+        };
+      }
+    };
+  }
+};
+
+// src/providers/openai.ts
+import OpenAI, { APIError } from "openai";
+var OpenAIProvider = class extends AbstractProvider.withKind(
+  `${PEERBENCH_NAMESPACE}/llm/openai`
+) {
+  client;
+  rateLimiter;
+  maxRetries;
+  constructor(config) {
+    super();
+    this.maxRetries = config.maxRetries ?? 3;
+    this.rateLimiter = config.rateLimiter ?? new RateLimiter({
+      maxWeight: 20,
+      timeWindow: 3e3
+    });
+    this.client = new OpenAI({
+      baseURL: config.baseURL,
+      apiKey: config.apiKey,
+      timeout: config.timeout,
+      dangerouslyAllowBrowser: true
+    });
+  }
+  model(config) {
+    return {
+      slug: config.model,
+      provider: this,
+      forward: async (args) => {
+        let retryCount = this.maxRetries;
+        while (retryCount > 0) {
+          let startedAt = /* @__PURE__ */ new Date();
+          try {
+            const response = await this.rateLimiter.execute(
+              async () => {
+                startedAt = /* @__PURE__ */ new Date();
+                return await this.client.chat.completions.create(
+                  {
+                    model: config.model,
+                    messages: args.messages,
+                    temperature: args.temperature,
+                    response_format: args.responseFormat
+                  },
+                  { signal: args.abortSignal }
+                );
+              },
+              { signal: args.abortSignal }
+            );
+            if ("error" in response) {
+              const err = response.error;
+              throw new Error(
+                `${err.message} - Code ${err.code} - ${JSON.stringify(err)}`
+              );
+            }
+            if (!response?.choices?.[0]?.message?.content) {
+              throw new Error("No content returned from the model");
+            }
+            return {
+              data: response.choices[0].message.content,
+              inputTokensUsed: response?.usage?.prompt_tokens,
+              outputTokensUsed: response?.usage?.completion_tokens,
+              startedAt: startedAt.getTime(),
+              completedAt: Date.now()
+            };
+          } catch (err) {
+            if (err instanceof APIError && err.status === 401) {
+              throw new Error(`Invalid credentials provided`, { cause: err });
+            }
+            retryCount--;
+            if (err instanceof SyntaxError) {
+              console.debug(err);
+              continue;
+            }
+            if (retryCount !== 0) {
+              continue;
+            }
+            throw new Error(
+              `Failed to forward prompt to the model: ${err instanceof Error ? err.message : err}`,
+              { cause: err }
+            );
+          }
+        }
+        throw new Error(
+          `Failed to forward prompt to the model: Max retries reached`,
+          { cause: new Error("Max retries reached") }
+        );
+      }
+    };
+  }
+};
+
+// src/providers/openrouter.ts
+import Decimal from "decimal.js";
+import axios from "axios";
+var baseURL = "https://openrouter.ai/api/v1";
+var MODELS_CACHE_TTL = 1e3 * 60 * 60 * 24;
+var OpenRouterProvider = class extends AbstractProvider.withKind(
+  `${PEERBENCH_NAMESPACE}/llm/openrouter.ai`
+) {
+  models = void 0;
+  modelsCachePromise = Promise.resolve(void 0);
+  modelsUpdatedAt = 0;
+  openAIProvider;
+  constructor(config) {
+    super();
+    this.openAIProvider = new OpenAIProvider({
+      baseURL,
+      apiKey: config.apiKey,
+      maxRetries: config.maxRetries,
+      timeout: config.timeout,
+      rateLimiter: config.rateLimiter
+    });
+  }
+  model(config) {
+    const openAICallable = this.openAIProvider.model({ model: config.model });
+    this.updateModelsCache().catch(() => {
+    });
+    return {
+      slug: config.model,
+      provider: this,
+      forward: async (args) => {
+        const response = await openAICallable.forward(args);
+        const modelInfo = this.models?.data.find(
+          (m) => m.id === config.model
+        );
+        let inputCost = void 0;
+        let outputCost = void 0;
+        if (modelInfo !== void 0) {
+          if (response.inputTokensUsed !== void 0) {
+            inputCost = new Decimal(modelInfo.pricing.prompt).mul(response.inputTokensUsed).toFixed(10);
+          }
+          if (response.outputTokensUsed !== void 0) {
+            outputCost = new Decimal(modelInfo.pricing.completion).mul(response.outputTokensUsed).toFixed(10);
+          }
+        }
+        return { ...response, inputCost, outputCost };
+      }
+    };
+  }
+  async updateModelsCache() {
+    this.modelsCachePromise = this.modelsCachePromise.then(async () => {
+      if (this.models !== void 0 && Date.now() - this.modelsUpdatedAt < MODELS_CACHE_TTL) {
+        return this.models;
+      }
+      return axios.get(`${baseURL}/models`).then((res) => res.data).then((data) => {
+        data = {
+          data: data.data.filter(
+            (m) => m.architecture.input_modalities.includes("text") && m.architecture.output_modalities.includes("text") && ![
+              "morph/morph-v3-large",
+              "morph/morph-v3-fast",
+              "relace/relace-apply-3"
+            ].includes(m.id)
+          )
+        };
+        this.models = data;
+        this.modelsUpdatedAt = Date.now();
+        return data;
+      });
+    }).catch(() => void 0);
+    await this.modelsCachePromise;
+  }
+};
 export {
-  AbstractLLMProvider,
   AbstractProvider,
   MastraProvider,
   OpenAIProvider,

package/dist/providers/index.js.map

@@ -1 +1 @@
-{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}
+{"version":3,"sources":["../../src/providers/abstract.ts","../../src/providers/mastra.ts","../../src/providers/openai.ts","../../src/providers/openrouter.ts"],"sourcesContent":["export abstract class AbstractProvider {\n  readonly kind: string;\n\n  constructor() {\n    this.kind = (this.constructor as any).kind;\n    if (!this.kind)\n      throw new Error(\n        `${this.constructor.name} must define \"static readonly kind\" as a constant property.`\n      );\n  }\n  static withKind<\n    TKind extends string,\n    TThis extends abstract new (...args: any[]) => AbstractProvider,\n  >(this: TThis, kind: TKind) {\n    const base = this as unknown as new (...args: any[]) => any;\n    const derived = class extends base {\n      static readonly kind: TKind = kind;\n      declare readonly kind: TKind;\n    };\n\n    return derived as unknown as (new () => InstanceType<TThis> & {\n      readonly kind: TKind;\n    }) & {\n      readonly kind: TKind;\n    };\n  }\n}\n\nexport type ProviderResponse<TData = unknown> = {\n  startedAt: number;\n  completedAt: number;\n  data: TData;\n};\n","import { PEERBENCH_NAMESPACE } from \"@/constants\";\nimport { AbstractProvider } from \"./abstract\";\nimport {\n  type CallableLLM,\n  type LLMResponse,\n  type CallableLLMForwardArgs,\n} from \"./callables/llm\";\nimport { MastraClient, type GetAgentResponse } from \"@mastra/client-js\";\nimport type { RequestContext } from \"@mastra/core/request-context\";\nimport { AgentMemoryOption } from \"@mastra/core/agent\";\nimport { ProviderOptions } from \"@mastra/core/dist/llm/model/provider-options\";\nimport { CoreMessage } from \"@mastra/core/llm\";\n\nexport class MastraProvider extends AbstractProvider.withKind(\n  `${PEERBENCH_NAMESPACE}/llm/mastra`\n) {\n  private readonly endpoint: string;\n  private readonly authToken?: string;\n  private client: MastraClient;\n\n  constructor(params: { endpoint: string; authToken?: string }) {\n    super();\n    this.endpoint = params.endpoint;\n    this.authToken = params.authToken;\n    this.client = new MastraClient({\n      baseUrl: this.endpoint,\n      headers: this.authToken\n        ? {\n          Authorization: `Bearer ${this.authToken}`,\n        }\n        : undefined,\n    });\n  }\n\n  async getAgentInfo(args: {\n    agentId: string;\n    requestContext?: RequestContext;\n  }) {\n    return await this.client\n      .getAgent(args.agentId)\n      .details(args.requestContext);\n  }\n\n  async getAgents(params?: {\n    requestContext?: RequestContext;\n    partial?: boolean;\n  }): Promise<Record<string, GetAgentResponse>> {\n    return this.client.listAgents(params?.requestContext, params?.partial);\n  }\n\n  agent(config: {\n    agentId: string;\n    memory?: AgentMemoryOption;\n    requestContext?: RequestContext;\n    providerOptions?: ProviderOptions;\n  }): CallableLLM<MastraProvider> {\n    return {\n      slug: config.agentId,\n      provider: this,\n      forward: async (\n        args: CallableLLMForwardArgs\n      ): Promise<LLMResponse> => {\n        const apiMessages = args.messages\n          .filter((m) => m.role === \"user\" || m.role === \"assistant\")\n          .map<CoreMessage>((m) => ({\n            role: m.role,\n            content: String(m.content ?? \"\"),\n          }));\n\n        const agent = this.client.getAgent(config.agentId);\n        const startedAt = Date.now();\n\n        const response = await agent.stream(\n          apiMessages,\n          {\n            providerOptions: config.providerOptions,\n            memory: config.memory,\n            requestContext: config.requestContext,\n          }\n        );\n\n        let text = \"\";\n        let firstTokenAt: number | undefined;\n        let inputTokensUsed: number | undefined;\n        let outputTokensUsed: number | undefined;\n        const metadata: Record<string, unknown> = {};\n\n        await response.processDataStream({\n          onChunk: async (chunk) => {\n            if (chunk.type === \"text-delta\") {\n              if (firstTokenAt === undefined) {\n                firstTokenAt = Date.now();\n              }\n              const payload = chunk.payload as { text?: string };\n              text += payload.text ?? \"\";\n            }\n\n            if (chunk.type === \"finish\") {\n              const payload = chunk.payload as {\n                output?: {\n                  usage?: {\n                    inputTokens?: number;\n                    outputTokens?: number;\n                  };\n                };\n              };\n              inputTokensUsed = payload.output?.usage?.inputTokens;\n              outputTokensUsed = payload.output?.usage?.outputTokens;\n            }\n          },\n        });\n\n        return {\n          data: text,\n          startedAt,\n          completedAt: Date.now(),\n          inputTokensUsed,\n          outputTokensUsed,\n          timeToFirstToken:\n            firstTokenAt !== undefined ? firstTokenAt - startedAt : undefined,\n          metadata:\n            Object.keys(metadata).length > 0 ? metadata : undefined,\n        };\n      },\n    };\n  }\n}\n","import { RateLimiter } from \"@/utils\";\nimport OpenAI, { APIError } from \"openai\";\nimport { AbstractProvider } from \"./abstract\";\nimport { PEERBENCH_NAMESPACE } from \"@/constants\";\nimport {\n  type CallableLLM,\n  type LLMResponse,\n  type CallableLLMForwardArgs,\n} from \"./callables/llm\";\n\n/**\n * Provider implementation that uses OpenAI SDK. It can be used with\n * any OpenAI compatible API.\n *\n * @example\n * ```ts\n * const provider = new OpenAIProvider({\n *   apiKey: \"sk-1234567890\",\n *   baseURL: \"https://openrouter.ai/api/v1\",\n * });\n * \n * const model = provider.model({ model: \"gpt-4o\" });\n * \n * const response = await model.forward({\n *   messages: [{ role: \"user\", content: \"Hello, how are you?\" }],\n * });\n * \n * console.log(response.data);\n * ```\n */\nexport class OpenAIProvider extends AbstractProvider.withKind(\n  `${PEERBENCH_NAMESPACE}/llm/openai`\n) {\n  private client: OpenAI;\n  private rateLimiter: RateLimiter;\n  private maxRetries: number;\n\n  constructor(config: {\n    apiKey: string;\n    baseURL: string;\n    maxRetries?: number;\n    timeout?: number;\n    rateLimiter?: RateLimiter;\n  }) {\n    super();\n    this.maxRetries = config.maxRetries ?? 3;\n    this.rateLimiter =\n      config.rateLimiter ??\n      new RateLimiter({\n        maxWeight: 20,\n        timeWindow: 3_000,\n      });\n\n    this.client = new OpenAI({\n      baseURL: config.baseURL,\n      apiKey: config.apiKey,\n      timeout: config.timeout,\n      dangerouslyAllowBrowser: true,\n    });\n  }\n\n  model(config: { model: string }): CallableLLM<OpenAIProvider> {\n    return {\n      slug: config.model,\n      provider: this,\n      forward: async (args: CallableLLMForwardArgs): Promise<LLMResponse> => {\n        let retryCount = this.maxRetries;\n        while (retryCount > 0) {\n          let startedAt: Date = new Date();\n\n          try {\n            const response = await this.rateLimiter.execute(\n              async () => {\n                startedAt = new Date();\n                return await this.client.chat.completions.create(\n                  {\n                    model: config.model,\n                    messages: args.messages,\n                    temperature: args.temperature,\n                    response_format: args.responseFormat,\n                  },\n                  { signal: args.abortSignal }\n                );\n              },\n              { signal: args.abortSignal }\n            );\n\n            if (\"error\" in response) {\n              const err = response.error as any;\n              throw new Error(\n                `${err.message} - Code ${err.code} - ${JSON.stringify(err)}`\n              );\n            }\n\n            if (!response?.choices?.[0]?.message?.content) {\n              throw new Error(\"No content returned from the model\");\n            }\n\n            return {\n              data: response.choices[0].message.content,\n              inputTokensUsed: response?.usage?.prompt_tokens,\n              outputTokensUsed: response?.usage?.completion_tokens,\n              startedAt: startedAt.getTime(),\n              completedAt: Date.now(),\n            };\n          } catch (err) {\n            if (err instanceof APIError && err.status === 401) {\n              throw new Error(`Invalid credentials provided`, { cause: err });\n            }\n\n            retryCount--;\n\n            if (err instanceof SyntaxError) {\n              console.debug(err);\n              continue;\n            }\n\n            if (retryCount !== 0) {\n              continue;\n            }\n\n            throw new Error(\n              `Failed to forward prompt to the model: ${err instanceof Error ? err.message : err}`,\n              { cause: err }\n            );\n          }\n        }\n\n        throw new Error(\n          `Failed to forward prompt to the model: Max retries reached`,\n          { cause: new Error(\"Max retries reached\") }\n        );\n      },\n    };\n  }\n}\n","import { AbstractProvider } from \"./abstract\";\nimport {\n  type CallableLLM,\n  type LLMResponse,\n  type CallableLLMForwardArgs,\n} from \"./callables/llm\";\nimport { RateLimiter } from \"@/utils\";\nimport { OpenAIProvider } from \"./openai\";\nimport { PEERBENCH_NAMESPACE } from \"@/constants\";\nimport Decimal from \"decimal.js\";\nimport axios from \"axios\";\n\nconst baseURL = \"https://openrouter.ai/api/v1\";\nconst MODELS_CACHE_TTL = 1000 * 60 * 60 * 24; // 24 hours\n\nexport class OpenRouterProvider extends AbstractProvider.withKind(\n  `${PEERBENCH_NAMESPACE}/llm/openrouter.ai`\n) {\n  private models: ModelsResponse | undefined = undefined;\n  private modelsCachePromise: Promise<ModelsResponse | undefined> =\n    Promise.resolve(undefined);\n  private modelsUpdatedAt = 0;\n  private openAIProvider: OpenAIProvider;\n\n  constructor(config: {\n    apiKey: string;\n    maxRetries?: number;\n    timeout?: number;\n    rateLimiter?: RateLimiter;\n  }) {\n    super();\n    this.openAIProvider = new OpenAIProvider({\n      baseURL,\n      apiKey: config.apiKey,\n      maxRetries: config.maxRetries,\n      timeout: config.timeout,\n      rateLimiter: config.rateLimiter,\n    });\n  }\n\n  model(config: { model: string }): CallableLLM<OpenRouterProvider> {\n    const openAICallable = this.openAIProvider.model({ model: config.model });\n    this.updateModelsCache().catch(() => { });\n\n    return {\n      slug: config.model,\n      provider: this,\n      forward: async (\n        args: CallableLLMForwardArgs\n      ): Promise<LLMResponse> => {\n        const response = await openAICallable.forward(args);\n\n        const modelInfo = this.models?.data.find(\n          (m) => m.id === config.model\n        );\n        let inputCost: string | undefined = undefined;\n        let outputCost: string | undefined = undefined;\n\n        if (modelInfo !== undefined) {\n          if (response.inputTokensUsed !== undefined) {\n            inputCost = new Decimal(modelInfo.pricing.prompt)\n              .mul(response.inputTokensUsed)\n              .toFixed(10);\n          }\n          if (response.outputTokensUsed !== undefined) {\n            outputCost = new Decimal(modelInfo.pricing.completion)\n              .mul(response.outputTokensUsed)\n              .toFixed(10);\n          }\n        }\n\n        return { ...response, inputCost, outputCost };\n      },\n    };\n  }\n\n  private async updateModelsCache() {\n    this.modelsCachePromise = this.modelsCachePromise\n      .then(async () => {\n        if (\n          this.models !== undefined &&\n          Date.now() - this.modelsUpdatedAt < MODELS_CACHE_TTL\n        ) {\n          return this.models;\n        }\n\n        return axios\n          .get<ModelsResponse>(`${baseURL}/models`)\n          .then((res) => res.data)\n          .then((data) => {\n            data = {\n              data: data.data.filter(\n                (m) =>\n                  m.architecture.input_modalities.includes(\"text\") &&\n                  m.architecture.output_modalities.includes(\"text\") &&\n                  ![\n                    \"morph/morph-v3-large\",\n                    \"morph/morph-v3-fast\",\n                    \"relace/relace-apply-3\",\n                  ].includes(m.id)\n              ),\n            };\n\n            this.models = data;\n            this.modelsUpdatedAt = Date.now();\n\n            return data;\n          });\n      })\n      .catch(() => undefined);\n\n    await this.modelsCachePromise;\n  }\n}\n\ntype PutModality = \"text\" | \"image\" | \"file\" | \"audio\";\ntype Modality = \"text->text\" | \"text+image->text\" | \"text+image->text+image\";\n\ntype ModelInfo = {\n  readonly id: string;\n  readonly canonical_slug: string;\n  readonly hugging_face_id: null | string;\n  readonly name: string;\n  readonly created: number;\n  readonly description: string;\n  readonly context_length: number;\n  readonly architecture: {\n    readonly modality: Modality;\n    readonly input_modalities: PutModality[];\n    readonly output_modalities: PutModality[];\n    readonly instruct_type: null | string;\n  };\n  readonly pricing: {\n    readonly prompt: string;\n    readonly completion: string;\n    readonly request?: string;\n    readonly image?: string;\n    readonly web_search?: string;\n    readonly internal_reasoning?: string;\n    readonly input_cache_read?: string;\n    readonly input_cache_write?: string;\n    readonly audio?: string;\n  };\n};\n\ntype ModelsResponse = {\n  data: ModelInfo[];\n};\n"],"mappings":";;;;;;;;;AAAO,IAAe,mBAAf,MAAgC;AAAA,EAC5B;AAAA,EAET,cAAc;AACZ,SAAK,OAAQ,KAAK,YAAoB;AACtC,QAAI,CAAC,KAAK;AACR,YAAM,IAAI;AAAA,QACR,GAAG,KAAK,YAAY,IAAI;AAAA,MAC1B;AAAA,EACJ;AAAA,EACA,OAAO,SAGQ,MAAa;AAC1B,UAAM,OAAO;AACb,UAAM,UAAU,cAAc,KAAK;AAAA,MACjC,OAAgB,OAAc;AAAA,IAEhC;AAEA,WAAO;AAAA,EAKT;AACF;;;ACnBA,SAAS,oBAA2C;AAM7C,IAAM,iBAAN,cAA6B,iBAAiB;AAAA,EACnD,GAAG,mBAAmB;AACxB,EAAE;AAAA,EACiB;AAAA,EACA;AAAA,EACT;AAAA,EAER,YAAY,QAAkD;AAC5D,UAAM;AACN,SAAK,WAAW,OAAO;AACvB,SAAK,YAAY,OAAO;AACxB,SAAK,SAAS,IAAI,aAAa;AAAA,MAC7B,SAAS,KAAK;AAAA,MACd,SAAS,KAAK,YACV;AAAA,QACA,eAAe,UAAU,KAAK,SAAS;AAAA,MACzC,IACE;AAAA,IACN,CAAC;AAAA,EACH;AAAA,EAEA,MAAM,aAAa,MAGhB;AACD,WAAO,MAAM,KAAK,OACf,SAAS,KAAK,OAAO,EACrB,QAAQ,KAAK,cAAc;AAAA,EAChC;AAAA,EAEA,MAAM,UAAU,QAG8B;AAC5C,WAAO,KAAK,OAAO,WAAW,QAAQ,gBAAgB,QAAQ,OAAO;AAAA,EACvE;AAAA,EAEA,MAAM,QAK0B;AAC9B,WAAO;AAAA,MACL,MAAM,OAAO;AAAA,MACb,UAAU;AAAA,MACV,SAAS,OACP,SACyB;AACzB,cAAM,cAAc,KAAK,SACtB,OAAO,CAAC,MAAM,EAAE,SAAS,UAAU,EAAE,SAAS,WAAW,EACzD,IAAiB,CAAC,OAAO;AAAA,UACxB,MAAM,EAAE;AAAA,UACR,SAAS,OAAO,EAAE,WAAW,EAAE;AAAA,QACjC,EAAE;AAEJ,cAAM,QAAQ,KAAK,OAAO,SAAS,OAAO,OAAO;AACjD,cAAM,YAAY,KAAK,IAAI;AAE3B,cAAM,WAAW,MAAM,MAAM;AAAA,UAC3B;AAAA,UACA;AAAA,YACE,iBAAiB,OAAO;AAAA,YACxB,QAAQ,OAAO;AAAA,YACf,gBAAgB,OAAO;AAAA,UACzB;AAAA,QACF;AAEA,YAAI,OAAO;AACX,YAAI;AACJ,YAAI;AACJ,YAAI;AACJ,cAAM,WAAoC,CAAC;AAE3C,cAAM,SAAS,kBAAkB;AAAA,UAC/B,SAAS,OAAO,UAAU;AACxB,gBAAI,MAAM,SAAS,cAAc;AAC/B,kBAAI,iBAAiB,QAAW;AAC9B,+BAAe,KAAK,IAAI;AAAA,cAC1B;AACA,oBAAM,UAAU,MAAM;AACtB,sBAAQ,QAAQ,QAAQ;AAAA,YAC1B;AAEA,gBAAI,MAAM,SAAS,UAAU;AAC3B,oBAAM,UAAU,MAAM;AAQtB,gCAAkB,QAAQ,QAAQ,OAAO;AACzC,iCAAmB,QAAQ,QAAQ,OAAO;AAAA,YAC5C;AAAA,UACF;AAAA,QACF,CAAC;AAED,eAAO;AAAA,UACL,MAAM;AAAA,UACN;AAAA,UACA,aAAa,KAAK,IAAI;AAAA,UACtB;AAAA,UACA;AAAA,UACA,kBACE,iBAAiB,SAAY,eAAe,YAAY;AAAA,UAC1D,UACE,OAAO,KAAK,QAAQ,EAAE,SAAS,IAAI,WAAW;AAAA,QAClD;AAAA,MACF;AAAA,IACF;AAAA,EACF;AACF;;;AC7HA,OAAO,UAAU,gBAAgB;AA6B1B,IAAM,iBAAN,cAA6B,iBAAiB;AAAA,EACnD,GAAG,mBAAmB;AACxB,EAAE;AAAA,EACQ;AAAA,EACA;AAAA,EACA;AAAA,EAER,YAAY,QAMT;AACD,UAAM;AACN,SAAK,aAAa,OAAO,cAAc;AACvC,SAAK,cACH,OAAO,eACP,IAAI,YAAY;AAAA,MACd,WAAW;AAAA,MACX,YAAY;AAAA,IACd,CAAC;AAEH,SAAK,SAAS,IAAI,OAAO;AAAA,MACvB,SAAS,OAAO;AAAA,MAChB,QAAQ,OAAO;AAAA,MACf,SAAS,OAAO;AAAA,MAChB,yBAAyB;AAAA,IAC3B,CAAC;AAAA,EACH;AAAA,EAEA,MAAM,QAAwD;AAC5D,WAAO;AAAA,MACL,MAAM,OAAO;AAAA,MACb,UAAU;AAAA,MACV,SAAS,OAAO,SAAuD;AACrE,YAAI,aAAa,KAAK;AACtB,eAAO,aAAa,GAAG;AACrB,cAAI,YAAkB,oBAAI,KAAK;AAE/B,cAAI;AACF,kBAAM,WAAW,MAAM,KAAK,YAAY;AAAA,cACtC,YAAY;AACV,4BAAY,oBAAI,KAAK;AACrB,uBAAO,MAAM,KAAK,OAAO,KAAK,YAAY;AAAA,kBACxC;AAAA,oBACE,OAAO,OAAO;AAAA,oBACd,UAAU,KAAK;AAAA,oBACf,aAAa,KAAK;AAAA,oBAClB,iBAAiB,KAAK;AAAA,kBACxB;AAAA,kBACA,EAAE,QAAQ,KAAK,YAAY;AAAA,gBAC7B;AAAA,cACF;AAAA,cACA,EAAE,QAAQ,KAAK,YAAY;AAAA,YAC7B;AAEA,gBAAI,WAAW,UAAU;AACvB,oBAAM,MAAM,SAAS;AACrB,oBAAM,IAAI;AAAA,gBACR,GAAG,IAAI,OAAO,WAAW,IAAI,IAAI,MAAM,KAAK,UAAU,GAAG,CAAC;AAAA,cAC5D;AAAA,YACF;AAEA,gBAAI,CAAC,UAAU,UAAU,CAAC,GAAG,SAAS,SAAS;AAC7C,oBAAM,IAAI,MAAM,oCAAoC;AAAA,YACtD;AAEA,mBAAO;AAAA,cACL,MAAM,SAAS,QAAQ,CAAC,EAAE,QAAQ;AAAA,cAClC,iBAAiB,UAAU,OAAO;AAAA,cAClC,kBAAkB,UAAU,OAAO;AAAA,cACnC,WAAW,UAAU,QAAQ;AAAA,cAC7B,aAAa,KAAK,IAAI;AAAA,YACxB;AAAA,UACF,SAAS,KAAK;AACZ,gBAAI,eAAe,YAAY,IAAI,WAAW,KAAK;AACjD,oBAAM,IAAI,MAAM,gCAAgC,EAAE,OAAO,IAAI,CAAC;AAAA,YAChE;AAEA;AAEA,gBAAI,eAAe,aAAa;AAC9B,sBAAQ,MAAM,GAAG;AACjB;AAAA,YACF;AAEA,gBAAI,eAAe,GAAG;AACpB;AAAA,YACF;AAEA,kBAAM,IAAI;AAAA,cACR,0CAA0C,eAAe,QAAQ,IAAI,UAAU,GAAG;AAAA,cAClF,EAAE,OAAO,IAAI;AAAA,YACf;AAAA,UACF;AAAA,QACF;AAEA,cAAM,IAAI;AAAA,UACR;AAAA,UACA,EAAE,OAAO,IAAI,MAAM,qBAAqB,EAAE;AAAA,QAC5C;AAAA,MACF;AAAA,IACF;AAAA,EACF;AACF;;;AC9HA,OAAO,aAAa;AACpB,OAAO,WAAW;AAElB,IAAM,UAAU;AAChB,IAAM,mBAAmB,MAAO,KAAK,KAAK;AAEnC,IAAM,qBAAN,cAAiC,iBAAiB;AAAA,EACvD,GAAG,mBAAmB;AACxB,EAAE;AAAA,EACQ,SAAqC;AAAA,EACrC,qBACN,QAAQ,QAAQ,MAAS;AAAA,EACnB,kBAAkB;AAAA,EAClB;AAAA,EAER,YAAY,QAKT;AACD,UAAM;AACN,SAAK,iBAAiB,IAAI,eAAe;AAAA,MACvC;AAAA,MACA,QAAQ,OAAO;AAAA,MACf,YAAY,OAAO;AAAA,MACnB,SAAS,OAAO;AAAA,MAChB,aAAa,OAAO;AAAA,IACtB,CAAC;AAAA,EACH;AAAA,EAEA,MAAM,QAA4D;AAChE,UAAM,iBAAiB,KAAK,eAAe,MAAM,EAAE,OAAO,OAAO,MAAM,CAAC;AACxE,SAAK,kBAAkB,EAAE,MAAM,MAAM;AAAA,IAAE,CAAC;AAExC,WAAO;AAAA,MACL,MAAM,OAAO;AAAA,MACb,UAAU;AAAA,MACV,SAAS,OACP,SACyB;AACzB,cAAM,WAAW,MAAM,eAAe,QAAQ,IAAI;AAElD,cAAM,YAAY,KAAK,QAAQ,KAAK;AAAA,UAClC,CAAC,MAAM,EAAE,OAAO,OAAO;AAAA,QACzB;AACA,YAAI,YAAgC;AACpC,YAAI,aAAiC;AAErC,YAAI,cAAc,QAAW;AAC3B,cAAI,SAAS,oBAAoB,QAAW;AAC1C,wBAAY,IAAI,QAAQ,UAAU,QAAQ,MAAM,EAC7C,IAAI,SAAS,eAAe,EAC5B,QAAQ,EAAE;AAAA,UACf;AACA,cAAI,SAAS,qBAAqB,QAAW;AAC3C,yBAAa,IAAI,QAAQ,UAAU,QAAQ,UAAU,EAClD,IAAI,SAAS,gBAAgB,EAC7B,QAAQ,EAAE;AAAA,UACf;AAAA,QACF;AAEA,eAAO,EAAE,GAAG,UAAU,WAAW,WAAW;AAAA,MAC9C;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAc,oBAAoB;AAChC,SAAK,qBAAqB,KAAK,mBAC5B,KAAK,YAAY;AAChB,UACE,KAAK,WAAW,UAChB,KAAK,IAAI,IAAI,KAAK,kBAAkB,kBACpC;AACA,eAAO,KAAK;AAAA,MACd;AAEA,aAAO,MACJ,IAAoB,GAAG,OAAO,SAAS,EACvC,KAAK,CAAC,QAAQ,IAAI,IAAI,EACtB,KAAK,CAAC,SAAS;AACd,eAAO;AAAA,UACL,MAAM,KAAK,KAAK;AAAA,YACd,CAAC,MACC,EAAE,aAAa,iBAAiB,SAAS,MAAM,KAC/C,EAAE,aAAa,kBAAkB,SAAS,MAAM,KAChD,CAAC;AAAA,cACC;AAAA,cACA;AAAA,cACA;AAAA,YACF,EAAE,SAAS,EAAE,EAAE;AAAA,UACnB;AAAA,QACF;AAEA,aAAK,SAAS;AACd,aAAK,kBAAkB,KAAK,IAAI;AAEhC,eAAO;AAAA,MACT,CAAC;AAAA,IACL,CAAC,EACA,MAAM,MAAM,MAAS;AAExB,UAAM,KAAK;AAAA,EACb;AACF;","names":[]}

package/dist/providers/mastra.d.ts

@@ -1,6 +1,10 @@
-import { AbstractLLMProvider, type ChatResponse, type LLMProviderForwardArgs } from "./abstract/llm";
-import { MastraClient, type GetAgentResponse } from "@mastra/client-js";
-declare const MastraProvider_base: (new () => AbstractLLMProvider & {
+import { AbstractProvider } from "./abstract";
+import { type CallableLLM } from "./callables/llm";
+import { type GetAgentResponse } from "@mastra/client-js";
+import type { RequestContext } from "@mastra/core/request-context";
+import { AgentMemoryOption } from "@mastra/core/agent";
+import { ProviderOptions } from "@mastra/core/dist/llm/model/provider-options";
+declare const MastraProvider_base: (new () => AbstractProvider & {
     readonly kind: "peerbench.ai/llm/mastra";
 }) & {
     readonly kind: "peerbench.ai/llm/mastra";
@@ -9,32 +13,23 @@ export declare class MastraProvider extends MastraProvider_base {
     private readonly endpoint;
     private readonly authToken?;
     private client;
-    private memory?;
     constructor(params: {
         endpoint: string;
         authToken?: string;
-        memory?: AgentMemoryOption;
     });
-    forward(args: LLMProviderForwardArgs & {
-        memory?: AgentMemoryOption;
-        /**
-         * The model that will be used as the brain for the agent.
-         */
-        modelName?: string;
-    }): Promise<ChatResponse>;
     getAgentInfo(args: {
         agentId: string;
-        runtimeContext?: MastraRuntimeContext;
+        requestContext?: RequestContext;
     }): Promise<GetAgentResponse>;
-    getAgents(args?: {
-        runtimeContext?: MastraRuntimeContext;
+    getAgents(params?: {
+        requestContext?: RequestContext;
         partial?: boolean;
     }): Promise<Record<string, GetAgentResponse>>;
+    agent(config: {
+        agentId: string;
+        memory?: AgentMemoryOption;
+        requestContext?: RequestContext;
+        providerOptions?: ProviderOptions;
+    }): CallableLLM<MastraProvider>;
 }
-export type AgentMemoryOption = Parameters<Parameters<MastraClient["getAgent"]>["0"] extends string ? ReturnType<MastraClient["getAgent"]>["generate"] : never>[0] extends {
-    memory?: infer M;
-} ? M : never;
-type MastraRuntimeContext = Parameters<Parameters<MastraClient["getAgent"]>["0"] extends string ? ReturnType<MastraClient["getAgent"]>["generate"] : never>[0] extends {
-    runtimeContext?: infer R;
-} ? R : never;
 export {};

package/dist/providers/openai.d.ts

@@ -1,12 +1,31 @@
 import { RateLimiter } from "../utils";
-import { ChatCompletionMessageParam } from "openai/resources/chat/completions";
-import { ResponseFormatJSONObject, ResponseFormatJSONSchema, ResponseFormatText } from "openai/resources/shared";
-import { AbstractLLMProvider, ChatResponse } from "./abstract/llm";
-declare const OpenAIProvider_base: (new () => AbstractLLMProvider & {
+import { AbstractProvider } from "./abstract";
+import { type CallableLLM } from "./callables/llm";
+declare const OpenAIProvider_base: (new () => AbstractProvider & {
     readonly kind: "peerbench.ai/llm/openai";
 }) & {
     readonly kind: "peerbench.ai/llm/openai";
 };
+/**
+ * Provider implementation that uses OpenAI SDK. It can be used with
+ * any OpenAI compatible API.
+ *
+ * @example
+ * ```ts
+ * const provider = new OpenAIProvider({
+ *   apiKey: "sk-1234567890",
+ *   baseURL: "https://openrouter.ai/api/v1",
+ * });
+ *
+ * const model = provider.model({ model: "gpt-4o" });
+ *
+ * const response = await model.forward({
+ *   messages: [{ role: "user", content: "Hello, how are you?" }],
+ * });
+ *
+ * console.log(response.data);
+ * ```
+ */
 export declare class OpenAIProvider extends OpenAIProvider_base {
     private client;
     private rateLimiter;
@@ -18,12 +37,8 @@ export declare class OpenAIProvider extends OpenAIProvider_base {
         timeout?: number;
         rateLimiter?: RateLimiter;
     });
-    forward(args: {
-        messages: ChatCompletionMessageParam[];
+    model(config: {
         model: string;
-        abortSignal?: AbortSignal;
-        temperature?: number;
-        responseFormat?: ResponseFormatText | ResponseFormatJSONSchema | ResponseFormatJSONObject;
-    }): Promise<ChatResponse>;
+    }): CallableLLM<OpenAIProvider>;
 }
 export {};