peerbench 0.0.6 → 0.0.8

package/dist/index.d.ts

@@ -1,101 +1,6 @@
-import { I as IdGenerator, P as ProviderCtor, S as ScorerCtor } from './index-BAioQhp2.js';
-export { A as AbstractClassConstructor, C as ClassConstructor, b as Id, a as InferExtension, M as MaybePromise, c as ScoringMethod, W as WidenZodObject } from './index-BAioQhp2.js';
-export { R as RateLimiter, b as RateLimiterCallOptions, a as RateLimiterOptions } from './rate-limiter-CSmVIRsM.js';
-import z__default from 'zod';
-import { A as AbstractProvider } from './provider-BDjGp2y-.js';
-import { A as AbstractScorer } from './abstract-Dec9Sc5O.js';
-
-declare function sleep(ms: number, signal?: AbortSignal): Promise<void>;
-
-/**
- * Tries to repair and parse LLM response as a JSON object. LLM must
- * be configured to return a JSON object. This function only helps to
- * get rid out of some additional formatting (e.g. ```json) and repair
- * the JSON syntax (e.g missing comma, single quotes instead double).
- */
-declare function parseResponseAsJSON<T>(response: string): T | undefined;
-
-/**
- * Converts the given byte array to a string
- */
-declare function bufferToString(buffer: Uint8Array, encoding?: BufferEncoding): string;
-/**
- * Converts the given string to a byte array
- */
-declare function stringToBuffer(str: string): Uint8Array;
-
-declare const idGeneratorUUIDv7: IdGenerator;
-
-type Runner<TTestCase extends z__default.ZodObject, TResponse extends z__default.ZodObject, TScore extends z__default.ZodObject, TProvider extends AbstractProvider, TScorer extends AbstractScorer, TRunConfig extends Record<string, unknown>> = (params: {
-    testCase: z__default.infer<TTestCase>;
-    provider: TProvider;
-    scorer?: TScorer;
-    runConfig: TRunConfig;
-    idGenerators?: {
-        response?: IdGenerator;
-        score?: IdGenerator;
-    };
-}) => Promise<{
-    response: z__default.infer<TResponse>;
-    score?: z__default.infer<TScore>;
-}>;
-type InferRunConfig<TRunConfigSchema extends z__default.ZodRawShape> = z__default.infer<z__default.ZodObject<TRunConfigSchema>>;
-
-declare class PeerbenchError extends Error {
-    code: number;
-    constructor(message?: string, options?: ErrorOptions & {
-        code: number;
-    });
-}
-
-declare function captureStackTrace(error: Error, constructor: Function): void;
-
-declare const PEERBENCH_NAMESPACE: "peerbench.ai";
-declare const CATEGORIES: {
-    LLM: string;
-};
-
-declare function defineRunner<const TProviders extends ProviderCtor[], const TScorers extends ScorerCtor[], const TSchemaSets extends SchemaSetDefinition[], const TRunConfigSchema extends z__default.ZodRawShape = {}>(config: {
-    schemaSets: TSchemaSets;
-    providers: TProviders;
-    scorers: TScorers;
-    runConfigSchema?: TRunConfigSchema;
-    /**
-     * @default true
-     */
-    parseRunConfig?: boolean;
-    defaults?: {
-        scorer?: InstanceType<TScorers[number]>;
-        responseIdGenerator?: IdGenerator;
-        scoreIdGenerator?: IdGenerator;
-    };
-}, fn: Runner<TSchemaSets[number]["testCase"], TSchemaSets[number]["response"], TSchemaSets[number]["score"], InstanceType<TProviders[number]>, InstanceType<TScorers[number]>, InferRunConfig<TRunConfigSchema>>): ((params: Parameters<typeof fn>[0]) => Promise<{
-    response: z__default.core.output<TSchemaSets[number]["response"]>;
-    score?: z__default.core.output<TSchemaSets[number]["score"]> | undefined;
-}>) & {
-    /**
-     * The configuration that was used to define the runner.
-     */
-    config: {
-        runConfigSchema: z__default.ZodObject<{ -readonly [P in keyof TRunConfigSchema]: TRunConfigSchema[P]; }, z__default.core.$strip>;
-        schemaSets: TSchemaSets;
-        providers: TProviders;
-        scorers: TScorers;
-        /**
-         * @default true
-         */
-        parseRunConfig?: boolean;
-        defaults?: {
-            scorer?: InstanceType<TScorers[number]>;
-            responseIdGenerator?: IdGenerator;
-            scoreIdGenerator?: IdGenerator;
-        };
-    };
-};
-type SchemaSetDefinition<TTestCase extends z__default.ZodObject = z__default.ZodObject, TResponse extends z__default.ZodObject = z__default.ZodObject, TScore extends z__default.ZodObject = z__default.ZodObject> = {
-    testCase: TTestCase;
-    response: TResponse;
-    score: TScore;
-};
-
-export { CATEGORIES, IdGenerator, type InferRunConfig, PEERBENCH_NAMESPACE, PeerbenchError, ProviderCtor, type Runner, ScorerCtor, bufferToString, captureStackTrace, defineRunner, idGeneratorUUIDv7, parseResponseAsJSON, sleep, stringToBuffer };
+export * from "./types";
+export * from "./errors";
+export * from "./utils";
+export * from "./utilities";
+export * from "./constants";
+export * from "./helpers";

package/dist/providers/abstract/llm.d.ts

@@ -0,0 +1,20 @@
+import { AbstractProvider, ProviderResponse } from "./provider";
+import { ResponseFormatJSONObject, ResponseFormatJSONSchema, ResponseFormatText } from "openai/resources/shared";
+import { ChatCompletionMessageParam } from "openai/resources/chat/completions";
+export declare abstract class AbstractLLMProvider extends AbstractProvider {
+    abstract forward(args: LLMProviderForwardArgs): Promise<ChatResponse>;
+}
+export type LLMProviderForwardArgs = {
+    messages: ChatCompletionMessageParam[];
+    model: string;
+    abortSignal?: AbortSignal;
+    temperature?: number;
+    responseFormat?: ResponseFormatText | ResponseFormatJSONSchema | ResponseFormatJSONObject;
+};
+export type ChatResponse = ProviderResponse<string> & {
+    inputTokensUsed?: number;
+    outputTokensUsed?: number;
+    inputCost?: string;
+    outputCost?: string;
+    metadata?: Record<string, unknown>;
+};

package/dist/providers/abstract/provider.d.ts

@@ -0,0 +1,14 @@
+export declare abstract class AbstractProvider {
+    readonly kind: string;
+    constructor();
+    static withKind<TKind extends string, TThis extends abstract new (...args: any[]) => AbstractProvider>(this: TThis, kind: TKind): (new () => InstanceType<TThis> & {
+        readonly kind: TKind;
+    }) & {
+        readonly kind: TKind;
+    };
+}
+export type ProviderResponse<TData = unknown> = {
+    startedAt: number;
+    completedAt: number;
+    data: TData;
+};

package/dist/providers/example/echo.d.ts

@@ -0,0 +1,12 @@
+import { AbstractLLMProvider, ChatResponse, LLMProviderForwardArgs } from "../abstract/llm";
+/**
+ * Example provider implementation for local testing and as a reference.
+ *
+ * - Extends `AbstractLLMProvider`
+ * - Implements `forward({ messages, model, ... })`
+ * - Does not perform any network calls
+ */
+export declare class ExampleEchoLLMProvider extends AbstractLLMProvider {
+    readonly kind = "example.echo";
+    forward(args: LLMProviderForwardArgs): Promise<ChatResponse>;
+}

package/dist/providers/example/restapi.d.ts

@@ -0,0 +1,37 @@
+import { AbstractLLMProvider, ChatResponse, LLMProviderForwardArgs } from "../abstract/llm";
+/**
+ * Example "custom REST API LLM agent provider".
+ *
+ * Sometimes you don't call a model API directly. You call your own REST API, and *it* talks to the model.
+ * That REST API can hide secrets, run tools, do retrieval, apply guardrails, and whatever else your
+ * product needs.
+ *
+ * In the SDK we still want a clean abstraction, so we model that REST API as an `AbstractLLMProvider`.
+ * The runner (or host app) still passes `messages + model`, and the provider still returns one final string.
+ *
+ * If you’re implementing your own provider, this is the only part that matters: translate
+ * `LLMProviderForwardArgs` into your HTTP request, then translate your HTTP response back into
+ * `ChatResponse`.
+ */
+export declare class ExampleRestApiLLMAgentProvider extends AbstractLLMProvider {
+    readonly kind = "example.restapi.agent";
+    private readonly baseUrl;
+    private readonly apiKey?;
+    private readonly headers?;
+    constructor(config: ExampleRestApiAgentProviderConfig);
+    forward(args: LLMProviderForwardArgs): Promise<ChatResponse>;
+}
+type ExampleRestApiAgentProviderConfig = {
+    /**
+     * Base URL of your REST service.
+     * Example: `https://my-company.internal/api`
+     */
+    baseUrl: string;
+    /**
+     * Optional authentication.
+     * Many internal services accept `Authorization: Bearer <token>`, but you can also pass custom headers.
+     */
+    apiKey?: string;
+    headers?: Record<string, string>;
+};
+export {};

package/dist/providers/index.d.ts

@@ -1,84 +1,5 @@
-import { A as AbstractLLMProvider, L as LLMProviderForwardArgs, C as ChatResponse } from '../llm-BND163ns.js';
-export { A as AbstractProvider, P as ProviderResponse } from '../provider-BDjGp2y-.js';
-import { MastraClient, GetAgentResponse } from '@mastra/client-js';
-import { R as RateLimiter } from '../rate-limiter-CSmVIRsM.js';
-import { ChatCompletionMessageParam } from 'openai/resources/chat/completions';
-import { ResponseFormatText, ResponseFormatJSONSchema, ResponseFormatJSONObject } from 'openai/resources/shared';
-
-declare class MastraProvider extends AbstractLLMProvider {
-    readonly kind = "mastra";
-    private readonly endpoint;
-    private readonly authToken?;
-    private client;
-    private memory?;
-    constructor(params: {
-        endpoint: string;
-        authToken?: string;
-        memory?: AgentMemoryOption;
-    });
-    forward(args: LLMProviderForwardArgs & {
-        memory?: AgentMemoryOption;
-        /**
-         * The model that will be used as the brain for the agent.
-         */
-        modelName?: string;
-    }): Promise<ChatResponse>;
-    getAgentInfo(args: {
-        agentId: string;
-        runtimeContext?: MastraRuntimeContext;
-    }): Promise<GetAgentResponse>;
-    getAgents(args?: {
-        runtimeContext?: MastraRuntimeContext;
-        partial?: boolean;
-    }): Promise<Record<string, GetAgentResponse>>;
-}
-type AgentMemoryOption = Parameters<Parameters<MastraClient["getAgent"]>["0"] extends string ? ReturnType<MastraClient["getAgent"]>["generate"] : never>[0] extends {
-    memory?: infer M;
-} ? M : never;
-type MastraRuntimeContext = Parameters<Parameters<MastraClient["getAgent"]>["0"] extends string ? ReturnType<MastraClient["getAgent"]>["generate"] : never>[0] extends {
-    runtimeContext?: infer R;
-} ? R : never;
-
-declare class OpenAIProvider extends AbstractLLMProvider {
-    readonly kind: "peerbench.ai/llm/openai";
-    private client;
-    private rateLimiter;
-    private maxRetries;
-    constructor(config: {
-        apiKey: string;
-        baseURL: string;
-        maxRetries?: number;
-        timeout?: number;
-        rateLimiter?: RateLimiter;
-    });
-    forward(args: {
-        messages: ChatCompletionMessageParam[];
-        model: string;
-        abortSignal?: AbortSignal;
-        temperature?: number;
-        responseFormat?: ResponseFormatText | ResponseFormatJSONSchema | ResponseFormatJSONObject;
-    }): Promise<ChatResponse>;
-}
-
-declare class OpenRouterProvider extends AbstractLLMProvider {
-    readonly kind: "peerbench.ai/llm/openrouter.ai";
-    private models;
-    private modelsCachePromise;
-    private modelsUpdatedAt;
-    private openAIProvider;
-    constructor(config: {
-        apiKey: string;
-        maxRetries?: number;
-        timeout?: number;
-        rateLimiter?: RateLimiter;
-    });
-    forward(args: LLMProviderForwardArgs): Promise<ChatResponse>;
-    /**
-     * Updates the cache that holds information about OpenRouter models
-     * including pricing information. It will be valid for 24 hours as
-     * long as the instance of this Provider object is alive.
-     */
-    private updateModelsCache;
-}
-
-export { AbstractLLMProvider, type AgentMemoryOption, ChatResponse, LLMProviderForwardArgs, MastraProvider, OpenAIProvider, OpenRouterProvider };
+export * from "./abstract/llm";
+export * from "./abstract/provider";
+export * from "./mastra";
+export * from "./openai";
+export * from "./openrouter";

package/dist/providers/index.js

@@ -4,7 +4,7 @@ import {
   MastraProvider,
   OpenAIProvider,
   OpenRouterProvider
-} from "../chunk-3JF7SHLC.js";
+} from "../chunk-SMLNDQFX.js";
 import "../chunk-UHHHSYVE.js";
 import "../chunk-4UBK6452.js";
 import "../chunk-PZ5AY32C.js";

package/dist/providers/mastra.d.ts

@@ -0,0 +1,40 @@
+import { AbstractLLMProvider, type ChatResponse, type LLMProviderForwardArgs } from "./abstract/llm";
+import { MastraClient, type GetAgentResponse } from "@mastra/client-js";
+declare const MastraProvider_base: (new () => AbstractLLMProvider & {
+    readonly kind: "peerbench.ai/llm/mastra";
+}) & {
+    readonly kind: "peerbench.ai/llm/mastra";
+};
+export declare class MastraProvider extends MastraProvider_base {
+    private readonly endpoint;
+    private readonly authToken?;
+    private client;
+    private memory?;
+    constructor(params: {
+        endpoint: string;
+        authToken?: string;
+        memory?: AgentMemoryOption;
+    });
+    forward(args: LLMProviderForwardArgs & {
+        memory?: AgentMemoryOption;
+        /**
+         * The model that will be used as the brain for the agent.
+         */
+        modelName?: string;
+    }): Promise<ChatResponse>;
+    getAgentInfo(args: {
+        agentId: string;
+        runtimeContext?: MastraRuntimeContext;
+    }): Promise<GetAgentResponse>;
+    getAgents(args?: {
+        runtimeContext?: MastraRuntimeContext;
+        partial?: boolean;
+    }): Promise<Record<string, GetAgentResponse>>;
+}
+export type AgentMemoryOption = Parameters<Parameters<MastraClient["getAgent"]>["0"] extends string ? ReturnType<MastraClient["getAgent"]>["generate"] : never>[0] extends {
+    memory?: infer M;
+} ? M : never;
+type MastraRuntimeContext = Parameters<Parameters<MastraClient["getAgent"]>["0"] extends string ? ReturnType<MastraClient["getAgent"]>["generate"] : never>[0] extends {
+    runtimeContext?: infer R;
+} ? R : never;
+export {};

package/dist/providers/openai.d.ts

@@ -0,0 +1,29 @@
+import { RateLimiter } from "../utils";
+import { ChatCompletionMessageParam } from "openai/resources/chat/completions";
+import { ResponseFormatJSONObject, ResponseFormatJSONSchema, ResponseFormatText } from "openai/resources/shared";
+import { AbstractLLMProvider, ChatResponse } from "./abstract/llm";
+declare const OpenAIProvider_base: (new () => AbstractLLMProvider & {
+    readonly kind: "peerbench.ai/llm/openai";
+}) & {
+    readonly kind: "peerbench.ai/llm/openai";
+};
+export declare class OpenAIProvider extends OpenAIProvider_base {
+    private client;
+    private rateLimiter;
+    private maxRetries;
+    constructor(config: {
+        apiKey: string;
+        baseURL: string;
+        maxRetries?: number;
+        timeout?: number;
+        rateLimiter?: RateLimiter;
+    });
+    forward(args: {
+        messages: ChatCompletionMessageParam[];
+        model: string;
+        abortSignal?: AbortSignal;
+        temperature?: number;
+        responseFormat?: ResponseFormatText | ResponseFormatJSONSchema | ResponseFormatJSONObject;
+    }): Promise<ChatResponse>;
+}
+export {};

package/dist/providers/openrouter.d.ts

@@ -0,0 +1,27 @@
+import { AbstractLLMProvider, ChatResponse, LLMProviderForwardArgs } from "./abstract/llm";
+import { RateLimiter } from "../utils";
+declare const OpenRouterProvider_base: (new () => AbstractLLMProvider & {
+    readonly kind: "peerbench.ai/llm/openrouter.ai";
+}) & {
+    readonly kind: "peerbench.ai/llm/openrouter.ai";
+};
+export declare class OpenRouterProvider extends OpenRouterProvider_base {
+    private models;
+    private modelsCachePromise;
+    private modelsUpdatedAt;
+    private openAIProvider;
+    constructor(config: {
+        apiKey: string;
+        maxRetries?: number;
+        timeout?: number;
+        rateLimiter?: RateLimiter;
+    });
+    forward(args: LLMProviderForwardArgs): Promise<ChatResponse>;
+    /**
+     * Updates the cache that holds information about OpenRouter models
+     * including pricing information. It will be valid for 24 hours as
+     * long as the instance of this Provider object is alive.
+     */
+    private updateModelsCache;
+}
+export {};

package/dist/schemas/extensions/index.d.ts

@@ -1,28 +1,24 @@
-import * as z from 'zod';
-
-declare const ScoreExtensions: {
+export declare const ScoreExtensions: {
     ExtensionLLMAsAJudgeScoreFieldsV1: {
-        scorerAISystemPrompt: z.ZodOptional<z.ZodString>;
-        scorerAISystemPromptId: z.ZodOptional<z.ZodString>;
-        scorerAIProvider: z.ZodOptional<z.ZodString>;
-        scorerAIModelSlug: z.ZodOptional<z.ZodString>;
-        scorerAIInputTokensUsed: z.ZodOptional<z.ZodNumber>;
-        scorerAIOutputTokensUsed: z.ZodOptional<z.ZodNumber>;
-        scorerAIInputCost: z.ZodOptional<z.ZodString>;
-        scorerAIOutputCost: z.ZodOptional<z.ZodString>;
+        scorerAISystemPrompt: import("zod").ZodOptional<import("zod").ZodString>;
+        scorerAISystemPromptId: import("zod").ZodOptional<import("zod").ZodString>;
+        scorerAIProvider: import("zod").ZodOptional<import("zod").ZodString>;
+        scorerAIModelSlug: import("zod").ZodOptional<import("zod").ZodString>;
+        scorerAIInputTokensUsed: import("zod").ZodOptional<import("zod").ZodNumber>;
+        scorerAIOutputTokensUsed: import("zod").ZodOptional<import("zod").ZodNumber>;
+        scorerAIInputCost: import("zod").ZodOptional<import("zod").ZodString>;
+        scorerAIOutputCost: import("zod").ZodOptional<import("zod").ZodString>;
     };
 };
-declare const ResponseExtensions: {
+export declare const ResponseExtensions: {
     ExtensionLLMResponseFieldsV1: {
-        data: z.ZodString;
-        modelSlug: z.ZodString;
-        provider: z.ZodString;
-        systemPromptId: z.ZodOptional<z.ZodString>;
-        inputTokensUsed: z.ZodOptional<z.ZodNumber>;
-        outputTokensUsed: z.ZodOptional<z.ZodNumber>;
-        inputCost: z.ZodOptional<z.ZodString>;
-        outputCost: z.ZodOptional<z.ZodString>;
+        data: import("zod").ZodString;
+        modelSlug: import("zod").ZodString;
+        provider: import("zod").ZodString;
+        systemPromptId: import("zod").ZodOptional<import("zod").ZodString>;
+        inputTokensUsed: import("zod").ZodOptional<import("zod").ZodNumber>;
+        outputTokensUsed: import("zod").ZodOptional<import("zod").ZodNumber>;
+        inputCost: import("zod").ZodOptional<import("zod").ZodString>;
+        outputCost: import("zod").ZodOptional<import("zod").ZodString>;
     };
 };
-
-export { ResponseExtensions, ScoreExtensions };

package/dist/schemas/extensions/response/llm.d.ts

@@ -0,0 +1,14 @@
+import z from "zod";
+/**
+ * Provides a set of fields that holds information about the LLM and its response.
+ */
+export declare const ExtensionLLMResponseFieldsV1: {
+    data: z.ZodString;
+    modelSlug: z.ZodString;
+    provider: z.ZodString;
+    systemPromptId: z.ZodOptional<z.ZodString>;
+    inputTokensUsed: z.ZodOptional<z.ZodNumber>;
+    outputTokensUsed: z.ZodOptional<z.ZodNumber>;
+    inputCost: z.ZodOptional<z.ZodString>;
+    outputCost: z.ZodOptional<z.ZodString>;
+};

package/dist/schemas/extensions/score/llm-as-a-judge-scorer.d.ts

@@ -0,0 +1,15 @@
+import z from "zod";
+/**
+ * Provides a set of fields that holds information about the LLM model
+ * that was used to judge the response.
+ */
+export declare const ExtensionLLMAsAJudgeScoreFieldsV1: {
+    scorerAISystemPrompt: z.ZodOptional<z.ZodString>;
+    scorerAISystemPromptId: z.ZodOptional<z.ZodString>;
+    scorerAIProvider: z.ZodOptional<z.ZodString>;
+    scorerAIModelSlug: z.ZodOptional<z.ZodString>;
+    scorerAIInputTokensUsed: z.ZodOptional<z.ZodNumber>;
+    scorerAIOutputTokensUsed: z.ZodOptional<z.ZodNumber>;
+    scorerAIInputCost: z.ZodOptional<z.ZodString>;
+    scorerAIOutputCost: z.ZodOptional<z.ZodString>;
+};

package/dist/schemas/id.d.ts

@@ -0,0 +1,2 @@
+import z from "zod";
+export declare const IdSchema: z.ZodString;