peerbench 0.0.2-alpha.0 → 0.0.2-alpha.2

package/dist/chunk-VBOM2YEG.js

@@ -1,47 +0,0 @@
-// src/errors/polyfill.ts
-function captureStackTrace(error, constructor) {
-  if (typeof window === "undefined") {
-    Error.captureStackTrace(error, constructor);
-  } else {
-    error.stack = new Error(error.message).stack;
-  }
-}
-
-// src/errors/peerbench.ts
-var PeerbenchError = class extends Error {
-  code;
-  constructor(message, options) {
-    super(message, options);
-    this.code = options?.code ?? 0;
-    captureStackTrace(this, this.constructor);
-  }
-};
-
-// src/errors/catalog.ts
-var CatalogItemNotFoundError = class extends PeerbenchError {
-  constructor(itemName) {
-    super(
-      typeof itemName === "string" ? `Item with name "${itemName}" not found in the catalog` : itemName.message,
-      {
-        code: 100
-      }
-    );
-    this.itemName = itemName;
-  }
-};
-var CatalogItemHasNoInstantiateMethodError = class extends PeerbenchError {
-  constructor(itemName) {
-    super(`Item with name "${itemName}" has no instantiate method`, {
-      code: 101
-    });
-    this.itemName = itemName;
-  }
-};
-
-export {
-  captureStackTrace,
-  PeerbenchError,
-  CatalogItemNotFoundError,
-  CatalogItemHasNoInstantiateMethodError
-};
-//# sourceMappingURL=chunk-VBOM2YEG.js.map

package/dist/chunk-VBOM2YEG.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../src/errors/polyfill.ts","../src/errors/peerbench.ts","../src/errors/catalog.ts"],"sourcesContent":["// This is an internal polyfill that is used by custom Error classes\n// to make them work in both Node.js and browser environments.\n\nexport function captureStackTrace(\n  error: Error,\n  // eslint-disable-next-line @typescript-eslint/no-unsafe-function-type\n  constructor: Function\n) {\n  if (typeof window === \"undefined\") {\n    Error.captureStackTrace(error, constructor);\n  } else {\n    error.stack = new Error(error.message).stack;\n  }\n}\n","import { captureStackTrace } from \"./polyfill\";\n\nexport class PeerbenchError extends Error {\n  code: number;\n\n  constructor(message?: string, options?: ErrorOptions & { code: number }) {\n    super(message, options);\n    this.code = options?.code ?? 0;\n    captureStackTrace(this, this.constructor);\n  }\n}\n","import { PeerbenchError } from \"./peerbench\";\n\nexport class CatalogItemNotFoundError extends PeerbenchError {\n  constructor(public readonly itemName: string | { message: string }) {\n    super(\n      typeof itemName === \"string\"\n        ? `Item with name \"${itemName}\" not found in the catalog`\n        : itemName.message,\n      {\n        code: 100,\n      }\n    );\n  }\n}\n\nexport class CatalogItemHasNoInstantiateMethodError extends PeerbenchError {\n  constructor(public readonly itemName: string) {\n    super(`Item with name \"${itemName}\" has no instantiate method`, {\n      code: 101,\n    });\n  }\n}\n"],"mappings":";AAGO,SAAS,kBACd,OAEA,aACA;AACA,MAAI,OAAO,WAAW,aAAa;AACjC,UAAM,kBAAkB,OAAO,WAAW;AAAA,EAC5C,OAAO;AACL,UAAM,QAAQ,IAAI,MAAM,MAAM,OAAO,EAAE;AAAA,EACzC;AACF;;;ACXO,IAAM,iBAAN,cAA6B,MAAM;AAAA,EACxC;AAAA,EAEA,YAAY,SAAkB,SAA2C;AACvE,UAAM,SAAS,OAAO;AACtB,SAAK,OAAO,SAAS,QAAQ;AAC7B,sBAAkB,MAAM,KAAK,WAAW;AAAA,EAC1C;AACF;;;ACRO,IAAM,2BAAN,cAAuC,eAAe;AAAA,EAC3D,YAA4B,UAAwC;AAClE;AAAA,MACE,OAAO,aAAa,WAChB,mBAAmB,QAAQ,+BAC3B,SAAS;AAAA,MACb;AAAA,QACE,MAAM;AAAA,MACR;AAAA,IACF;AAR0B;AAAA,EAS5B;AACF;AAEO,IAAM,yCAAN,cAAqD,eAAe;AAAA,EACzE,YAA4B,UAAkB;AAC5C,UAAM,mBAAmB,QAAQ,+BAA+B;AAAA,MAC9D,MAAM;AAAA,IACR,CAAC;AAHyB;AAAA,EAI5B;AACF;","names":[]}

package/dist/chunk-ZJWSK4VO.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../src/types/index.ts"],"sourcesContent":["export * from \"./runner\";\n\nimport { IdSchema } from \"@/schemas/id\";\nimport z from \"zod\";\n\nexport type Id = z.infer<typeof IdSchema>;\n\nexport type IdGenerator<TInput = unknown> = (input: TInput) => MaybePromise<Id>;\n\nexport type MaybePromise<T> = T | Promise<T>;\n\nexport const ScoringMethod = {\n  ai: \"ai\",\n  human: \"human\",\n  algo: \"algo\",\n} as const;\nexport type ScoringMethod = (typeof ScoringMethod)[keyof typeof ScoringMethod];\n"],"mappings":";AAWO,IAAM,gBAAgB;AAAA,EAC3B,IAAI;AAAA,EACJ,OAAO;AAAA,EACP,MAAM;AACR;","names":[]}

package/dist/data-BmN5WjZ4.d.ts

@@ -1,57 +0,0 @@
-import { BaseTestCaseV1, BaseBenchmarkSpecV1, BaseResponseV1, BaseScoreV1 } from './schemas/index.js';
-import { A as AbstractProvider } from './provider-BDjGp2y-.js';
-import { I as IdGenerator, M as MaybePromise } from './index-WiPjF2AL.js';
-import { A as AbstractScorer } from './abstract-Dec9Sc5O.js';
-import { a as BaseSystemPromptV1 } from './system-prompt--0FdPWqK.js';
-
-/**
- * Minimum set of parameters that a runner needs to accept
- */
-type RunnerParams = {
-    testCase: BaseTestCaseV1;
-    provider: AbstractProvider;
-    spec?: BaseBenchmarkSpecV1;
-    scorer?: AbstractScorer;
-    runConfig: {
-        model: string;
-        llmJudgeModel?: string;
-    };
-    idGenerators?: {
-        response?: IdGenerator;
-        score?: IdGenerator;
-    };
-    [key: string]: unknown;
-};
-type RunnerResult<TResponse extends BaseResponseV1 = BaseResponseV1, TScore extends BaseScoreV1 = BaseScoreV1> = {
-    response: TResponse;
-    score?: TScore;
-    [key: string]: unknown;
-};
-type Runner<TResponse extends BaseResponseV1 = BaseResponseV1, TScore extends BaseScoreV1 = BaseScoreV1, TParams extends RunnerParams = RunnerParams> = (params: TParams) => Promise<RunnerResult<TResponse, TScore>>;
-type LLMChatRunnerParams = RunnerParams & {
-    systemPrompt?: BaseSystemPromptV1;
-};
-type LLMChatRunner<TResponse extends BaseResponseV1 = BaseResponseV1, TScore extends BaseScoreV1 = BaseScoreV1, TParams extends LLMChatRunnerParams = LLMChatRunnerParams> = (params: TParams) => Promise<RunnerResult<TResponse, TScore>>;
-
-type LoaderResult<TTestCase extends BaseTestCaseV1 = BaseTestCaseV1, TResponse extends BaseResponseV1 = BaseResponseV1, TScore extends BaseScoreV1 = BaseScoreV1, TBenchmarkSpec extends BaseBenchmarkSpecV1 = BaseBenchmarkSpecV1> = {
-    testCases: TTestCase[];
-    responses: TResponse[];
-    scores: TScore[];
-    benchmarkSpec?: TBenchmarkSpec;
-};
-declare abstract class AbstractLoader {
-    abstract readonly kind: string;
-    abstract loadData(params: unknown): MaybePromise<LoaderResult>;
-    abstract loadBenchmarkSpec(params: unknown): MaybePromise<BaseBenchmarkSpecV1>;
-}
-
-declare abstract class AbstractDataLoader extends AbstractLoader {
-    abstract loadData(params: {
-        content: Uint8Array;
-    }): MaybePromise<LoaderResult>;
-    abstract loadBenchmarkSpec(params: {
-        content: Uint8Array;
-    }): MaybePromise<BaseBenchmarkSpecV1>;
-}
-
-export { AbstractDataLoader as A, type LoaderResult as L, type Runner as R, AbstractLoader as a, type RunnerParams as b, type RunnerResult as c, type LLMChatRunnerParams as d, type LLMChatRunner as e };

package/dist/generic-array-DLHWSvf1.d.ts

@@ -1,22 +0,0 @@
-import { BaseTestCaseV1, BaseResponseV1, BaseScoreV1 } from './schemas/index.js';
-import { M as MaybePromise } from './index-WiPjF2AL.js';
-import { L as LoaderResult, A as AbstractDataLoader } from './data-BmN5WjZ4.js';
-
-type GenericJSONArrayLoaderResult<TTestCase extends BaseTestCaseV1 = BaseTestCaseV1, TResponse extends BaseResponseV1 = BaseResponseV1, TScore extends BaseScoreV1 = BaseScoreV1> = LoaderResult<TTestCase, TResponse, TScore>;
-declare abstract class GenericJSONArrayDataLoader<TTestCase extends BaseTestCaseV1 = BaseTestCaseV1, TResponse extends BaseResponseV1 = BaseResponseV1, TScore extends BaseScoreV1 = BaseScoreV1> extends AbstractDataLoader {
-    protected abstract testCaseBuilder(data: any, context: {
-        result: GenericJSONArrayLoaderResult<TTestCase, TResponse, TScore>;
-    }): MaybePromise<TTestCase | undefined>;
-    protected responseBuilder(data: any, context: {
-        result: GenericJSONArrayLoaderResult<TTestCase, TResponse, TScore>;
-    }): Promise<TResponse | undefined>;
-    protected scoreBuilder(data: any, context: {
-        result: GenericJSONArrayLoaderResult<TTestCase, TResponse, TScore>;
-    }): Promise<TScore | undefined>;
-    loadData(params: {
-        content: Uint8Array;
-    }): Promise<GenericJSONArrayLoaderResult<TTestCase, TResponse, TScore>>;
-    private transformArrayToResult;
-}
-
-export { type GenericJSONArrayLoaderResult as G, GenericJSONArrayDataLoader as a };

package/dist/index-WiPjF2AL.d.ts

@@ -1,15 +0,0 @@
-import z__default from 'zod';
-
-declare const IdSchema: z__default.ZodString;
-
-type Id = z__default.infer<typeof IdSchema>;
-type IdGenerator<TInput = unknown> = (input: TInput) => MaybePromise<Id>;
-type MaybePromise<T> = T | Promise<T>;
-declare const ScoringMethod: {
-    readonly ai: "ai";
-    readonly human: "human";
-    readonly algo: "algo";
-};
-type ScoringMethod = (typeof ScoringMethod)[keyof typeof ScoringMethod];
-
-export { type IdGenerator as I, type MaybePromise as M, ScoringMethod as S, type Id as a, IdSchema as b };

package/dist/llm-judge-DIG1f1Az.d.ts

@@ -1,67 +0,0 @@
-import { A as AbstractScorer, B as BaseScorerResult } from './abstract-Dec9Sc5O.js';
-import { A as AbstractLLMProvider } from './llm-DNj_tp2T.js';
-import { R as RateLimiter } from './rate-limiter-CSmVIRsM.js';
-
-type MCQScorerParams = {
-    response: string;
-    choices: Record<string, string>;
-    correctAnswers: string[];
-};
-declare class MCQScorer extends AbstractScorer {
-    readonly kind = "mcq";
-    private regexScorer;
-    score(params: MCQScorerParams): Promise<BaseScorerResult & {
-        extractedAnswers: string[];
-    }>;
-    private buildPatternsForAnswer;
-}
-
-declare class LLMJudgeScorer extends AbstractScorer {
-    readonly kind = "llmJudge";
-    private provider;
-    constructor(provider: AbstractLLMProvider);
-    score(params: LLMJudgeScorerParams): Promise<LLMJudgeScorerResult | null>;
-}
-type LLMJudgeCriterion = {
-    id: string;
-    description: string;
-    weight?: number;
-    scale?: {
-        min: number;
-        max: number;
-    };
-};
-type LLMJudgeScorerResult = BaseScorerResult & {
-    provider: string;
-    inputTokensUsed?: number;
-    outputTokensUsed?: number;
-    inputCost?: string;
-    outputCost?: string;
-    verdict?: "pass" | "borderline" | "fail";
-};
-type LLMJudgeScorerParams = {
-    task: string;
-    candidateAnswer: string;
-    referenceAnswer?: string;
-    model: string;
-    /**
-     * The rubric used for judging (defaults to a generic set).
-     */
-    criteria?: LLMJudgeCriterion[];
-    /**
-     * Optional extra context that the judge can use (constraints, references, etc.).
-     */
-    meta?: Record<string, unknown>;
-    /**
-     * Optional rate limiter wrapper for provider calls.
-     */
-    rateLimiter?: RateLimiter;
-    /**
-     * Optional prompt tweaks.
-     */
-    systemPrompt?: string;
-    promptPrefix?: string;
-    promptSuffix?: string;
-};
-
-export { LLMJudgeScorer as L, type MCQScorerParams as M, MCQScorer as a, type LLMJudgeCriterion as b, type LLMJudgeScorerResult as c, type LLMJudgeScorerParams as d };

package/dist/simple-system-prompt-CzPYuvo0.d.ts

@@ -1,49 +0,0 @@
-import { I as IdGenerator } from './index-WiPjF2AL.js';
-import { z } from 'zod';
-
-declare const SimpleSystemPromptSchemaV1: z.ZodObject<Omit<{
-    id: z.ZodString;
-    kind: z.ZodString;
-    schemaVersion: z.ZodNumber;
-    version: z.ZodNumber;
-    metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
-}, "kind" | "schemaVersion"> & {
-    content: z.ZodString;
-} & {
-    kind: z.ZodLiteral<"sys-prompt.simple">;
-    schemaVersion: z.ZodLiteral<1>;
-}, z.core.$strip> & {
-    new: (input: Omit<{
-        id: string;
-        version: number;
-        content: string;
-        kind: "sys-prompt.simple";
-        schemaVersion: 1;
-        metadata?: Record<string, unknown> | undefined;
-    }, "kind" | "schemaVersion">) => {
-        id: string;
-        version: number;
-        content: string;
-        kind: "sys-prompt.simple";
-        schemaVersion: 1;
-        metadata?: Record<string, unknown> | undefined;
-    };
-    newWithId(input: Omit<{
-        id: string;
-        version: number;
-        content: string;
-        kind: "sys-prompt.simple";
-        schemaVersion: 1;
-        metadata?: Record<string, unknown> | undefined;
-    }, "id" | "kind" | "schemaVersion">, generator: IdGenerator): Promise<{
-        id: string;
-        version: number;
-        content: string;
-        kind: "sys-prompt.simple";
-        schemaVersion: 1;
-        metadata?: Record<string, unknown> | undefined;
-    }>;
-};
-type SimpleSystemPromptV1 = z.infer<typeof SimpleSystemPromptSchemaV1>;
-
-export { SimpleSystemPromptSchemaV1 as S, type SimpleSystemPromptV1 as a };

package/dist/system-prompt--0FdPWqK.d.ts

@@ -1,58 +0,0 @@
-import { I as IdGenerator } from './index-WiPjF2AL.js';
-import { z } from 'zod';
-
-declare const BaseSystemPromptSchemaV1: z.ZodObject<{
-    id: z.ZodString;
-    kind: z.ZodString;
-    schemaVersion: z.ZodNumber;
-    version: z.ZodNumber;
-    metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
-}, z.core.$strip>;
-type BaseSystemPromptV1 = z.infer<typeof BaseSystemPromptSchemaV1>;
-declare const defineSystemPromptSchema: <TBaseSchema extends z.ZodObject<Omit<{
-    id: z.ZodString;
-    kind: z.ZodString;
-    schemaVersion: z.ZodNumber;
-    version: z.ZodNumber;
-    metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
-}, "kind" | "schemaVersion"> & {
-    kind: z.ZodString | z.ZodLiteral<string>;
-    schemaVersion: z.ZodNumber | z.ZodLiteral<number>;
-}, z.core.$strip>, TKind extends string | undefined = undefined, TSchemaVersion extends number | undefined = undefined, TFields extends Readonly<{
-    [k: string]: z.core.$ZodType<unknown, unknown, z.core.$ZodTypeInternals<unknown, unknown>>;
-}> = {}>(config: {
-    baseSchema: TBaseSchema;
-    kind?: TKind | undefined;
-    schemaVersion?: TSchemaVersion | undefined;
-    fields?: TFields | undefined;
-}) => (TBaseSchema extends z.ZodObject<infer U extends Readonly<{
-    [k: string]: z.core.$ZodType<unknown, unknown, z.core.$ZodTypeInternals<unknown, unknown>>;
-}>, z.core.$strip> ? z.ZodObject<Omit<U, "kind" | "schemaVersion"> & TFields & {
-    kind: TKind extends undefined ? U["kind"] : z.ZodLiteral<TKind>;
-    schemaVersion: TSchemaVersion extends undefined ? U["schemaVersion"] : z.ZodLiteral<TSchemaVersion>;
-}, z.core.$strip> : never) & {
-    new: (input: Omit<z.core.output<TBaseSchema extends z.ZodObject<infer U extends Readonly<{
-        [k: string]: z.core.$ZodType<unknown, unknown, z.core.$ZodTypeInternals<unknown, unknown>>;
-    }>, z.core.$strip> ? z.ZodObject<Omit<U, "kind" | "schemaVersion"> & TFields & {
-        kind: TKind extends undefined ? U["kind"] : z.ZodLiteral<TKind>;
-        schemaVersion: TSchemaVersion extends undefined ? U["schemaVersion"] : z.ZodLiteral<TSchemaVersion>;
-    }, z.core.$strip> : never>, "kind" | "schemaVersion">) => z.core.output<TBaseSchema extends z.ZodObject<infer U extends Readonly<{
-        [k: string]: z.core.$ZodType<unknown, unknown, z.core.$ZodTypeInternals<unknown, unknown>>;
-    }>, z.core.$strip> ? z.ZodObject<Omit<U, "kind" | "schemaVersion"> & TFields & {
-        kind: TKind extends undefined ? U["kind"] : z.ZodLiteral<TKind>;
-        schemaVersion: TSchemaVersion extends undefined ? U["schemaVersion"] : z.ZodLiteral<TSchemaVersion>;
-    }, z.core.$strip> : never>;
-    newWithId(input: Omit<z.core.output<TBaseSchema extends z.ZodObject<infer U extends Readonly<{
-        [k: string]: z.core.$ZodType<unknown, unknown, z.core.$ZodTypeInternals<unknown, unknown>>;
-    }>, z.core.$strip> ? z.ZodObject<Omit<U, "kind" | "schemaVersion"> & TFields & {
-        kind: TKind extends undefined ? U["kind"] : z.ZodLiteral<TKind>;
-        schemaVersion: TSchemaVersion extends undefined ? U["schemaVersion"] : z.ZodLiteral<TSchemaVersion>;
-    }, z.core.$strip> : never>, "kind" | "schemaVersion" | "id">, generator: IdGenerator): Promise<z.core.output<TBaseSchema extends z.ZodObject<infer U extends Readonly<{
-        [k: string]: z.core.$ZodType<unknown, unknown, z.core.$ZodTypeInternals<unknown, unknown>>;
-    }>, z.core.$strip> ? z.ZodObject<Omit<U, "kind" | "schemaVersion"> & TFields & {
-        kind: TKind extends undefined ? U["kind"] : z.ZodLiteral<TKind>;
-        schemaVersion: TSchemaVersion extends undefined ? U["schemaVersion"] : z.ZodLiteral<TSchemaVersion>;
-    }, z.core.$strip> : never>>;
-};
-
-export { BaseSystemPromptSchemaV1 as B, type BaseSystemPromptV1 as a, defineSystemPromptSchema as d };

package/dist/utilities-BrRH32rD.d.ts

@@ -1,30 +0,0 @@
-import { M as MaybePromise } from './index-WiPjF2AL.js';
-import { a as AbstractLoader, L as LoaderResult } from './data-BmN5WjZ4.js';
-import { BufferEncodingOption } from 'node:fs';
-import { BaseBenchmarkSpecV1 } from './schemas/index.js';
-
-declare abstract class AbstractFileLoader extends AbstractLoader {
-    abstract loadData(params: {
-        path: string;
-        encoding?: BufferEncodingOption;
-    }): MaybePromise<LoaderResult>;
-    abstract loadBenchmarkSpec(params: {
-        path: string;
-        encoding?: BufferEncodingOption;
-    }): MaybePromise<BaseBenchmarkSpecV1>;
-}
-
-declare abstract class AbstractHttpLoader extends AbstractLoader {
-    abstract loadData(params: {
-        url: string;
-        options?: RequestInit;
-    }): MaybePromise<LoaderResult>;
-    abstract loadBenchmarkSpec(params: {
-        url: string;
-        options?: RequestInit;
-    }): MaybePromise<BaseBenchmarkSpecV1>;
-}
-
-type ClassConstructor<T> = new (...args: any[]) => T;
-
-export { AbstractFileLoader as A, type ClassConstructor as C, AbstractHttpLoader as a };