peerbench 0.0.7 → 0.0.8

package/dist/storages/http.d.ts

@@ -0,0 +1,22 @@
+import { AbstractStorage } from "../storages/abstract";
+export declare class HttpStorage<TObject> extends AbstractStorage<TObject> {
+    protected readonly url: string;
+    protected readonly codec: AbstractHttpStorageCodec<TObject>;
+    protected readonly fetchFn: typeof fetch;
+    constructor(config: {
+        url: string;
+        codec: AbstractHttpStorageCodec<TObject>;
+        fetchFn?: typeof fetch;
+    });
+    init(_params?: unknown): Promise<void>;
+    read(_key: string, _params?: unknown): Promise<TObject | null>;
+    readAll(params?: unknown): Promise<TObject[]>;
+    write(_key: string, _value: TObject, _params?: unknown): Promise<unknown>;
+}
+export declare abstract class AbstractHttpStorageCodec<TObject> {
+    abstract readAll(params: {
+        url: string;
+        response: Response;
+        params?: unknown;
+    }): Promise<TObject[]>;
+}

package/dist/storages/index.d.ts

@@ -1,69 +1,5 @@
-import { A as AbstractStorage } from '../json-file-Bgv9TLcX.js';
-export { c as AbstractFileStorageCodec, F as FileStorage, b as FileStorageFileHandle, J as JSONFileStorage, a as JSONFileStorageCodec } from '../json-file-Bgv9TLcX.js';
-import * as better_sqlite3 from 'better-sqlite3';
-import 'node:fs/promises';
-import 'zod';
-
-declare class HttpStorage<TObject> extends AbstractStorage<TObject> {
-    protected readonly url: string;
-    protected readonly codec: AbstractHttpStorageCodec<TObject>;
-    protected readonly fetchFn: typeof fetch;
-    constructor(config: {
-        url: string;
-        codec: AbstractHttpStorageCodec<TObject>;
-        fetchFn?: typeof fetch;
-    });
-    init(_params?: unknown): Promise<void>;
-    read(_key: string, _params?: unknown): Promise<TObject | null>;
-    readAll(params?: unknown): Promise<TObject[]>;
-    write(_key: string, _value: TObject, _params?: unknown): Promise<unknown>;
-}
-declare abstract class AbstractHttpStorageCodec<TObject> {
-    abstract readAll(params: {
-        url: string;
-        response: Response;
-        params?: unknown;
-    }): Promise<TObject[]>;
-}
-
-declare class SQLiteStorage<TObject> extends AbstractStorage<TObject> {
-    protected readonly path: string;
-    protected readonly codec: AbstractSQLiteStorageCodec<TObject>;
-    protected db: SQLiteDatabase | undefined;
-    constructor(config: {
-        path: string;
-        codec: AbstractSQLiteStorageCodec<TObject>;
-    });
-    init(params?: unknown): Promise<void>;
-    read(key: string, params?: unknown): Promise<TObject | null>;
-    readAll(params?: unknown): Promise<TObject[]>;
-    write(key: string, value: TObject, params?: unknown): Promise<unknown>;
-    close(): void;
-    protected assertInitialized(): asserts this is this & {
-        db: SQLiteDatabase;
-    };
-}
-declare abstract class AbstractSQLiteStorageCodec<TObject> {
-    init?(params: {
-        db: SQLiteDatabase;
-        params?: unknown;
-    }): Promise<void> | void;
-    abstract read(params: {
-        db: SQLiteDatabase;
-        key: string;
-        params?: unknown;
-    }): Promise<TObject | null>;
-    abstract readAll(params: {
-        db: SQLiteDatabase;
-        params?: unknown;
-    }): Promise<TObject[]>;
-    abstract write(params: {
-        db: SQLiteDatabase;
-        key: string;
-        value: TObject;
-        params?: unknown;
-    }): Promise<unknown>;
-}
-type SQLiteDatabase = better_sqlite3.Database;
-
-export { AbstractHttpStorageCodec, AbstractSQLiteStorageCodec, AbstractStorage, HttpStorage, SQLiteStorage };
+export * from "./abstract";
+export * from "./http";
+export * from "./json-file";
+export * from "./file";
+export * from "./sqlite";

package/dist/storages/json-file.d.ts

@@ -0,0 +1,21 @@
+import { AbstractFileStorageCodec, FileStorage, FileStorageFileHandle } from "./file";
+import z from "zod";
+export declare class JSONFileStorage<TObject> extends FileStorage<TObject> {
+    codec: JSONFileStorageCodec<TObject>;
+    constructor(config: {
+        path: string;
+        chunkSize?: number;
+        schema: z.ZodType<TObject>;
+    });
+}
+export declare class JSONFileStorageCodec<TObject> extends AbstractFileStorageCodec<TObject> {
+    private chunkSize;
+    private schema;
+    constructor(config: {
+        chunkSize?: number;
+        schema: z.ZodType<TObject>;
+    });
+    readAll(params: {
+        fileHandle: FileStorageFileHandle;
+    }): Promise<TObject[]>;
+}

package/dist/storages/sqlite.d.ts

@@ -0,0 +1,41 @@
+import { AbstractStorage } from "../storages/abstract";
+export declare class SQLiteStorage<TObject> extends AbstractStorage<TObject> {
+    protected readonly path: string;
+    protected readonly codec: AbstractSQLiteStorageCodec<TObject>;
+    protected db: SQLiteDatabase | undefined;
+    constructor(config: {
+        path: string;
+        codec: AbstractSQLiteStorageCodec<TObject>;
+    });
+    init(params?: unknown): Promise<void>;
+    read(key: string, params?: unknown): Promise<TObject | null>;
+    readAll(params?: unknown): Promise<TObject[]>;
+    write(key: string, value: TObject, params?: unknown): Promise<unknown>;
+    close(): void;
+    protected assertInitialized(): asserts this is this & {
+        db: SQLiteDatabase;
+    };
+}
+export declare abstract class AbstractSQLiteStorageCodec<TObject> {
+    init?(params: {
+        db: SQLiteDatabase;
+        params?: unknown;
+    }): Promise<void> | void;
+    abstract read(params: {
+        db: SQLiteDatabase;
+        key: string;
+        params?: unknown;
+    }): Promise<TObject | null>;
+    abstract readAll(params: {
+        db: SQLiteDatabase;
+        params?: unknown;
+    }): Promise<TObject[]>;
+    abstract write(params: {
+        db: SQLiteDatabase;
+        key: string;
+        value: TObject;
+        params?: unknown;
+    }): Promise<unknown>;
+}
+type SQLiteDatabase = import("better-sqlite3").Database;
+export {};

package/dist/types/index.d.ts

@@ -0,0 +1,17 @@
+export * from "./runner";
+import { AbstractProvider } from "../providers/index.js";
+import { IdSchema } from "../schemas/id";
+import { AbstractScorer } from "../scorers/abstract";
+import { AbstractClassConstructor, ClassConstructor } from "../utilities";
+import z from "zod";
+export type Id = z.infer<typeof IdSchema>;
+export type IdGenerator<TInput = unknown> = (input: TInput) => MaybePromise<Id>;
+export type MaybePromise<T> = T | Promise<T>;
+export declare const ScoringMethod: {
+    readonly ai: "ai";
+    readonly human: "human";
+    readonly algo: "algo";
+};
+export type ScoringMethod = (typeof ScoringMethod)[keyof typeof ScoringMethod];
+export type ScorerCtor = ClassConstructor<AbstractScorer> | AbstractClassConstructor<AbstractScorer>;
+export type ProviderCtor = ClassConstructor<AbstractProvider> | AbstractClassConstructor<AbstractProvider>;

package/dist/types/runner.d.ts

@@ -0,0 +1,18 @@
+import { AbstractProvider } from "../providers/index.js";
+import { AbstractScorer } from "../scorers/index.js";
+import { IdGenerator } from ".";
+import z from "zod";
+export type Runner<TTestCase extends z.ZodObject, TResponse extends z.ZodObject, TScore extends z.ZodObject, TProvider extends AbstractProvider, TScorer extends AbstractScorer, TRunConfig extends Record<string, unknown>> = (params: {
+    testCase: z.infer<TTestCase>;
+    provider: TProvider;
+    scorer?: TScorer;
+    runConfig: TRunConfig;
+    idGenerators?: {
+        response?: IdGenerator;
+        score?: IdGenerator;
+    };
+}) => Promise<{
+    response: z.infer<TResponse>;
+    score?: z.infer<TScore>;
+}>;
+export type InferRunConfig<TRunConfigSchema extends z.ZodRawShape> = z.infer<z.ZodObject<TRunConfigSchema>>;

package/dist/utilities.d.ts

@@ -0,0 +1,9 @@
+import type { z } from "zod";
+export type ClassConstructor<T> = new (...args: any[]) => T;
+export type AbstractClassConstructor<T> = abstract new (...args: any[]) => T;
+export type InferExtension<TExtension extends z.ZodRawShape, TBase = unknown> = z.infer<z.ZodObject<TExtension>> & TBase;
+type WidenLiteralSchema<S> = S extends z.ZodString ? z.ZodString | z.ZodLiteral<string> : S extends z.ZodNumber ? z.ZodNumber | z.ZodLiteral<number> : S extends z.ZodBoolean ? z.ZodBoolean | z.ZodLiteral<boolean> : S extends z.ZodLiteral<infer L> ? L extends string ? z.ZodString | z.ZodLiteral<L> : L extends number ? z.ZodNumber | z.ZodLiteral<L> : L extends boolean ? z.ZodBoolean | z.ZodLiteral<L> : S : S extends z.ZodOptional<infer O> ? z.ZodOptional<WidenLiteralSchema<O>> : S extends z.ZodType ? S : never;
+export type WidenZodObject<T extends z.ZodObject> = z.ZodObject<{
+    [K in keyof T["shape"]]: WidenLiteralSchema<T["shape"][K]>;
+}>;
+export {};

package/dist/utils/id-generator.d.ts

@@ -0,0 +1,2 @@
+import { IdGenerator } from "../types";
+export declare const idGeneratorUUIDv7: IdGenerator;

package/dist/utils/index.d.ts

@@ -0,0 +1,5 @@
+export * from "./sleep";
+export * from "./llm";
+export * from "./rate-limiter";
+export * from "./string";
+export * from "./id-generator";

package/dist/utils/json.d.ts

@@ -0,0 +1,17 @@
+/**
+ * Parses JSONL formatted string into an array
+ * @returns An array of parsed JSON lines
+ */
+export declare function parseJSONL<T>(str: string, options?: {
+    errorOnInvalid?: boolean;
+}): T[];
+/**
+ * Tries to parse the given string as JSON.
+ * Returns `undefined` if it is not a valid JSON entity.
+ */
+export declare function tryParseJson<T = any>(content: string): T | undefined;
+/**
+ * Stringifies the given JSON value using `safe-stable-stringify` in a stable manner.
+ * This stable method generates the same string output for the same input value (including objects).
+ */
+export declare function stableStringify(value: any): string | undefined;

package/dist/utils/llm.d.ts

@@ -0,0 +1,7 @@
+/**
+ * Tries to repair and parse LLM response as a JSON object. LLM must
+ * be configured to return a JSON object. This function only helps to
+ * get rid out of some additional formatting (e.g. ```json) and repair
+ * the JSON syntax (e.g missing comma, single quotes instead double).
+ */
+export declare function parseResponseAsJSON<T>(response: string): T | undefined;

package/dist/{rate-limiter-CSmVIRsM.d.ts → utils/rate-limiter.d.ts}

@@ -1,4 +1,4 @@
-interface RateLimiterOptions {
+export interface RateLimiterOptions {
     /**
      * Maximum weight of calls allowed within a time window. Each call has a weight of 1 by default.
      * Rate limiting is disabled when this is 0 or negative.
@@ -11,7 +11,7 @@ interface RateLimiterOptions {
      */
     timeWindow?: number;
 }
-interface RateLimiterCallOptions {
+export interface RateLimiterCallOptions {
     /**
      * Weight of the call. Rate limiting is applied based on the total weight of the calls.
      * @default 1
@@ -26,7 +26,7 @@ interface RateLimiterCallOptions {
  * Generic rate limiter. It can be used to limit async function calls
  * by a given number of calls within a time window.
  */
-declare class RateLimiter {
+export declare class RateLimiter {
     maxWeight: number;
     timeWindow: number;
     private timestamps;
@@ -56,5 +56,3 @@ declare class RateLimiter {
      */
     reset(): void;
 }
-
-export { RateLimiter as R, type RateLimiterOptions as a, type RateLimiterCallOptions as b };

package/dist/utils/sleep.d.ts

@@ -0,0 +1 @@
+export declare function sleep(ms: number, signal?: AbortSignal): Promise<void>;

package/dist/utils/string.d.ts

@@ -0,0 +1,8 @@
+/**
+ * Converts the given byte array to a string
+ */
+export declare function bufferToString(buffer: Uint8Array, encoding?: BufferEncoding): string;
+/**
+ * Converts the given string to a byte array
+ */
+export declare function stringToBuffer(str: string): Uint8Array;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "peerbench",
-  "version": "0.0.7",
+  "version": "0.0.8",
   "description": "Tooling help work with AI benchmarking data and create trustworthy AI",
   "keywords": [
     "ai",
@@ -78,8 +78,8 @@
   },
   "scripts": {
     "lint": "tsc && eslint",
-    "dev": "tsup --silent --watch --onSuccess 'tsc-alias -p tsconfig.json -f'",
-    "build": "tsup --onSuccess 'tsc-alias -p tsconfig.json -f'",
+    "dev": "tsup --silent --watch --onSuccess 'tsc -p tsconfig.build.json && tsc-alias -p tsconfig.build.json -f'",
+    "build": "tsup && tsc -p tsconfig.build.json && tsc-alias -p tsconfig.build.json -f",
     "test": "vitest run",
     "clean": "rm -rf dist"
   },

package/dist/index-Cn20kPrz.d.ts

@@ -1,27 +0,0 @@
-import { A as AbstractProvider } from './provider-DnEBdl1n.js';
-import z__default, { z } from 'zod';
-import { A as AbstractScorer } from './abstract-BdgLjkNC.js';
-
-declare const IdSchema: z__default.ZodString;
-
-type ClassConstructor<T> = new (...args: any[]) => T;
-type AbstractClassConstructor<T> = abstract new (...args: any[]) => T;
-type InferExtension<TExtension extends z.ZodRawShape, TBase = unknown> = z.infer<z.ZodObject<TExtension>> & TBase;
-type WidenLiteralSchema<S> = S extends z.ZodString ? z.ZodString | z.ZodLiteral<string> : S extends z.ZodNumber ? z.ZodNumber | z.ZodLiteral<number> : S extends z.ZodBoolean ? z.ZodBoolean | z.ZodLiteral<boolean> : S extends z.ZodLiteral<infer L> ? L extends string ? z.ZodString | z.ZodLiteral<L> : L extends number ? z.ZodNumber | z.ZodLiteral<L> : L extends boolean ? z.ZodBoolean | z.ZodLiteral<L> : S : S extends z.ZodOptional<infer O> ? z.ZodOptional<WidenLiteralSchema<O>> : S extends z.ZodType ? S : never;
-type WidenZodObject<T extends z.ZodObject> = z.ZodObject<{
-    [K in keyof T["shape"]]: WidenLiteralSchema<T["shape"][K]>;
-}>;
-
-type Id = z__default.infer<typeof IdSchema>;
-type IdGenerator<TInput = unknown> = (input: TInput) => MaybePromise<Id>;
-type MaybePromise<T> = T | Promise<T>;
-declare const ScoringMethod: {
-    readonly ai: "ai";
-    readonly human: "human";
-    readonly algo: "algo";
-};
-type ScoringMethod = (typeof ScoringMethod)[keyof typeof ScoringMethod];
-type ScorerCtor = ClassConstructor<AbstractScorer> | AbstractClassConstructor<AbstractScorer>;
-type ProviderCtor = ClassConstructor<AbstractProvider> | AbstractClassConstructor<AbstractProvider>;
-
-export { type AbstractClassConstructor as A, type ClassConstructor as C, type IdGenerator as I, type MaybePromise as M, type ProviderCtor as P, type ScorerCtor as S, type WidenZodObject as W, type InferExtension as a, type Id as b, ScoringMethod as c, IdSchema as d };

package/dist/json-file-Bgv9TLcX.d.ts

@@ -1,74 +0,0 @@
-import { FileHandle } from 'node:fs/promises';
-import z__default from 'zod';
-
-declare abstract class AbstractStorage<TObject> {
-    abstract init(params?: unknown): Promise<void>;
-    abstract read(key: string, params?: unknown): Promise<TObject | null>;
-    abstract readAll(params?: unknown): Promise<TObject[]>;
-    abstract write(key: string, value: TObject, params?: unknown): Promise<unknown>;
-    count(): Promise<number>;
-}
-
-declare class FileStorage<TObject> extends AbstractStorage<TObject> {
-    protected path: string;
-    protected codec: AbstractFileStorageCodec<TObject>;
-    protected fileHandle: FileHandle | undefined;
-    constructor(config: {
-        path: string;
-        codec: AbstractFileStorageCodec<TObject>;
-    });
-    init(): Promise<void>;
-    read(_key: string, _params?: unknown): Promise<TObject | null>;
-    readAll(_params?: unknown): Promise<TObject[]>;
-    write(_key: string, _value: TObject, _params?: unknown): Promise<unknown>;
-    protected readChunks(params: {
-        startOffset?: number;
-        chunkSize: number;
-    }): AsyncIterable<{
-        offset: number;
-        bytes: Uint8Array;
-    }>;
-    protected size(): Promise<number>;
-    protected readAt(offset: number, length: number): Promise<Uint8Array>;
-    protected assertInitialized(): asserts this is this & {
-        fileHandle: FileHandle;
-    };
-}
-type FileStorageFileHandle = {
-    size(): Promise<number>;
-    readAt(offset: number, length: number): Promise<Uint8Array>;
-    readChunks(params: {
-        startOffset?: number;
-        chunkSize: number;
-    }): AsyncIterable<{
-        offset: number;
-        bytes: Uint8Array;
-    }>;
-};
-declare abstract class AbstractFileStorageCodec<TObject> {
-    abstract readAll(params: {
-        fileHandle: FileStorageFileHandle;
-    }): Promise<TObject[]>;
-}
-
-declare class JSONFileStorage<TObject> extends FileStorage<TObject> {
-    codec: JSONFileStorageCodec<TObject>;
-    constructor(config: {
-        path: string;
-        chunkSize?: number;
-        schema: z__default.ZodType<TObject>;
-    });
-}
-declare class JSONFileStorageCodec<TObject> extends AbstractFileStorageCodec<TObject> {
-    private chunkSize;
-    private schema;
-    constructor(config: {
-        chunkSize?: number;
-        schema: z__default.ZodType<TObject>;
-    });
-    readAll(params: {
-        fileHandle: FileStorageFileHandle;
-    }): Promise<TObject[]>;
-}
-
-export { AbstractStorage as A, FileStorage as F, JSONFileStorage as J, JSONFileStorageCodec as a, type FileStorageFileHandle as b, AbstractFileStorageCodec as c };

package/dist/llm-8ecJmwKJ.d.ts

@@ -1,23 +0,0 @@
-import { A as AbstractProvider, P as ProviderResponse } from './provider-DnEBdl1n.js';
-import { ResponseFormatText, ResponseFormatJSONSchema, ResponseFormatJSONObject } from 'openai/resources/shared';
-import { ChatCompletionMessageParam } from 'openai/resources/chat/completions';
-
-declare abstract class AbstractLLMProvider extends AbstractProvider {
-    abstract forward(args: LLMProviderForwardArgs): Promise<ChatResponse>;
-}
-type LLMProviderForwardArgs = {
-    messages: ChatCompletionMessageParam[];
-    model: string;
-    abortSignal?: AbortSignal;
-    temperature?: number;
-    responseFormat?: ResponseFormatText | ResponseFormatJSONSchema | ResponseFormatJSONObject;
-};
-type ChatResponse = ProviderResponse<string> & {
-    inputTokensUsed?: number;
-    outputTokensUsed?: number;
-    inputCost?: string;
-    outputCost?: string;
-    metadata?: Record<string, unknown>;
-};
-
-export { AbstractLLMProvider as A, type ChatResponse as C, type LLMProviderForwardArgs as L };

package/dist/llm-judge-BuF80-5-.d.ts

@@ -1,75 +0,0 @@
-import { A as AbstractScorer, B as BaseScorerResult } from './abstract-BdgLjkNC.js';
-import { A as AbstractLLMProvider } from './llm-8ecJmwKJ.js';
-import { R as RateLimiter } from './rate-limiter-CSmVIRsM.js';
-import z__default from 'zod';
-
-type MCQScorerParams = {
-    response: string;
-    choices: Record<string, string>;
-    correctAnswers: string[];
-};
-declare const MCQScorer_base: (new () => AbstractScorer & {
-    readonly kind: "peerbench.ai/mcq";
-}) & {
-    readonly kind: "peerbench.ai/mcq";
-};
-declare class MCQScorer extends MCQScorer_base {
-    private regexScorer;
-    score(params: MCQScorerParams): Promise<BaseScorerResult & {
-        extractedAnswers: string[];
-    }>;
-    private buildPatternsForAnswer;
-}
-
-declare const LLMAsAJudgeScorer_base: (new () => AbstractScorer & {
-    readonly kind: "peerbench.ai/llm-as-a-judge";
-}) & {
-    readonly kind: "peerbench.ai/llm-as-a-judge";
-};
-declare class LLMAsAJudgeScorer extends LLMAsAJudgeScorer_base {
-    private provider;
-    constructor(config: {
-        provider: AbstractLLMProvider;
-        rateLimiter?: RateLimiter;
-    });
-    score<T extends z__default.ZodRawShape>(params: LLMAsAJudgeScoreParams & {
-        fieldsToExtract: T;
-    }): Promise<ScorerResultWithExtractedFields<T> | null>;
-    score(params: LLMAsAJudgeScoreParams & {
-        fieldsToExtract?: never;
-    }): Promise<ScorerResultWithoutExtractedFields | null>;
-}
-type LLMAsAJudgeCriterion = {
-    id: string;
-    description: string;
-    weight: number;
-    scale?: {
-        min: number;
-        max: number;
-    };
-};
-type LLMAsAJudgeScoreParams = {
-    model: string;
-    response: string;
-    rubric: string;
-    criteria: LLMAsAJudgeCriterion[];
-    systemPrompt?: string;
-    maxExplanationLength?: number;
-};
-type ScorerResultWithoutExtractedFields = BaseScorerResult & {
-    results: {
-        id: string;
-        score: number;
-        explanation: string;
-    }[];
-    provider: string;
-    inputTokensUsed?: number;
-    outputTokensUsed?: number;
-    inputCost?: string;
-    outputCost?: string;
-};
-type ScorerResultWithExtractedFields<T extends z__default.ZodRawShape> = ScorerResultWithoutExtractedFields & {
-    extractedFields: z__default.infer<z__default.ZodObject<T>>;
-};
-
-export { LLMAsAJudgeScorer as L, type MCQScorerParams as M, MCQScorer as a, type LLMAsAJudgeCriterion as b, type LLMAsAJudgeScoreParams as c };