peerbench 0.0.7 → 0.0.8

package/dist/aggregators/abstract.d.ts

@@ -0,0 +1,10 @@
+import { BaseResponseV1, BaseTestCaseV1 } from "../schemas/index.js";
+import { BaseScoreV1 } from "../schemas/score";
+export declare abstract class AbstractAggregator {
+    abstract push(params: {
+        score: BaseScoreV1;
+        testCase?: BaseTestCaseV1;
+        response?: BaseResponseV1;
+    }): Promise<void>;
+    abstract aggregate(config?: unknown): Promise<unknown>;
+}

package/dist/aggregators/index.d.ts

@@ -1,67 +1,2 @@
-import { BaseScoreV1, BaseTestCaseV1, BaseResponseV1 } from '../schemas/index.js';
-import { a as InferExtension } from '../index-Cn20kPrz.js';
-import z__default from 'zod';
-import '../provider-DnEBdl1n.js';
-import '../abstract-BdgLjkNC.js';
-
-declare abstract class AbstractAggregator {
-    abstract push(params: {
-        score: BaseScoreV1;
-        testCase?: BaseTestCaseV1;
-        response?: BaseResponseV1;
-    }): Promise<void>;
-    abstract aggregate(config?: unknown): Promise<unknown>;
-}
-
-/**
- * Provides a set of fields that holds information about the LLM and its response.
- */
-declare const ExtensionLLMResponseFieldsV1: {
-    data: z__default.ZodString;
-    modelSlug: z__default.ZodString;
-    provider: z__default.ZodString;
-    systemPromptId: z__default.ZodOptional<z__default.ZodString>;
-    inputTokensUsed: z__default.ZodOptional<z__default.ZodNumber>;
-    outputTokensUsed: z__default.ZodOptional<z__default.ZodNumber>;
-    inputCost: z__default.ZodOptional<z__default.ZodString>;
-    outputCost: z__default.ZodOptional<z__default.ZodString>;
-};
-
-/**
- * Provides a set of fields that holds information about the LLM model
- * that was used to judge the response.
- */
-declare const ExtensionLLMAsAJudgeScoreFieldsV1: {
-    scorerAISystemPrompt: z__default.ZodOptional<z__default.ZodString>;
-    scorerAISystemPromptId: z__default.ZodOptional<z__default.ZodString>;
-    scorerAIProvider: z__default.ZodOptional<z__default.ZodString>;
-    scorerAIModelSlug: z__default.ZodOptional<z__default.ZodString>;
-    scorerAIInputTokensUsed: z__default.ZodOptional<z__default.ZodNumber>;
-    scorerAIOutputTokensUsed: z__default.ZodOptional<z__default.ZodNumber>;
-    scorerAIInputCost: z__default.ZodOptional<z__default.ZodString>;
-    scorerAIOutputCost: z__default.ZodOptional<z__default.ZodString>;
-};
-
-declare class AvgAggregator extends AbstractAggregator {
-    private separateBySystemPrompt;
-    private scores;
-    constructor(params: {
-        separateBySystemPrompt?: boolean;
-    });
-    push(params: {
-        score: InferExtension<typeof ExtensionLLMAsAJudgeScoreFieldsV1, BaseScoreV1>;
-        response: InferExtension<typeof ExtensionLLMResponseFieldsV1, BaseResponseV1>;
-        testCase?: BaseTestCaseV1;
-    }): Promise<void>;
-    aggregate(): Promise<{
-        [k: string]: {
-            average: number;
-            model: string;
-            total: number;
-            count: number;
-            systemPromptId?: string;
-        };
-    }>;
-}
-
-export { AbstractAggregator, AvgAggregator };
+export * from "./abstract";
+export * from "./llm/avg";

package/dist/aggregators/llm/avg.d.ts

@@ -0,0 +1,26 @@
+import { BaseResponseV1, BaseScoreV1, BaseTestCaseV1 } from "../../schemas/index.js";
+import { AbstractAggregator } from "../abstract";
+import { InferExtension } from "../../utilities";
+import { ExtensionLLMResponseFieldsV1 } from "../../schemas/extensions/response/llm";
+import { ExtensionLLMAsAJudgeScoreFieldsV1 } from "../../schemas/extensions/score/llm-as-a-judge-scorer";
+export declare class AvgAggregator extends AbstractAggregator {
+    private separateBySystemPrompt;
+    private scores;
+    constructor(params: {
+        separateBySystemPrompt?: boolean;
+    });
+    push(params: {
+        score: InferExtension<typeof ExtensionLLMAsAJudgeScoreFieldsV1, BaseScoreV1>;
+        response: InferExtension<typeof ExtensionLLMResponseFieldsV1, BaseResponseV1>;
+        testCase?: BaseTestCaseV1;
+    }): Promise<void>;
+    aggregate(): Promise<{
+        [k: string]: {
+            average: number;
+            model: string;
+            total: number;
+            count: number;
+            systemPromptId?: string;
+        };
+    }>;
+}

package/dist/benchmarks/examples/echo-basic/index.d.ts

@@ -0,0 +1,4 @@
+export * from "./runner";
+export * from "./schema-sets/echo.v1";
+export * from "./storages/json";
+export * from "./storages/text";

package/dist/benchmarks/examples/echo-basic/runner.d.ts

@@ -0,0 +1,273 @@
+import { AbstractLLMProvider } from "../../../providers/index.js";
+import { z } from "zod";
+/**
+ * Runners are the backbone of a benchmark. They are responsible for executing the test cases and producing
+ * the responses and scores. As the benchmark builder, you define what schemas the runner can work with,
+ * what are the providers and scorers are supported and what configurations can be passed by the caller
+ * at the execution phase.
+ */
+export declare const echoBasicRunner: ((params: {
+    testCase: {
+        id: string;
+        input: string;
+        namespace: "example.peerbench.ai";
+        kind: "llm/echo-basic.tc";
+        schemaVersion: 1;
+        metadata?: Record<string, unknown> | undefined;
+    };
+    provider: AbstractLLMProvider;
+    scorer?: undefined;
+    runConfig: {
+        model: string;
+    };
+    idGenerators?: {
+        response?: import("../../../index.js").IdGenerator;
+        score?: import("../../../index.js").IdGenerator;
+    };
+}) => Promise<{
+    response: {
+        startedAt: number;
+        completedAt: number;
+        id: string;
+        testCaseId: string;
+        data: string;
+        modelSlug: string;
+        provider: string;
+        namespace: "example.peerbench.ai";
+        kind: "llm/echo-basic.rs";
+        schemaVersion: 1;
+        metadata?: Record<string, unknown> | undefined;
+        systemPromptId?: string | undefined;
+        inputTokensUsed?: number | undefined;
+        outputTokensUsed?: number | undefined;
+        inputCost?: string | undefined;
+        outputCost?: string | undefined;
+    };
+    score?: {
+        id: string;
+        value: number;
+        responseId: string;
+        scoringMethod: "ai" | "human" | "algo";
+        namespace: "example.peerbench.ai";
+        kind: "llm/echo-basic.sc";
+        schemaVersion: 1;
+        metadata?: Record<string, unknown> | undefined;
+        explanation?: string | undefined;
+    } | undefined;
+}>) & {
+    config: {
+        runConfigSchema: z.ZodObject<{
+            model: z.ZodString;
+        }, z.core.$strip>;
+        schemaSets: [{
+            readonly testCase: z.ZodObject<Omit<{
+                id: z.ZodString;
+                namespace: z.ZodString;
+                schemaVersion: z.ZodNumber;
+                kind: z.ZodString;
+                metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+            }, "kind" | "namespace" | "schemaVersion"> & {
+                input: z.ZodString;
+            } & {
+                namespace: z.ZodLiteral<"example.peerbench.ai">;
+                kind: z.ZodLiteral<"llm/echo-basic.tc">;
+                schemaVersion: z.ZodLiteral<1>;
+            }, z.core.$strip> & {
+                new: (input: Omit<{
+                    id: string;
+                    input: string;
+                    namespace: "example.peerbench.ai";
+                    kind: "llm/echo-basic.tc";
+                    schemaVersion: 1;
+                    metadata?: Record<string, unknown> | undefined;
+                }, "kind" | "namespace" | "schemaVersion">) => {
+                    id: string;
+                    input: string;
+                    namespace: "example.peerbench.ai";
+                    kind: "llm/echo-basic.tc";
+                    schemaVersion: 1;
+                    metadata?: Record<string, unknown> | undefined;
+                };
+                newWithId(input: Omit<{
+                    id: string;
+                    input: string;
+                    namespace: "example.peerbench.ai";
+                    kind: "llm/echo-basic.tc";
+                    schemaVersion: 1;
+                    metadata?: Record<string, unknown> | undefined;
+                }, "kind" | "id" | "namespace" | "schemaVersion">, generator: import("../../../index.js").IdGenerator): Promise<{
+                    id: string;
+                    input: string;
+                    namespace: "example.peerbench.ai";
+                    kind: "llm/echo-basic.tc";
+                    schemaVersion: 1;
+                    metadata?: Record<string, unknown> | undefined;
+                }>;
+            };
+            readonly response: z.ZodObject<Omit<{
+                id: z.ZodString;
+                namespace: z.ZodString;
+                schemaVersion: z.ZodNumber;
+                kind: z.ZodString;
+                startedAt: z.ZodNumber;
+                completedAt: z.ZodNumber;
+                testCaseId: z.ZodString;
+                metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+            }, "kind" | "namespace" | "schemaVersion"> & {
+                data: z.ZodString;
+                modelSlug: z.ZodString;
+                provider: z.ZodString;
+                systemPromptId: z.ZodOptional<z.ZodString>;
+                inputTokensUsed: z.ZodOptional<z.ZodNumber>;
+                outputTokensUsed: z.ZodOptional<z.ZodNumber>;
+                inputCost: z.ZodOptional<z.ZodString>;
+                outputCost: z.ZodOptional<z.ZodString>;
+            } & {
+                namespace: z.ZodLiteral<"example.peerbench.ai">;
+                kind: z.ZodLiteral<"llm/echo-basic.rs">;
+                schemaVersion: z.ZodLiteral<1>;
+            }, z.core.$strip> & {
+                new: (input: Omit<{
+                    startedAt: number;
+                    completedAt: number;
+                    id: string;
+                    testCaseId: string;
+                    data: string;
+                    modelSlug: string;
+                    provider: string;
+                    namespace: "example.peerbench.ai";
+                    kind: "llm/echo-basic.rs";
+                    schemaVersion: 1;
+                    metadata?: Record<string, unknown> | undefined;
+                    systemPromptId?: string | undefined;
+                    inputTokensUsed?: number | undefined;
+                    outputTokensUsed?: number | undefined;
+                    inputCost?: string | undefined;
+                    outputCost?: string | undefined;
+                }, "kind" | "namespace" | "schemaVersion">) => {
+                    startedAt: number;
+                    completedAt: number;
+                    id: string;
+                    testCaseId: string;
+                    data: string;
+                    modelSlug: string;
+                    provider: string;
+                    namespace: "example.peerbench.ai";
+                    kind: "llm/echo-basic.rs";
+                    schemaVersion: 1;
+                    metadata?: Record<string, unknown> | undefined;
+                    systemPromptId?: string | undefined;
+                    inputTokensUsed?: number | undefined;
+                    outputTokensUsed?: number | undefined;
+                    inputCost?: string | undefined;
+                    outputCost?: string | undefined;
+                };
+                newWithId(input: Omit<{
+                    startedAt: number;
+                    completedAt: number;
+                    id: string;
+                    testCaseId: string;
+                    data: string;
+                    modelSlug: string;
+                    provider: string;
+                    namespace: "example.peerbench.ai";
+                    kind: "llm/echo-basic.rs";
+                    schemaVersion: 1;
+                    metadata?: Record<string, unknown> | undefined;
+                    systemPromptId?: string | undefined;
+                    inputTokensUsed?: number | undefined;
+                    outputTokensUsed?: number | undefined;
+                    inputCost?: string | undefined;
+                    outputCost?: string | undefined;
+                }, "kind" | "id" | "namespace" | "schemaVersion">, generator: import("../../../index.js").IdGenerator): Promise<{
+                    startedAt: number;
+                    completedAt: number;
+                    id: string;
+                    testCaseId: string;
+                    data: string;
+                    modelSlug: string;
+                    provider: string;
+                    namespace: "example.peerbench.ai";
+                    kind: "llm/echo-basic.rs";
+                    schemaVersion: 1;
+                    metadata?: Record<string, unknown> | undefined;
+                    systemPromptId?: string | undefined;
+                    inputTokensUsed?: number | undefined;
+                    outputTokensUsed?: number | undefined;
+                    inputCost?: string | undefined;
+                    outputCost?: string | undefined;
+                }>;
+            };
+            readonly score: z.ZodObject<Omit<{
+                id: z.ZodString;
+                namespace: z.ZodString;
+                kind: z.ZodString;
+                schemaVersion: z.ZodNumber;
+                value: z.ZodNumber;
+                responseId: z.ZodString;
+                explanation: z.ZodOptional<z.ZodString>;
+                metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+                scoringMethod: z.ZodEnum<{
+                    readonly ai: "ai";
+                    readonly human: "human";
+                    readonly algo: "algo";
+                }>;
+            }, "kind" | "namespace" | "schemaVersion"> & {
+                namespace: z.ZodLiteral<"example.peerbench.ai">;
+                kind: z.ZodLiteral<"llm/echo-basic.sc">;
+                schemaVersion: z.ZodLiteral<1>;
+            }, z.core.$strip> & {
+                new: (input: Omit<{
+                    id: string;
+                    value: number;
+                    responseId: string;
+                    scoringMethod: "ai" | "human" | "algo";
+                    namespace: "example.peerbench.ai";
+                    kind: "llm/echo-basic.sc";
+                    schemaVersion: 1;
+                    metadata?: Record<string, unknown> | undefined;
+                    explanation?: string | undefined;
+                }, "kind" | "namespace" | "schemaVersion">) => {
+                    id: string;
+                    value: number;
+                    responseId: string;
+                    scoringMethod: "ai" | "human" | "algo";
+                    namespace: "example.peerbench.ai";
+                    kind: "llm/echo-basic.sc";
+                    schemaVersion: 1;
+                    metadata?: Record<string, unknown> | undefined;
+                    explanation?: string | undefined;
+                };
+                newWithId(input: Omit<{
+                    id: string;
+                    value: number;
+                    responseId: string;
+                    scoringMethod: "ai" | "human" | "algo";
+                    namespace: "example.peerbench.ai";
+                    kind: "llm/echo-basic.sc";
+                    schemaVersion: 1;
+                    metadata?: Record<string, unknown> | undefined;
+                    explanation?: string | undefined;
+                }, "kind" | "id" | "namespace" | "schemaVersion">, generator: import("../../../index.js").IdGenerator): Promise<{
+                    id: string;
+                    value: number;
+                    responseId: string;
+                    scoringMethod: "ai" | "human" | "algo";
+                    namespace: "example.peerbench.ai";
+                    kind: "llm/echo-basic.sc";
+                    schemaVersion: 1;
+                    metadata?: Record<string, unknown> | undefined;
+                    explanation?: string | undefined;
+                }>;
+            };
+        }];
+        providers: [typeof AbstractLLMProvider];
+        scorers: [];
+        parseRunConfig?: boolean;
+        defaults?: {
+            scorer?: undefined;
+            responseIdGenerator?: import("../../../index.js").IdGenerator;
+            scoreIdGenerator?: import("../../../index.js").IdGenerator;
+        } | undefined;
+    };
+};

package/dist/benchmarks/examples/echo-basic/schema-sets/echo.v1.d.ts

@@ -0,0 +1,241 @@
+import { z } from "zod";
+/**
+ * Schemas are the core components of a benchmark. They are the entities which hold the data.
+ * There are three base different type of entities live inside a benchmark which are connected to each other:
+ * - Test case: Entity that holds a single input/task. That task often paired with an expected output (e.g multiple choice questions, Q&A)
+ * - Response: Entity that holds an output for a test case in a structured format.
+ * - Score: Entity that holds scoring result for a response.
+ *
+ * The hierarchy starts from test case, then response, then score. Every test case definition
+ * must have a corresponding response and every response must have a corresponding score definition.
+ * With this way we can easily identify each object in their own context.
+ *
+ * All of those entities are versioned via the `schemaVersion` field. Versions are recommended to
+ * start at 1 and increment whenever the schema changes in a breaking way. Handling version changes
+ * are the runtime's responsibility.
+ *
+ * `kind` field is a unique string that identifies the type of the entity. Depends on the
+ * entity type, kind field is extended as "*.tc" (test case), "*.rs" (response) or "*.sc" (score). Defining this field
+ * allow you to have strong type safety and identity for the raw objects that are stored outside your codebase.
+ *
+ * `namespace` field is another string identifier that tells you where this object definition coming from.
+ * It is recommended to use domain like syntax for it (such as "peerbench.ai" or "example.com") to avoid
+ * collisions with other namespaces out there.
+ *
+ * Each entity also has a field that refers to the upper level entity in the hierarchy via its ID.
+ * For instance: Scores have a `responseId` field and responses have a `testCaseId` field.
+ * This way we can easily identify which entity is generated for which entity.
+ *
+ * Managing those relations are the runtime's responsibility.
+ *
+ * Each entity must be following its base definition defined by the SDK.
+ * As long as the given `baseSchema` is an extension of the SDK level base schema (e.g `BaseTestCaseSchemaV1`),
+ * you can pass a different base schema. You can extend the well-known fields for your schemas using the extensions
+ * provided by the SDK. For instance below you can find that we are using `ExtensionLLMResponseFieldsV1` to extend the
+ * response schema with LLM specific fields (because our benchmark focuses on LLM chat models).
+ */
+export declare const EchoBasicNamespace: "example.peerbench.ai";
+export declare const EchoBasicKind: "llm/echo-basic";
+export declare const EchoBasicTestCaseSchemaV1: z.ZodObject<Omit<{
+    id: z.ZodString;
+    namespace: z.ZodString;
+    schemaVersion: z.ZodNumber;
+    kind: z.ZodString;
+    metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+}, "kind" | "namespace" | "schemaVersion"> & {
+    input: z.ZodString;
+} & {
+    namespace: z.ZodLiteral<"example.peerbench.ai">;
+    kind: z.ZodLiteral<"llm/echo-basic.tc">;
+    schemaVersion: z.ZodLiteral<1>;
+}, z.core.$strip> & {
+    new: (input: Omit<{
+        id: string;
+        input: string;
+        namespace: "example.peerbench.ai";
+        kind: "llm/echo-basic.tc";
+        schemaVersion: 1;
+        metadata?: Record<string, unknown> | undefined;
+    }, "kind" | "namespace" | "schemaVersion">) => {
+        id: string;
+        input: string;
+        namespace: "example.peerbench.ai";
+        kind: "llm/echo-basic.tc";
+        schemaVersion: 1;
+        metadata?: Record<string, unknown> | undefined;
+    };
+    newWithId(input: Omit<{
+        id: string;
+        input: string;
+        namespace: "example.peerbench.ai";
+        kind: "llm/echo-basic.tc";
+        schemaVersion: 1;
+        metadata?: Record<string, unknown> | undefined;
+    }, "kind" | "id" | "namespace" | "schemaVersion">, generator: import("../../../../index.js").IdGenerator): Promise<{
+        id: string;
+        input: string;
+        namespace: "example.peerbench.ai";
+        kind: "llm/echo-basic.tc";
+        schemaVersion: 1;
+        metadata?: Record<string, unknown> | undefined;
+    }>;
+};
+export type EchoBasicTestCaseV1 = z.infer<typeof EchoBasicTestCaseSchemaV1>;
+export declare const EchoBasicResponseSchemaV1: z.ZodObject<Omit<{
+    id: z.ZodString;
+    namespace: z.ZodString;
+    schemaVersion: z.ZodNumber;
+    kind: z.ZodString;
+    startedAt: z.ZodNumber;
+    completedAt: z.ZodNumber;
+    testCaseId: z.ZodString;
+    metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+}, "kind" | "namespace" | "schemaVersion"> & {
+    data: z.ZodString;
+    modelSlug: z.ZodString;
+    provider: z.ZodString;
+    systemPromptId: z.ZodOptional<z.ZodString>;
+    inputTokensUsed: z.ZodOptional<z.ZodNumber>;
+    outputTokensUsed: z.ZodOptional<z.ZodNumber>;
+    inputCost: z.ZodOptional<z.ZodString>;
+    outputCost: z.ZodOptional<z.ZodString>;
+} & {
+    namespace: z.ZodLiteral<"example.peerbench.ai">;
+    kind: z.ZodLiteral<"llm/echo-basic.rs">;
+    schemaVersion: z.ZodLiteral<1>;
+}, z.core.$strip> & {
+    new: (input: Omit<{
+        startedAt: number;
+        completedAt: number;
+        id: string;
+        testCaseId: string;
+        data: string;
+        modelSlug: string;
+        provider: string;
+        namespace: "example.peerbench.ai";
+        kind: "llm/echo-basic.rs";
+        schemaVersion: 1;
+        metadata?: Record<string, unknown> | undefined;
+        systemPromptId?: string | undefined;
+        inputTokensUsed?: number | undefined;
+        outputTokensUsed?: number | undefined;
+        inputCost?: string | undefined;
+        outputCost?: string | undefined;
+    }, "kind" | "namespace" | "schemaVersion">) => {
+        startedAt: number;
+        completedAt: number;
+        id: string;
+        testCaseId: string;
+        data: string;
+        modelSlug: string;
+        provider: string;
+        namespace: "example.peerbench.ai";
+        kind: "llm/echo-basic.rs";
+        schemaVersion: 1;
+        metadata?: Record<string, unknown> | undefined;
+        systemPromptId?: string | undefined;
+        inputTokensUsed?: number | undefined;
+        outputTokensUsed?: number | undefined;
+        inputCost?: string | undefined;
+        outputCost?: string | undefined;
+    };
+    newWithId(input: Omit<{
+        startedAt: number;
+        completedAt: number;
+        id: string;
+        testCaseId: string;
+        data: string;
+        modelSlug: string;
+        provider: string;
+        namespace: "example.peerbench.ai";
+        kind: "llm/echo-basic.rs";
+        schemaVersion: 1;
+        metadata?: Record<string, unknown> | undefined;
+        systemPromptId?: string | undefined;
+        inputTokensUsed?: number | undefined;
+        outputTokensUsed?: number | undefined;
+        inputCost?: string | undefined;
+        outputCost?: string | undefined;
+    }, "kind" | "id" | "namespace" | "schemaVersion">, generator: import("../../../../index.js").IdGenerator): Promise<{
+        startedAt: number;
+        completedAt: number;
+        id: string;
+        testCaseId: string;
+        data: string;
+        modelSlug: string;
+        provider: string;
+        namespace: "example.peerbench.ai";
+        kind: "llm/echo-basic.rs";
+        schemaVersion: 1;
+        metadata?: Record<string, unknown> | undefined;
+        systemPromptId?: string | undefined;
+        inputTokensUsed?: number | undefined;
+        outputTokensUsed?: number | undefined;
+        inputCost?: string | undefined;
+        outputCost?: string | undefined;
+    }>;
+};
+export type EchoBasicResponseV1 = z.infer<typeof EchoBasicResponseSchemaV1>;
+export declare const EchoBasicScoreSchemaV1: z.ZodObject<Omit<{
+    id: z.ZodString;
+    namespace: z.ZodString;
+    kind: z.ZodString;
+    schemaVersion: z.ZodNumber;
+    value: z.ZodNumber;
+    responseId: z.ZodString;
+    explanation: z.ZodOptional<z.ZodString>;
+    metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+    scoringMethod: z.ZodEnum<{
+        readonly ai: "ai";
+        readonly human: "human";
+        readonly algo: "algo";
+    }>;
+}, "kind" | "namespace" | "schemaVersion"> & {
+    namespace: z.ZodLiteral<"example.peerbench.ai">;
+    kind: z.ZodLiteral<"llm/echo-basic.sc">;
+    schemaVersion: z.ZodLiteral<1>;
+}, z.core.$strip> & {
+    new: (input: Omit<{
+        id: string;
+        value: number;
+        responseId: string;
+        scoringMethod: "ai" | "human" | "algo";
+        namespace: "example.peerbench.ai";
+        kind: "llm/echo-basic.sc";
+        schemaVersion: 1;
+        metadata?: Record<string, unknown> | undefined;
+        explanation?: string | undefined;
+    }, "kind" | "namespace" | "schemaVersion">) => {
+        id: string;
+        value: number;
+        responseId: string;
+        scoringMethod: "ai" | "human" | "algo";
+        namespace: "example.peerbench.ai";
+        kind: "llm/echo-basic.sc";
+        schemaVersion: 1;
+        metadata?: Record<string, unknown> | undefined;
+        explanation?: string | undefined;
+    };
+    newWithId(input: Omit<{
+        id: string;
+        value: number;
+        responseId: string;
+        scoringMethod: "ai" | "human" | "algo";
+        namespace: "example.peerbench.ai";
+        kind: "llm/echo-basic.sc";
+        schemaVersion: 1;
+        metadata?: Record<string, unknown> | undefined;
+        explanation?: string | undefined;
+    }, "kind" | "id" | "namespace" | "schemaVersion">, generator: import("../../../../index.js").IdGenerator): Promise<{
+        id: string;
+        value: number;
+        responseId: string;
+        scoringMethod: "ai" | "human" | "algo";
+        namespace: "example.peerbench.ai";
+        kind: "llm/echo-basic.sc";
+        schemaVersion: 1;
+        metadata?: Record<string, unknown> | undefined;
+        explanation?: string | undefined;
+    }>;
+};
+export type EchoBasicScoreV1 = z.infer<typeof EchoBasicScoreSchemaV1>;

package/dist/benchmarks/examples/echo-basic/storages/json.d.ts

@@ -0,0 +1,14 @@
+import { JSONFileStorage } from "../../../../storages/json-file";
+import { EchoBasicResponseV1, EchoBasicScoreV1, EchoBasicTestCaseV1 } from "../schema-sets/echo.v1";
+/**
+ * Storages are the abstractions that are responsible for persisting benchmark entities outside your codebase.
+ *
+ * This example uses one of the predefined Storage implementations from the SDK which is JSONFileStorage.
+ * This storage allows you to store benchmark entity(ies) in a regular JSON file as an array of objects.
+ */
+export declare class EchoBasicJSONStorage extends JSONFileStorage<EchoBasicTestCaseV1 | EchoBasicResponseV1 | EchoBasicScoreV1> {
+    constructor(config: {
+        path: string;
+        chunkSize?: number;
+    });
+}

package/dist/benchmarks/examples/echo-basic/storages/text.d.ts

@@ -0,0 +1,24 @@
+import { FileStorage } from "../../../../storages/file";
+import { EchoBasicResponseV1, EchoBasicScoreV1, EchoBasicTestCaseV1 } from "../schema-sets/echo.v1";
+/**
+ * Example custom file storage implementation that stores the benchmark entities in a custom formatted text file.
+ *
+ * The custom file format expected is:
+ *
+ * ```
+ * BEGIN PEERBENCH ENTITY
+ * id: <id>
+ * namespace: <namespace>
+ * kind: <kind>
+ * schemaVersion: <schemaVersion>
+ * json: <json>
+ * END PEERBENCH ENTITY
+ * ```
+ */
+export declare class EchoBasicTextStorage extends FileStorage<EchoBasicEntityV1> {
+    constructor(config: {
+        path: string;
+    });
+}
+type EchoBasicEntityV1 = EchoBasicTestCaseV1 | EchoBasicResponseV1 | EchoBasicScoreV1;
+export {};

package/dist/benchmarks/examples/exact-match-scorer/index.d.ts

@@ -0,0 +1,4 @@
+export * from "./runner";
+export * from "./scorer";
+export * from "./schema-sets/exact-match.v1";
+export * from "./storages/json";