peerbench 0.0.7 → 0.0.8

package/dist/benchmarks/examples/exact-match-scorer/schema-sets/exact-match.v1.d.ts

@@ -0,0 +1,287 @@
+import { z } from "zod";
+export declare const ExactMatchNamespace: "example.peerbench.ai";
+export declare const ExactMatchKind: "llm/exact-match";
+export declare const ExactMatchTestCaseSchemaV1: z.ZodObject<Omit<{
+    id: z.ZodString;
+    namespace: z.ZodString;
+    schemaVersion: z.ZodNumber;
+    kind: z.ZodString;
+    metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+}, "kind" | "namespace" | "schemaVersion"> & {
+    instruction: z.ZodString;
+    input: z.ZodString;
+    expectedOutput: z.ZodString;
+    normalize: z.ZodOptional<z.ZodBoolean>;
+} & {
+    namespace: z.ZodLiteral<"example.peerbench.ai">;
+    kind: z.ZodLiteral<"llm/exact-match.tc">;
+    schemaVersion: z.ZodLiteral<1>;
+}, z.core.$strip> & {
+    new: (input: Omit<{
+        id: string;
+        instruction: string;
+        input: string;
+        expectedOutput: string;
+        namespace: "example.peerbench.ai";
+        kind: "llm/exact-match.tc";
+        schemaVersion: 1;
+        metadata?: Record<string, unknown> | undefined;
+        normalize?: boolean | undefined;
+    }, "kind" | "namespace" | "schemaVersion">) => {
+        id: string;
+        instruction: string;
+        input: string;
+        expectedOutput: string;
+        namespace: "example.peerbench.ai";
+        kind: "llm/exact-match.tc";
+        schemaVersion: 1;
+        metadata?: Record<string, unknown> | undefined;
+        normalize?: boolean | undefined;
+    };
+    newWithId(input: Omit<{
+        id: string;
+        instruction: string;
+        input: string;
+        expectedOutput: string;
+        namespace: "example.peerbench.ai";
+        kind: "llm/exact-match.tc";
+        schemaVersion: 1;
+        metadata?: Record<string, unknown> | undefined;
+        normalize?: boolean | undefined;
+    }, "kind" | "id" | "namespace" | "schemaVersion">, generator: import("../../../../index.js").IdGenerator): Promise<{
+        id: string;
+        instruction: string;
+        input: string;
+        expectedOutput: string;
+        namespace: "example.peerbench.ai";
+        kind: "llm/exact-match.tc";
+        schemaVersion: 1;
+        metadata?: Record<string, unknown> | undefined;
+        normalize?: boolean | undefined;
+    }>;
+};
+export type ExactMatchTestCaseV1 = z.infer<typeof ExactMatchTestCaseSchemaV1>;
+export declare const ExactMatchResponseSchemaV1: z.ZodObject<Omit<{
+    id: z.ZodString;
+    namespace: z.ZodString;
+    schemaVersion: z.ZodNumber;
+    kind: z.ZodString;
+    startedAt: z.ZodNumber;
+    completedAt: z.ZodNumber;
+    testCaseId: z.ZodString;
+    metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+}, "kind" | "namespace" | "schemaVersion"> & {
+    data: z.ZodString;
+    modelSlug: z.ZodString;
+    provider: z.ZodString;
+    systemPromptId: z.ZodOptional<z.ZodString>;
+    inputTokensUsed: z.ZodOptional<z.ZodNumber>;
+    outputTokensUsed: z.ZodOptional<z.ZodNumber>;
+    inputCost: z.ZodOptional<z.ZodString>;
+    outputCost: z.ZodOptional<z.ZodString>;
+} & {
+    namespace: z.ZodLiteral<"example.peerbench.ai">;
+    kind: z.ZodLiteral<"llm/exact-match.rs">;
+    schemaVersion: z.ZodLiteral<1>;
+}, z.core.$strip> & {
+    new: (input: Omit<{
+        startedAt: number;
+        completedAt: number;
+        id: string;
+        testCaseId: string;
+        data: string;
+        modelSlug: string;
+        provider: string;
+        namespace: "example.peerbench.ai";
+        kind: "llm/exact-match.rs";
+        schemaVersion: 1;
+        metadata?: Record<string, unknown> | undefined;
+        systemPromptId?: string | undefined;
+        inputTokensUsed?: number | undefined;
+        outputTokensUsed?: number | undefined;
+        inputCost?: string | undefined;
+        outputCost?: string | undefined;
+    }, "kind" | "namespace" | "schemaVersion">) => {
+        startedAt: number;
+        completedAt: number;
+        id: string;
+        testCaseId: string;
+        data: string;
+        modelSlug: string;
+        provider: string;
+        namespace: "example.peerbench.ai";
+        kind: "llm/exact-match.rs";
+        schemaVersion: 1;
+        metadata?: Record<string, unknown> | undefined;
+        systemPromptId?: string | undefined;
+        inputTokensUsed?: number | undefined;
+        outputTokensUsed?: number | undefined;
+        inputCost?: string | undefined;
+        outputCost?: string | undefined;
+    };
+    newWithId(input: Omit<{
+        startedAt: number;
+        completedAt: number;
+        id: string;
+        testCaseId: string;
+        data: string;
+        modelSlug: string;
+        provider: string;
+        namespace: "example.peerbench.ai";
+        kind: "llm/exact-match.rs";
+        schemaVersion: 1;
+        metadata?: Record<string, unknown> | undefined;
+        systemPromptId?: string | undefined;
+        inputTokensUsed?: number | undefined;
+        outputTokensUsed?: number | undefined;
+        inputCost?: string | undefined;
+        outputCost?: string | undefined;
+    }, "kind" | "id" | "namespace" | "schemaVersion">, generator: import("../../../../index.js").IdGenerator): Promise<{
+        startedAt: number;
+        completedAt: number;
+        id: string;
+        testCaseId: string;
+        data: string;
+        modelSlug: string;
+        provider: string;
+        namespace: "example.peerbench.ai";
+        kind: "llm/exact-match.rs";
+        schemaVersion: 1;
+        metadata?: Record<string, unknown> | undefined;
+        systemPromptId?: string | undefined;
+        inputTokensUsed?: number | undefined;
+        outputTokensUsed?: number | undefined;
+        inputCost?: string | undefined;
+        outputCost?: string | undefined;
+    }>;
+};
+export type ExactMatchResponseV1 = z.infer<typeof ExactMatchResponseSchemaV1>;
+export declare const ExactMatchScoreSchemaV1: z.ZodObject<Omit<{
+    id: z.ZodString;
+    namespace: z.ZodString;
+    kind: z.ZodString;
+    schemaVersion: z.ZodNumber;
+    value: z.ZodNumber;
+    responseId: z.ZodString;
+    explanation: z.ZodOptional<z.ZodString>;
+    metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+    scoringMethod: z.ZodEnum<{
+        readonly ai: "ai";
+        readonly human: "human";
+        readonly algo: "algo";
+    }>;
+}, "kind" | "namespace" | "schemaVersion"> & {
+    match: z.ZodBoolean;
+    normalized: z.ZodOptional<z.ZodObject<{
+        expected: z.ZodString;
+        actual: z.ZodString;
+    }, z.core.$strip>>;
+    scorerAISystemPrompt: z.ZodOptional<z.ZodString>;
+    scorerAISystemPromptId: z.ZodOptional<z.ZodString>;
+    scorerAIProvider: z.ZodOptional<z.ZodString>;
+    scorerAIModelSlug: z.ZodOptional<z.ZodString>;
+    scorerAIInputTokensUsed: z.ZodOptional<z.ZodNumber>;
+    scorerAIOutputTokensUsed: z.ZodOptional<z.ZodNumber>;
+    scorerAIInputCost: z.ZodOptional<z.ZodString>;
+    scorerAIOutputCost: z.ZodOptional<z.ZodString>;
+} & {
+    namespace: z.ZodLiteral<"example.peerbench.ai">;
+    kind: z.ZodLiteral<"llm/exact-match.sc">;
+    schemaVersion: z.ZodLiteral<1>;
+}, z.core.$strip> & {
+    new: (input: Omit<{
+        id: string;
+        value: number;
+        responseId: string;
+        scoringMethod: "ai" | "human" | "algo";
+        match: boolean;
+        namespace: "example.peerbench.ai";
+        kind: "llm/exact-match.sc";
+        schemaVersion: 1;
+        metadata?: Record<string, unknown> | undefined;
+        explanation?: string | undefined;
+        normalized?: {
+            expected: string;
+            actual: string;
+        } | undefined;
+        scorerAISystemPrompt?: string | undefined;
+        scorerAISystemPromptId?: string | undefined;
+        scorerAIProvider?: string | undefined;
+        scorerAIModelSlug?: string | undefined;
+        scorerAIInputTokensUsed?: number | undefined;
+        scorerAIOutputTokensUsed?: number | undefined;
+        scorerAIInputCost?: string | undefined;
+        scorerAIOutputCost?: string | undefined;
+    }, "kind" | "namespace" | "schemaVersion">) => {
+        id: string;
+        value: number;
+        responseId: string;
+        scoringMethod: "ai" | "human" | "algo";
+        match: boolean;
+        namespace: "example.peerbench.ai";
+        kind: "llm/exact-match.sc";
+        schemaVersion: 1;
+        metadata?: Record<string, unknown> | undefined;
+        explanation?: string | undefined;
+        normalized?: {
+            expected: string;
+            actual: string;
+        } | undefined;
+        scorerAISystemPrompt?: string | undefined;
+        scorerAISystemPromptId?: string | undefined;
+        scorerAIProvider?: string | undefined;
+        scorerAIModelSlug?: string | undefined;
+        scorerAIInputTokensUsed?: number | undefined;
+        scorerAIOutputTokensUsed?: number | undefined;
+        scorerAIInputCost?: string | undefined;
+        scorerAIOutputCost?: string | undefined;
+    };
+    newWithId(input: Omit<{
+        id: string;
+        value: number;
+        responseId: string;
+        scoringMethod: "ai" | "human" | "algo";
+        match: boolean;
+        namespace: "example.peerbench.ai";
+        kind: "llm/exact-match.sc";
+        schemaVersion: 1;
+        metadata?: Record<string, unknown> | undefined;
+        explanation?: string | undefined;
+        normalized?: {
+            expected: string;
+            actual: string;
+        } | undefined;
+        scorerAISystemPrompt?: string | undefined;
+        scorerAISystemPromptId?: string | undefined;
+        scorerAIProvider?: string | undefined;
+        scorerAIModelSlug?: string | undefined;
+        scorerAIInputTokensUsed?: number | undefined;
+        scorerAIOutputTokensUsed?: number | undefined;
+        scorerAIInputCost?: string | undefined;
+        scorerAIOutputCost?: string | undefined;
+    }, "kind" | "id" | "namespace" | "schemaVersion">, generator: import("../../../../index.js").IdGenerator): Promise<{
+        id: string;
+        value: number;
+        responseId: string;
+        scoringMethod: "ai" | "human" | "algo";
+        match: boolean;
+        namespace: "example.peerbench.ai";
+        kind: "llm/exact-match.sc";
+        schemaVersion: 1;
+        metadata?: Record<string, unknown> | undefined;
+        explanation?: string | undefined;
+        normalized?: {
+            expected: string;
+            actual: string;
+        } | undefined;
+        scorerAISystemPrompt?: string | undefined;
+        scorerAISystemPromptId?: string | undefined;
+        scorerAIProvider?: string | undefined;
+        scorerAIModelSlug?: string | undefined;
+        scorerAIInputTokensUsed?: number | undefined;
+        scorerAIOutputTokensUsed?: number | undefined;
+        scorerAIInputCost?: string | undefined;
+        scorerAIOutputCost?: string | undefined;
+    }>;
+};
+export type ExactMatchScoreV1 = z.infer<typeof ExactMatchScoreSchemaV1>;

package/dist/benchmarks/examples/exact-match-scorer/scorer.d.ts

@@ -0,0 +1,30 @@
+import { AbstractScorer, BaseScorerResult } from "../../../scorers/abstract";
+/**
+ * A "scorer" is the piece that turns the given data into a numeric score
+ * alongside with additional explanation and metadata.
+ *
+ * A runner can do scoring inline, but having a dedicated scorer is nice when:
+ * - you want to reuse the same scoring logic across multiple runners/benchmarks
+ * - you want to allow callers to swap scorers easily (e.g. deterministic scorer vs LLM-as-a-judge)
+ * - you want to test scoring in isolation without calling a provider
+ *
+ * Here is an example, simple scorer implementation. It checks exact match (optionally with normalization)
+ * of the given expected and actual values. Score values must be between 0 and 1.
+ */
+export declare class ExactMatchScorer extends AbstractScorer {
+    readonly kind: "example.peerbench.ai/exact-match";
+    score(params: {
+        expected: string;
+        actual: string;
+        normalize?: boolean;
+    }): Promise<BaseScorerResult & {
+        metadata: {
+            match: boolean;
+            normalize: boolean;
+            normalized?: {
+                expected: string;
+                actual: string;
+            };
+        };
+    }>;
+}

package/dist/benchmarks/examples/exact-match-scorer/storages/json.d.ts

@@ -0,0 +1,8 @@
+import { JSONFileStorage } from "../../../../storages/json-file";
+import { ExactMatchResponseV1, ExactMatchScoreV1, ExactMatchTestCaseV1 } from "../schema-sets/exact-match.v1";
+export declare class ExactMatchJSONStorage extends JSONFileStorage<ExactMatchTestCaseV1 | ExactMatchResponseV1 | ExactMatchScoreV1> {
+    constructor(config: {
+        path: string;
+        chunkSize?: number;
+    });
+}

package/dist/benchmarks/examples/text-transform/index.d.ts

@@ -0,0 +1,4 @@
+export * from "./runner";
+export * from "./schema-sets/echo.v1";
+export * from "./schema-sets/reverse.v1";
+export * from "./storages/json";