peerbench 0.0.6 → 0.0.8

package/dist/benchmarks/examples/exact-match-scorer/runner.d.ts

@@ -0,0 +1,428 @@
+import { AbstractLLMProvider } from "../../../providers/index.js";
+import { LLMAsAJudgeScorer } from "../../../scorers/index.js";
+import z from "zod";
+import { ExactMatchScorer } from "./scorer";
+export declare const exactMatchScorerRunner: ((params: {
+    testCase: {
+        id: string;
+        instruction: string;
+        input: string;
+        expectedOutput: string;
+        namespace: "example.peerbench.ai";
+        kind: "llm/exact-match.tc";
+        schemaVersion: 1;
+        metadata?: Record<string, unknown> | undefined;
+        normalize?: boolean | undefined;
+    };
+    provider: AbstractLLMProvider;
+    scorer?: LLMAsAJudgeScorer | ExactMatchScorer | undefined;
+    runConfig: {
+        model: string;
+        temperature?: number | undefined;
+        systemPrompt?: {
+            id: string;
+            version: number;
+            content: string;
+            namespace: "peerbench.ai";
+            kind: `${string}/simple.sys-prompt`;
+            schemaVersion: 1;
+            metadata?: Record<string, unknown> | undefined;
+        } | undefined;
+        llmJudgeModel?: string | undefined;
+    };
+    idGenerators?: {
+        response?: import("../../../types").IdGenerator;
+        score?: import("../../../types").IdGenerator;
+    };
+}) => Promise<{
+    response: {
+        startedAt: number;
+        completedAt: number;
+        id: string;
+        testCaseId: string;
+        data: string;
+        modelSlug: string;
+        provider: string;
+        namespace: "example.peerbench.ai";
+        kind: "llm/exact-match.rs";
+        schemaVersion: 1;
+        metadata?: Record<string, unknown> | undefined;
+        systemPromptId?: string | undefined;
+        inputTokensUsed?: number | undefined;
+        outputTokensUsed?: number | undefined;
+        inputCost?: string | undefined;
+        outputCost?: string | undefined;
+    };
+    score?: {
+        id: string;
+        value: number;
+        responseId: string;
+        scoringMethod: "ai" | "human" | "algo";
+        match: boolean;
+        namespace: "example.peerbench.ai";
+        kind: "llm/exact-match.sc";
+        schemaVersion: 1;
+        metadata?: Record<string, unknown> | undefined;
+        explanation?: string | undefined;
+        normalized?: {
+            expected: string;
+            actual: string;
+        } | undefined;
+        scorerAISystemPrompt?: string | undefined;
+        scorerAISystemPromptId?: string | undefined;
+        scorerAIProvider?: string | undefined;
+        scorerAIModelSlug?: string | undefined;
+        scorerAIInputTokensUsed?: number | undefined;
+        scorerAIOutputTokensUsed?: number | undefined;
+        scorerAIInputCost?: string | undefined;
+        scorerAIOutputCost?: string | undefined;
+    } | undefined;
+}>) & {
+    config: {
+        runConfigSchema: z.ZodObject<{
+            model: z.ZodString;
+            temperature: z.ZodOptional<z.ZodNumber>;
+            systemPrompt: z.ZodOptional<z.ZodObject<Omit<{
+                id: z.ZodString;
+                namespace: z.ZodString;
+                kind: z.ZodString;
+                schemaVersion: z.ZodNumber;
+                version: z.ZodNumber;
+                metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+            }, "kind" | "namespace" | "schemaVersion"> & {
+                content: z.ZodString;
+            } & {
+                namespace: z.ZodLiteral<"peerbench.ai">;
+                kind: z.ZodLiteral<`${string}/simple.sys-prompt`>;
+                schemaVersion: z.ZodLiteral<1>;
+            }, z.core.$strip> & {
+                new: (input: Omit<{
+                    id: string;
+                    version: number;
+                    content: string;
+                    namespace: "peerbench.ai";
+                    kind: `${string}/simple.sys-prompt`;
+                    schemaVersion: 1;
+                    metadata?: Record<string, unknown> | undefined;
+                }, "kind" | "namespace" | "schemaVersion">) => {
+                    id: string;
+                    version: number;
+                    content: string;
+                    namespace: "peerbench.ai";
+                    kind: `${string}/simple.sys-prompt`;
+                    schemaVersion: 1;
+                    metadata?: Record<string, unknown> | undefined;
+                };
+                newWithId(input: Omit<{
+                    id: string;
+                    version: number;
+                    content: string;
+                    namespace: "peerbench.ai";
+                    kind: `${string}/simple.sys-prompt`;
+                    schemaVersion: 1;
+                    metadata?: Record<string, unknown> | undefined;
+                }, "kind" | "id" | "namespace" | "schemaVersion">, generator: import("../../../types").IdGenerator): Promise<{
+                    id: string;
+                    version: number;
+                    content: string;
+                    namespace: "peerbench.ai";
+                    kind: `${string}/simple.sys-prompt`;
+                    schemaVersion: 1;
+                    metadata?: Record<string, unknown> | undefined;
+                }>;
+            }>;
+            llmJudgeModel: z.ZodOptional<z.ZodString>;
+        }, z.core.$strip>;
+        schemaSets: [{
+            readonly testCase: z.ZodObject<Omit<{
+                id: z.ZodString;
+                namespace: z.ZodString;
+                schemaVersion: z.ZodNumber;
+                kind: z.ZodString;
+                metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+            }, "kind" | "namespace" | "schemaVersion"> & {
+                instruction: z.ZodString;
+                input: z.ZodString;
+                expectedOutput: z.ZodString;
+                normalize: z.ZodOptional<z.ZodBoolean>;
+            } & {
+                namespace: z.ZodLiteral<"example.peerbench.ai">;
+                kind: z.ZodLiteral<"llm/exact-match.tc">;
+                schemaVersion: z.ZodLiteral<1>;
+            }, z.core.$strip> & {
+                new: (input: Omit<{
+                    id: string;
+                    instruction: string;
+                    input: string;
+                    expectedOutput: string;
+                    namespace: "example.peerbench.ai";
+                    kind: "llm/exact-match.tc";
+                    schemaVersion: 1;
+                    metadata?: Record<string, unknown> | undefined;
+                    normalize?: boolean | undefined;
+                }, "kind" | "namespace" | "schemaVersion">) => {
+                    id: string;
+                    instruction: string;
+                    input: string;
+                    expectedOutput: string;
+                    namespace: "example.peerbench.ai";
+                    kind: "llm/exact-match.tc";
+                    schemaVersion: 1;
+                    metadata?: Record<string, unknown> | undefined;
+                    normalize?: boolean | undefined;
+                };
+                newWithId(input: Omit<{
+                    id: string;
+                    instruction: string;
+                    input: string;
+                    expectedOutput: string;
+                    namespace: "example.peerbench.ai";
+                    kind: "llm/exact-match.tc";
+                    schemaVersion: 1;
+                    metadata?: Record<string, unknown> | undefined;
+                    normalize?: boolean | undefined;
+                }, "kind" | "id" | "namespace" | "schemaVersion">, generator: import("../../../types").IdGenerator): Promise<{
+                    id: string;
+                    instruction: string;
+                    input: string;
+                    expectedOutput: string;
+                    namespace: "example.peerbench.ai";
+                    kind: "llm/exact-match.tc";
+                    schemaVersion: 1;
+                    metadata?: Record<string, unknown> | undefined;
+                    normalize?: boolean | undefined;
+                }>;
+            };
+            readonly response: z.ZodObject<Omit<{
+                id: z.ZodString;
+                namespace: z.ZodString;
+                schemaVersion: z.ZodNumber;
+                kind: z.ZodString;
+                startedAt: z.ZodNumber;
+                completedAt: z.ZodNumber;
+                testCaseId: z.ZodString;
+                metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+            }, "kind" | "namespace" | "schemaVersion"> & {
+                data: z.ZodString;
+                modelSlug: z.ZodString;
+                provider: z.ZodString;
+                systemPromptId: z.ZodOptional<z.ZodString>;
+                inputTokensUsed: z.ZodOptional<z.ZodNumber>;
+                outputTokensUsed: z.ZodOptional<z.ZodNumber>;
+                inputCost: z.ZodOptional<z.ZodString>;
+                outputCost: z.ZodOptional<z.ZodString>;
+            } & {
+                namespace: z.ZodLiteral<"example.peerbench.ai">;
+                kind: z.ZodLiteral<"llm/exact-match.rs">;
+                schemaVersion: z.ZodLiteral<1>;
+            }, z.core.$strip> & {
+                new: (input: Omit<{
+                    startedAt: number;
+                    completedAt: number;
+                    id: string;
+                    testCaseId: string;
+                    data: string;
+                    modelSlug: string;
+                    provider: string;
+                    namespace: "example.peerbench.ai";
+                    kind: "llm/exact-match.rs";
+                    schemaVersion: 1;
+                    metadata?: Record<string, unknown> | undefined;
+                    systemPromptId?: string | undefined;
+                    inputTokensUsed?: number | undefined;
+                    outputTokensUsed?: number | undefined;
+                    inputCost?: string | undefined;
+                    outputCost?: string | undefined;
+                }, "kind" | "namespace" | "schemaVersion">) => {
+                    startedAt: number;
+                    completedAt: number;
+                    id: string;
+                    testCaseId: string;
+                    data: string;
+                    modelSlug: string;
+                    provider: string;
+                    namespace: "example.peerbench.ai";
+                    kind: "llm/exact-match.rs";
+                    schemaVersion: 1;
+                    metadata?: Record<string, unknown> | undefined;
+                    systemPromptId?: string | undefined;
+                    inputTokensUsed?: number | undefined;
+                    outputTokensUsed?: number | undefined;
+                    inputCost?: string | undefined;
+                    outputCost?: string | undefined;
+                };
+                newWithId(input: Omit<{
+                    startedAt: number;
+                    completedAt: number;
+                    id: string;
+                    testCaseId: string;
+                    data: string;
+                    modelSlug: string;
+                    provider: string;
+                    namespace: "example.peerbench.ai";
+                    kind: "llm/exact-match.rs";
+                    schemaVersion: 1;
+                    metadata?: Record<string, unknown> | undefined;
+                    systemPromptId?: string | undefined;
+                    inputTokensUsed?: number | undefined;
+                    outputTokensUsed?: number | undefined;
+                    inputCost?: string | undefined;
+                    outputCost?: string | undefined;
+                }, "kind" | "id" | "namespace" | "schemaVersion">, generator: import("../../../types").IdGenerator): Promise<{
+                    startedAt: number;
+                    completedAt: number;
+                    id: string;
+                    testCaseId: string;
+                    data: string;
+                    modelSlug: string;
+                    provider: string;
+                    namespace: "example.peerbench.ai";
+                    kind: "llm/exact-match.rs";
+                    schemaVersion: 1;
+                    metadata?: Record<string, unknown> | undefined;
+                    systemPromptId?: string | undefined;
+                    inputTokensUsed?: number | undefined;
+                    outputTokensUsed?: number | undefined;
+                    inputCost?: string | undefined;
+                    outputCost?: string | undefined;
+                }>;
+            };
+            readonly score: z.ZodObject<Omit<{
+                id: z.ZodString;
+                namespace: z.ZodString;
+                kind: z.ZodString;
+                schemaVersion: z.ZodNumber;
+                value: z.ZodNumber;
+                responseId: z.ZodString;
+                explanation: z.ZodOptional<z.ZodString>;
+                metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+                scoringMethod: z.ZodEnum<{
+                    readonly ai: "ai";
+                    readonly human: "human";
+                    readonly algo: "algo";
+                }>;
+            }, "kind" | "namespace" | "schemaVersion"> & {
+                match: z.ZodBoolean;
+                normalized: z.ZodOptional<z.ZodObject<{
+                    expected: z.ZodString;
+                    actual: z.ZodString;
+                }, z.core.$strip>>;
+                scorerAISystemPrompt: z.ZodOptional<z.ZodString>;
+                scorerAISystemPromptId: z.ZodOptional<z.ZodString>;
+                scorerAIProvider: z.ZodOptional<z.ZodString>;
+                scorerAIModelSlug: z.ZodOptional<z.ZodString>;
+                scorerAIInputTokensUsed: z.ZodOptional<z.ZodNumber>;
+                scorerAIOutputTokensUsed: z.ZodOptional<z.ZodNumber>;
+                scorerAIInputCost: z.ZodOptional<z.ZodString>;
+                scorerAIOutputCost: z.ZodOptional<z.ZodString>;
+            } & {
+                namespace: z.ZodLiteral<"example.peerbench.ai">;
+                kind: z.ZodLiteral<"llm/exact-match.sc">;
+                schemaVersion: z.ZodLiteral<1>;
+            }, z.core.$strip> & {
+                new: (input: Omit<{
+                    id: string;
+                    value: number;
+                    responseId: string;
+                    scoringMethod: "ai" | "human" | "algo";
+                    match: boolean;
+                    namespace: "example.peerbench.ai";
+                    kind: "llm/exact-match.sc";
+                    schemaVersion: 1;
+                    metadata?: Record<string, unknown> | undefined;
+                    explanation?: string | undefined;
+                    normalized?: {
+                        expected: string;
+                        actual: string;
+                    } | undefined;
+                    scorerAISystemPrompt?: string | undefined;
+                    scorerAISystemPromptId?: string | undefined;
+                    scorerAIProvider?: string | undefined;
+                    scorerAIModelSlug?: string | undefined;
+                    scorerAIInputTokensUsed?: number | undefined;
+                    scorerAIOutputTokensUsed?: number | undefined;
+                    scorerAIInputCost?: string | undefined;
+                    scorerAIOutputCost?: string | undefined;
+                }, "kind" | "namespace" | "schemaVersion">) => {
+                    id: string;
+                    value: number;
+                    responseId: string;
+                    scoringMethod: "ai" | "human" | "algo";
+                    match: boolean;
+                    namespace: "example.peerbench.ai";
+                    kind: "llm/exact-match.sc";
+                    schemaVersion: 1;
+                    metadata?: Record<string, unknown> | undefined;
+                    explanation?: string | undefined;
+                    normalized?: {
+                        expected: string;
+                        actual: string;
+                    } | undefined;
+                    scorerAISystemPrompt?: string | undefined;
+                    scorerAISystemPromptId?: string | undefined;
+                    scorerAIProvider?: string | undefined;
+                    scorerAIModelSlug?: string | undefined;
+                    scorerAIInputTokensUsed?: number | undefined;
+                    scorerAIOutputTokensUsed?: number | undefined;
+                    scorerAIInputCost?: string | undefined;
+                    scorerAIOutputCost?: string | undefined;
+                };
+                newWithId(input: Omit<{
+                    id: string;
+                    value: number;
+                    responseId: string;
+                    scoringMethod: "ai" | "human" | "algo";
+                    match: boolean;
+                    namespace: "example.peerbench.ai";
+                    kind: "llm/exact-match.sc";
+                    schemaVersion: 1;
+                    metadata?: Record<string, unknown> | undefined;
+                    explanation?: string | undefined;
+                    normalized?: {
+                        expected: string;
+                        actual: string;
+                    } | undefined;
+                    scorerAISystemPrompt?: string | undefined;
+                    scorerAISystemPromptId?: string | undefined;
+                    scorerAIProvider?: string | undefined;
+                    scorerAIModelSlug?: string | undefined;
+                    scorerAIInputTokensUsed?: number | undefined;
+                    scorerAIOutputTokensUsed?: number | undefined;
+                    scorerAIInputCost?: string | undefined;
+                    scorerAIOutputCost?: string | undefined;
+                }, "kind" | "id" | "namespace" | "schemaVersion">, generator: import("../../../types").IdGenerator): Promise<{
+                    id: string;
+                    value: number;
+                    responseId: string;
+                    scoringMethod: "ai" | "human" | "algo";
+                    match: boolean;
+                    namespace: "example.peerbench.ai";
+                    kind: "llm/exact-match.sc";
+                    schemaVersion: 1;
+                    metadata?: Record<string, unknown> | undefined;
+                    explanation?: string | undefined;
+                    normalized?: {
+                        expected: string;
+                        actual: string;
+                    } | undefined;
+                    scorerAISystemPrompt?: string | undefined;
+                    scorerAISystemPromptId?: string | undefined;
+                    scorerAIProvider?: string | undefined;
+                    scorerAIModelSlug?: string | undefined;
+                    scorerAIInputTokensUsed?: number | undefined;
+                    scorerAIOutputTokensUsed?: number | undefined;
+                    scorerAIInputCost?: string | undefined;
+                    scorerAIOutputCost?: string | undefined;
+                }>;
+            };
+        }];
+        providers: [typeof AbstractLLMProvider];
+        scorers: [typeof ExactMatchScorer, typeof LLMAsAJudgeScorer];
+        parseRunConfig?: boolean;
+        defaults?: {
+            scorer?: LLMAsAJudgeScorer | ExactMatchScorer | undefined;
+            responseIdGenerator?: import("../../../types").IdGenerator;
+            scoreIdGenerator?: import("../../../types").IdGenerator;
+        } | undefined;
+    };
+};