peerbench 0.0.7 → 0.0.9

package/dist/benchmarks/examples/text-transform/runner.d.ts

@@ -0,0 +1,524 @@
+import { AbstractLLMProvider } from "../../../providers/index.js";
+import z from "zod";
+export declare const textTransformRunner: ((params: {
+    testCase: {
+        id: string;
+        input: string;
+        namespace: "example.peerbench.ai";
+        kind: "llm/text-transform-echo.tc";
+        schemaVersion: 1;
+        metadata?: Record<string, unknown> | undefined;
+    } | {
+        id: string;
+        input: string;
+        namespace: "example.peerbench.ai";
+        kind: "llm/text-transform-reverse.tc";
+        schemaVersion: 1;
+        metadata?: Record<string, unknown> | undefined;
+    };
+    provider: AbstractLLMProvider;
+    scorer?: undefined;
+    runConfig: {
+        model: string;
+        temperature?: number | undefined;
+    };
+    idGenerators?: {
+        response?: import("../../../types").IdGenerator;
+        score?: import("../../../types").IdGenerator;
+    };
+}) => Promise<{
+    response: {
+        startedAt: number;
+        completedAt: number;
+        id: string;
+        testCaseId: string;
+        data: string;
+        modelSlug: string;
+        provider: string;
+        namespace: "example.peerbench.ai";
+        kind: "llm/text-transform-echo.rs";
+        schemaVersion: 1;
+        metadata?: Record<string, unknown> | undefined;
+        systemPromptId?: string | undefined;
+        inputTokensUsed?: number | undefined;
+        outputTokensUsed?: number | undefined;
+        inputCost?: string | undefined;
+        outputCost?: string | undefined;
+    } | {
+        startedAt: number;
+        completedAt: number;
+        id: string;
+        testCaseId: string;
+        data: string;
+        modelSlug: string;
+        provider: string;
+        namespace: "example.peerbench.ai";
+        kind: "llm/text-transform-reverse.rs";
+        schemaVersion: 1;
+        metadata?: Record<string, unknown> | undefined;
+        systemPromptId?: string | undefined;
+        inputTokensUsed?: number | undefined;
+        outputTokensUsed?: number | undefined;
+        inputCost?: string | undefined;
+        outputCost?: string | undefined;
+    };
+    score?: {
+        id: string;
+        value: number;
+        responseId: string;
+        scoringMethod: "ai" | "human" | "algo";
+        match: boolean;
+        namespace: "example.peerbench.ai";
+        kind: "llm/text-transform-echo.sc";
+        schemaVersion: 1;
+        metadata?: Record<string, unknown> | undefined;
+        explanation?: string | undefined;
+    } | {
+        id: string;
+        value: number;
+        responseId: string;
+        scoringMethod: "ai" | "human" | "algo";
+        match: boolean;
+        expected: string;
+        namespace: "example.peerbench.ai";
+        kind: "llm/text-transform-reverse.sc";
+        schemaVersion: 1;
+        metadata?: Record<string, unknown> | undefined;
+        explanation?: string | undefined;
+    } | undefined;
+}>) & {
+    config: {
+        runConfigSchema: z.ZodObject<{
+            model: z.ZodString;
+            temperature: z.ZodOptional<z.ZodNumber>;
+        }, z.core.$strip>;
+        schemaSets: [{
+            readonly testCase: z.ZodObject<Omit<{
+                id: z.ZodString;
+                namespace: z.ZodString;
+                schemaVersion: z.ZodNumber;
+                kind: z.ZodString;
+                metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+            }, "kind" | "namespace" | "schemaVersion"> & {
+                input: z.ZodString;
+            } & {
+                namespace: z.ZodLiteral<"example.peerbench.ai">;
+                kind: z.ZodLiteral<"llm/text-transform-echo.tc">;
+                schemaVersion: z.ZodLiteral<1>;
+            }, z.core.$strip> & {
+                new: (input: Omit<{
+                    id: string;
+                    input: string;
+                    namespace: "example.peerbench.ai";
+                    kind: "llm/text-transform-echo.tc";
+                    schemaVersion: 1;
+                    metadata?: Record<string, unknown> | undefined;
+                }, "kind" | "namespace" | "schemaVersion">) => {
+                    id: string;
+                    input: string;
+                    namespace: "example.peerbench.ai";
+                    kind: "llm/text-transform-echo.tc";
+                    schemaVersion: 1;
+                    metadata?: Record<string, unknown> | undefined;
+                };
+                newWithId(input: Omit<{
+                    id: string;
+                    input: string;
+                    namespace: "example.peerbench.ai";
+                    kind: "llm/text-transform-echo.tc";
+                    schemaVersion: 1;
+                    metadata?: Record<string, unknown> | undefined;
+                }, "kind" | "id" | "namespace" | "schemaVersion">, generator: import("../../../types").IdGenerator): Promise<{
+                    id: string;
+                    input: string;
+                    namespace: "example.peerbench.ai";
+                    kind: "llm/text-transform-echo.tc";
+                    schemaVersion: 1;
+                    metadata?: Record<string, unknown> | undefined;
+                }>;
+            };
+            readonly response: z.ZodObject<Omit<{
+                id: z.ZodString;
+                namespace: z.ZodString;
+                schemaVersion: z.ZodNumber;
+                kind: z.ZodString;
+                startedAt: z.ZodNumber;
+                completedAt: z.ZodNumber;
+                testCaseId: z.ZodString;
+                metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+            }, "kind" | "namespace" | "schemaVersion"> & {
+                data: z.ZodString;
+                modelSlug: z.ZodString;
+                provider: z.ZodString;
+                systemPromptId: z.ZodOptional<z.ZodString>;
+                inputTokensUsed: z.ZodOptional<z.ZodNumber>;
+                outputTokensUsed: z.ZodOptional<z.ZodNumber>;
+                inputCost: z.ZodOptional<z.ZodString>;
+                outputCost: z.ZodOptional<z.ZodString>;
+            } & {
+                namespace: z.ZodLiteral<"example.peerbench.ai">;
+                kind: z.ZodLiteral<"llm/text-transform-echo.rs">;
+                schemaVersion: z.ZodLiteral<1>;
+            }, z.core.$strip> & {
+                new: (input: Omit<{
+                    startedAt: number;
+                    completedAt: number;
+                    id: string;
+                    testCaseId: string;
+                    data: string;
+                    modelSlug: string;
+                    provider: string;
+                    namespace: "example.peerbench.ai";
+                    kind: "llm/text-transform-echo.rs";
+                    schemaVersion: 1;
+                    metadata?: Record<string, unknown> | undefined;
+                    systemPromptId?: string | undefined;
+                    inputTokensUsed?: number | undefined;
+                    outputTokensUsed?: number | undefined;
+                    inputCost?: string | undefined;
+                    outputCost?: string | undefined;
+                }, "kind" | "namespace" | "schemaVersion">) => {
+                    startedAt: number;
+                    completedAt: number;
+                    id: string;
+                    testCaseId: string;
+                    data: string;
+                    modelSlug: string;
+                    provider: string;
+                    namespace: "example.peerbench.ai";
+                    kind: "llm/text-transform-echo.rs";
+                    schemaVersion: 1;
+                    metadata?: Record<string, unknown> | undefined;
+                    systemPromptId?: string | undefined;
+                    inputTokensUsed?: number | undefined;
+                    outputTokensUsed?: number | undefined;
+                    inputCost?: string | undefined;
+                    outputCost?: string | undefined;
+                };
+                newWithId(input: Omit<{
+                    startedAt: number;
+                    completedAt: number;
+                    id: string;
+                    testCaseId: string;
+                    data: string;
+                    modelSlug: string;
+                    provider: string;
+                    namespace: "example.peerbench.ai";
+                    kind: "llm/text-transform-echo.rs";
+                    schemaVersion: 1;
+                    metadata?: Record<string, unknown> | undefined;
+                    systemPromptId?: string | undefined;
+                    inputTokensUsed?: number | undefined;
+                    outputTokensUsed?: number | undefined;
+                    inputCost?: string | undefined;
+                    outputCost?: string | undefined;
+                }, "kind" | "id" | "namespace" | "schemaVersion">, generator: import("../../../types").IdGenerator): Promise<{
+                    startedAt: number;
+                    completedAt: number;
+                    id: string;
+                    testCaseId: string;
+                    data: string;
+                    modelSlug: string;
+                    provider: string;
+                    namespace: "example.peerbench.ai";
+                    kind: "llm/text-transform-echo.rs";
+                    schemaVersion: 1;
+                    metadata?: Record<string, unknown> | undefined;
+                    systemPromptId?: string | undefined;
+                    inputTokensUsed?: number | undefined;
+                    outputTokensUsed?: number | undefined;
+                    inputCost?: string | undefined;
+                    outputCost?: string | undefined;
+                }>;
+            };
+            readonly score: z.ZodObject<Omit<{
+                id: z.ZodString;
+                namespace: z.ZodString;
+                kind: z.ZodString;
+                schemaVersion: z.ZodNumber;
+                value: z.ZodNumber;
+                responseId: z.ZodString;
+                explanation: z.ZodOptional<z.ZodString>;
+                metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+                scoringMethod: z.ZodEnum<{
+                    readonly ai: "ai";
+                    readonly human: "human";
+                    readonly algo: "algo";
+                }>;
+            }, "kind" | "namespace" | "schemaVersion"> & {
+                match: z.ZodBoolean;
+            } & {
+                namespace: z.ZodLiteral<"example.peerbench.ai">;
+                kind: z.ZodLiteral<"llm/text-transform-echo.sc">;
+                schemaVersion: z.ZodLiteral<1>;
+            }, z.core.$strip> & {
+                new: (input: Omit<{
+                    id: string;
+                    value: number;
+                    responseId: string;
+                    scoringMethod: "ai" | "human" | "algo";
+                    match: boolean;
+                    namespace: "example.peerbench.ai";
+                    kind: "llm/text-transform-echo.sc";
+                    schemaVersion: 1;
+                    metadata?: Record<string, unknown> | undefined;
+                    explanation?: string | undefined;
+                }, "kind" | "namespace" | "schemaVersion">) => {
+                    id: string;
+                    value: number;
+                    responseId: string;
+                    scoringMethod: "ai" | "human" | "algo";
+                    match: boolean;
+                    namespace: "example.peerbench.ai";
+                    kind: "llm/text-transform-echo.sc";
+                    schemaVersion: 1;
+                    metadata?: Record<string, unknown> | undefined;
+                    explanation?: string | undefined;
+                };
+                newWithId(input: Omit<{
+                    id: string;
+                    value: number;
+                    responseId: string;
+                    scoringMethod: "ai" | "human" | "algo";
+                    match: boolean;
+                    namespace: "example.peerbench.ai";
+                    kind: "llm/text-transform-echo.sc";
+                    schemaVersion: 1;
+                    metadata?: Record<string, unknown> | undefined;
+                    explanation?: string | undefined;
+                }, "kind" | "id" | "namespace" | "schemaVersion">, generator: import("../../../types").IdGenerator): Promise<{
+                    id: string;
+                    value: number;
+                    responseId: string;
+                    scoringMethod: "ai" | "human" | "algo";
+                    match: boolean;
+                    namespace: "example.peerbench.ai";
+                    kind: "llm/text-transform-echo.sc";
+                    schemaVersion: 1;
+                    metadata?: Record<string, unknown> | undefined;
+                    explanation?: string | undefined;
+                }>;
+            };
+        }, {
+            readonly testCase: z.ZodObject<Omit<{
+                id: z.ZodString;
+                namespace: z.ZodString;
+                schemaVersion: z.ZodNumber;
+                kind: z.ZodString;
+                metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+            }, "kind" | "namespace" | "schemaVersion"> & {
+                input: z.ZodString;
+            } & {
+                namespace: z.ZodLiteral<"example.peerbench.ai">;
+                kind: z.ZodLiteral<"llm/text-transform-reverse.tc">;
+                schemaVersion: z.ZodLiteral<1>;
+            }, z.core.$strip> & {
+                new: (input: Omit<{
+                    id: string;
+                    input: string;
+                    namespace: "example.peerbench.ai";
+                    kind: "llm/text-transform-reverse.tc";
+                    schemaVersion: 1;
+                    metadata?: Record<string, unknown> | undefined;
+                }, "kind" | "namespace" | "schemaVersion">) => {
+                    id: string;
+                    input: string;
+                    namespace: "example.peerbench.ai";
+                    kind: "llm/text-transform-reverse.tc";
+                    schemaVersion: 1;
+                    metadata?: Record<string, unknown> | undefined;
+                };
+                newWithId(input: Omit<{
+                    id: string;
+                    input: string;
+                    namespace: "example.peerbench.ai";
+                    kind: "llm/text-transform-reverse.tc";
+                    schemaVersion: 1;
+                    metadata?: Record<string, unknown> | undefined;
+                }, "kind" | "id" | "namespace" | "schemaVersion">, generator: import("../../../types").IdGenerator): Promise<{
+                    id: string;
+                    input: string;
+                    namespace: "example.peerbench.ai";
+                    kind: "llm/text-transform-reverse.tc";
+                    schemaVersion: 1;
+                    metadata?: Record<string, unknown> | undefined;
+                }>;
+            };
+            readonly response: z.ZodObject<Omit<{
+                id: z.ZodString;
+                namespace: z.ZodString;
+                schemaVersion: z.ZodNumber;
+                kind: z.ZodString;
+                startedAt: z.ZodNumber;
+                completedAt: z.ZodNumber;
+                testCaseId: z.ZodString;
+                metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+            }, "kind" | "namespace" | "schemaVersion"> & {
+                data: z.ZodString;
+                modelSlug: z.ZodString;
+                provider: z.ZodString;
+                systemPromptId: z.ZodOptional<z.ZodString>;
+                inputTokensUsed: z.ZodOptional<z.ZodNumber>;
+                outputTokensUsed: z.ZodOptional<z.ZodNumber>;
+                inputCost: z.ZodOptional<z.ZodString>;
+                outputCost: z.ZodOptional<z.ZodString>;
+            } & {
+                namespace: z.ZodLiteral<"example.peerbench.ai">;
+                kind: z.ZodLiteral<"llm/text-transform-reverse.rs">;
+                schemaVersion: z.ZodLiteral<1>;
+            }, z.core.$strip> & {
+                new: (input: Omit<{
+                    startedAt: number;
+                    completedAt: number;
+                    id: string;
+                    testCaseId: string;
+                    data: string;
+                    modelSlug: string;
+                    provider: string;
+                    namespace: "example.peerbench.ai";
+                    kind: "llm/text-transform-reverse.rs";
+                    schemaVersion: 1;
+                    metadata?: Record<string, unknown> | undefined;
+                    systemPromptId?: string | undefined;
+                    inputTokensUsed?: number | undefined;
+                    outputTokensUsed?: number | undefined;
+                    inputCost?: string | undefined;
+                    outputCost?: string | undefined;
+                }, "kind" | "namespace" | "schemaVersion">) => {
+                    startedAt: number;
+                    completedAt: number;
+                    id: string;
+                    testCaseId: string;
+                    data: string;
+                    modelSlug: string;
+                    provider: string;
+                    namespace: "example.peerbench.ai";
+                    kind: "llm/text-transform-reverse.rs";
+                    schemaVersion: 1;
+                    metadata?: Record<string, unknown> | undefined;
+                    systemPromptId?: string | undefined;
+                    inputTokensUsed?: number | undefined;
+                    outputTokensUsed?: number | undefined;
+                    inputCost?: string | undefined;
+                    outputCost?: string | undefined;
+                };
+                newWithId(input: Omit<{
+                    startedAt: number;
+                    completedAt: number;
+                    id: string;
+                    testCaseId: string;
+                    data: string;
+                    modelSlug: string;
+                    provider: string;
+                    namespace: "example.peerbench.ai";
+                    kind: "llm/text-transform-reverse.rs";
+                    schemaVersion: 1;
+                    metadata?: Record<string, unknown> | undefined;
+                    systemPromptId?: string | undefined;
+                    inputTokensUsed?: number | undefined;
+                    outputTokensUsed?: number | undefined;
+                    inputCost?: string | undefined;
+                    outputCost?: string | undefined;
+                }, "kind" | "id" | "namespace" | "schemaVersion">, generator: import("../../../types").IdGenerator): Promise<{
+                    startedAt: number;
+                    completedAt: number;
+                    id: string;
+                    testCaseId: string;
+                    data: string;
+                    modelSlug: string;
+                    provider: string;
+                    namespace: "example.peerbench.ai";
+                    kind: "llm/text-transform-reverse.rs";
+                    schemaVersion: 1;
+                    metadata?: Record<string, unknown> | undefined;
+                    systemPromptId?: string | undefined;
+                    inputTokensUsed?: number | undefined;
+                    outputTokensUsed?: number | undefined;
+                    inputCost?: string | undefined;
+                    outputCost?: string | undefined;
+                }>;
+            };
+            readonly score: z.ZodObject<Omit<{
+                id: z.ZodString;
+                namespace: z.ZodString;
+                kind: z.ZodString;
+                schemaVersion: z.ZodNumber;
+                value: z.ZodNumber;
+                responseId: z.ZodString;
+                explanation: z.ZodOptional<z.ZodString>;
+                metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+                scoringMethod: z.ZodEnum<{
+                    readonly ai: "ai";
+                    readonly human: "human";
+                    readonly algo: "algo";
+                }>;
+            }, "kind" | "namespace" | "schemaVersion"> & {
+                match: z.ZodBoolean;
+                expected: z.ZodString;
+            } & {
+                namespace: z.ZodLiteral<"example.peerbench.ai">;
+                kind: z.ZodLiteral<"llm/text-transform-reverse.sc">;
+                schemaVersion: z.ZodLiteral<1>;
+            }, z.core.$strip> & {
+                new: (input: Omit<{
+                    id: string;
+                    value: number;
+                    responseId: string;
+                    scoringMethod: "ai" | "human" | "algo";
+                    match: boolean;
+                    expected: string;
+                    namespace: "example.peerbench.ai";
+                    kind: "llm/text-transform-reverse.sc";
+                    schemaVersion: 1;
+                    metadata?: Record<string, unknown> | undefined;
+                    explanation?: string | undefined;
+                }, "kind" | "namespace" | "schemaVersion">) => {
+                    id: string;
+                    value: number;
+                    responseId: string;
+                    scoringMethod: "ai" | "human" | "algo";
+                    match: boolean;
+                    expected: string;
+                    namespace: "example.peerbench.ai";
+                    kind: "llm/text-transform-reverse.sc";
+                    schemaVersion: 1;
+                    metadata?: Record<string, unknown> | undefined;
+                    explanation?: string | undefined;
+                };
+                newWithId(input: Omit<{
+                    id: string;
+                    value: number;
+                    responseId: string;
+                    scoringMethod: "ai" | "human" | "algo";
+                    match: boolean;
+                    expected: string;
+                    namespace: "example.peerbench.ai";
+                    kind: "llm/text-transform-reverse.sc";
+                    schemaVersion: 1;
+                    metadata?: Record<string, unknown> | undefined;
+                    explanation?: string | undefined;
+                }, "kind" | "id" | "namespace" | "schemaVersion">, generator: import("../../../types").IdGenerator): Promise<{
+                    id: string;
+                    value: number;
+                    responseId: string;
+                    scoringMethod: "ai" | "human" | "algo";
+                    match: boolean;
+                    expected: string;
+                    namespace: "example.peerbench.ai";
+                    kind: "llm/text-transform-reverse.sc";
+                    schemaVersion: 1;
+                    metadata?: Record<string, unknown> | undefined;
+                    explanation?: string | undefined;
+                }>;
+            };
+        }];
+        providers: [typeof AbstractLLMProvider];
+        scorers: [];
+        parseRunConfig?: boolean;
+        defaults?: {
+            scorer?: undefined;
+            responseIdGenerator?: import("../../../types").IdGenerator;
+            scoreIdGenerator?: import("../../../types").IdGenerator;
+        } | undefined;
+    };
+};