peerbench 0.0.1 → 0.0.2-alpha-dev.0

package/dist/benchmarks/index.d.ts

@@ -0,0 +1,1041 @@
+import { I as IdGenerator } from '../index-BAioQhp2.js';
+import { z } from 'zod';
+import { A as AbstractLLMProvider } from '../llm-DNj_tp2T.js';
+import { a as MCQScorer, L as LLMAsAJudgeScorer } from '../llm-judge-QThCZ9TQ.js';
+import { J as JSONFileStorage } from '../json-file-ZwzLUbje.js';
+import '../provider-BDjGp2y-.js';
+import '../abstract-Dec9Sc5O.js';
+import 'openai/resources/shared';
+import 'openai/resources/chat/completions';
+import '../rate-limiter-CSmVIRsM.js';
+import 'node:fs/promises';
+
+declare const MCQKind: "llm/mcq";
+declare const MCQTestCaseSchemaV1: z.ZodObject<Omit<{
+    id: z.ZodString;
+    namespace: z.ZodString;
+    schemaVersion: z.ZodNumber;
+    kind: z.ZodString;
+    metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+}, "kind" | "namespace" | "schemaVersion"> & {
+    question: z.ZodString;
+    options: z.ZodRecord<z.ZodString, z.ZodString>;
+    correctAnswerKeys: z.ZodArray<z.ZodString>;
+} & {
+    namespace: z.ZodLiteral<"peerbench.ai">;
+    kind: z.ZodLiteral<"llm/mcq.tc">;
+    schemaVersion: z.ZodLiteral<1>;
+}, z.core.$strip> & {
+    new: (input: Omit<{
+        id: string;
+        question: string;
+        options: Record<string, string>;
+        correctAnswerKeys: string[];
+        namespace: "peerbench.ai";
+        kind: "llm/mcq.tc";
+        schemaVersion: 1;
+        metadata?: Record<string, unknown> | undefined;
+    }, "kind" | "namespace" | "schemaVersion">) => {
+        id: string;
+        question: string;
+        options: Record<string, string>;
+        correctAnswerKeys: string[];
+        namespace: "peerbench.ai";
+        kind: "llm/mcq.tc";
+        schemaVersion: 1;
+        metadata?: Record<string, unknown> | undefined;
+    };
+    newWithId(input: Omit<{
+        id: string;
+        question: string;
+        options: Record<string, string>;
+        correctAnswerKeys: string[];
+        namespace: "peerbench.ai";
+        kind: "llm/mcq.tc";
+        schemaVersion: 1;
+        metadata?: Record<string, unknown> | undefined;
+    }, "id" | "kind" | "namespace" | "schemaVersion">, generator: IdGenerator): Promise<{
+        id: string;
+        question: string;
+        options: Record<string, string>;
+        correctAnswerKeys: string[];
+        namespace: "peerbench.ai";
+        kind: "llm/mcq.tc";
+        schemaVersion: 1;
+        metadata?: Record<string, unknown> | undefined;
+    }>;
+};
+type MCQTestCaseV1 = z.infer<typeof MCQTestCaseSchemaV1>;
+declare const MCQResponseSchemaV1: z.ZodObject<Omit<{
+    id: z.ZodString;
+    namespace: z.ZodString;
+    schemaVersion: z.ZodNumber;
+    kind: z.ZodString;
+    startedAt: z.ZodNumber;
+    completedAt: z.ZodNumber;
+    testCaseId: z.ZodString;
+    metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+}, "kind" | "namespace" | "schemaVersion"> & {
+    data: z.ZodString;
+    modelSlug: z.ZodString;
+    provider: z.ZodString;
+    systemPromptId: z.ZodOptional<z.ZodString>;
+    inputTokensUsed: z.ZodOptional<z.ZodNumber>;
+    outputTokensUsed: z.ZodOptional<z.ZodNumber>;
+    inputCost: z.ZodOptional<z.ZodString>;
+    outputCost: z.ZodOptional<z.ZodString>;
+} & {
+    namespace: z.ZodLiteral<"peerbench.ai">;
+    kind: z.ZodLiteral<"llm/mcq.rs">;
+    schemaVersion: z.ZodLiteral<1>;
+}, z.core.$strip> & {
+    new: (input: Omit<{
+        startedAt: number;
+        completedAt: number;
+        id: string;
+        testCaseId: string;
+        data: string;
+        modelSlug: string;
+        provider: string;
+        namespace: "peerbench.ai";
+        kind: "llm/mcq.rs";
+        schemaVersion: 1;
+        metadata?: Record<string, unknown> | undefined;
+        systemPromptId?: string | undefined;
+        inputTokensUsed?: number | undefined;
+        outputTokensUsed?: number | undefined;
+        inputCost?: string | undefined;
+        outputCost?: string | undefined;
+    }, "kind" | "namespace" | "schemaVersion">) => {
+        startedAt: number;
+        completedAt: number;
+        id: string;
+        testCaseId: string;
+        data: string;
+        modelSlug: string;
+        provider: string;
+        namespace: "peerbench.ai";
+        kind: "llm/mcq.rs";
+        schemaVersion: 1;
+        metadata?: Record<string, unknown> | undefined;
+        systemPromptId?: string | undefined;
+        inputTokensUsed?: number | undefined;
+        outputTokensUsed?: number | undefined;
+        inputCost?: string | undefined;
+        outputCost?: string | undefined;
+    };
+    newWithId(input: Omit<{
+        startedAt: number;
+        completedAt: number;
+        id: string;
+        testCaseId: string;
+        data: string;
+        modelSlug: string;
+        provider: string;
+        namespace: "peerbench.ai";
+        kind: "llm/mcq.rs";
+        schemaVersion: 1;
+        metadata?: Record<string, unknown> | undefined;
+        systemPromptId?: string | undefined;
+        inputTokensUsed?: number | undefined;
+        outputTokensUsed?: number | undefined;
+        inputCost?: string | undefined;
+        outputCost?: string | undefined;
+    }, "id" | "kind" | "namespace" | "schemaVersion">, generator: IdGenerator): Promise<{
+        startedAt: number;
+        completedAt: number;
+        id: string;
+        testCaseId: string;
+        data: string;
+        modelSlug: string;
+        provider: string;
+        namespace: "peerbench.ai";
+        kind: "llm/mcq.rs";
+        schemaVersion: 1;
+        metadata?: Record<string, unknown> | undefined;
+        systemPromptId?: string | undefined;
+        inputTokensUsed?: number | undefined;
+        outputTokensUsed?: number | undefined;
+        inputCost?: string | undefined;
+        outputCost?: string | undefined;
+    }>;
+};
+type MCQResponseV1 = z.infer<typeof MCQResponseSchemaV1>;
+declare const MCQScoreSchemaV1: z.ZodObject<Omit<{
+    id: z.ZodString;
+    namespace: z.ZodString;
+    kind: z.ZodString;
+    schemaVersion: z.ZodNumber;
+    value: z.ZodNumber;
+    responseId: z.ZodString;
+    explanation: z.ZodOptional<z.ZodString>;
+    metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+    scoringMethod: z.ZodEnum<{
+        readonly ai: "ai";
+        readonly human: "human";
+        readonly algo: "algo";
+    }>;
+}, "kind" | "namespace" | "schemaVersion"> & {
+    extractedAnswers: z.ZodArray<z.ZodString>;
+    scorerAISystemPrompt: z.ZodOptional<z.ZodString>;
+    scorerAISystemPromptId: z.ZodOptional<z.ZodString>;
+    scorerAIProvider: z.ZodOptional<z.ZodString>;
+    scorerAIModelSlug: z.ZodOptional<z.ZodString>;
+    scorerAIInputTokensUsed: z.ZodOptional<z.ZodNumber>;
+    scorerAIOutputTokensUsed: z.ZodOptional<z.ZodNumber>;
+    scorerAIInputCost: z.ZodOptional<z.ZodString>;
+    scorerAIOutputCost: z.ZodOptional<z.ZodString>;
+} & {
+    namespace: z.ZodLiteral<"peerbench.ai">;
+    kind: z.ZodLiteral<"llm/mcq.sc">;
+    schemaVersion: z.ZodLiteral<1>;
+}, z.core.$strip> & {
+    new: (input: Omit<{
+        id: string;
+        value: number;
+        responseId: string;
+        scoringMethod: "ai" | "human" | "algo";
+        extractedAnswers: string[];
+        namespace: "peerbench.ai";
+        kind: "llm/mcq.sc";
+        schemaVersion: 1;
+        explanation?: string | undefined;
+        metadata?: Record<string, unknown> | undefined;
+        scorerAISystemPrompt?: string | undefined;
+        scorerAISystemPromptId?: string | undefined;
+        scorerAIProvider?: string | undefined;
+        scorerAIModelSlug?: string | undefined;
+        scorerAIInputTokensUsed?: number | undefined;
+        scorerAIOutputTokensUsed?: number | undefined;
+        scorerAIInputCost?: string | undefined;
+        scorerAIOutputCost?: string | undefined;
+    }, "kind" | "namespace" | "schemaVersion">) => {
+        id: string;
+        value: number;
+        responseId: string;
+        scoringMethod: "ai" | "human" | "algo";
+        extractedAnswers: string[];
+        namespace: "peerbench.ai";
+        kind: "llm/mcq.sc";
+        schemaVersion: 1;
+        explanation?: string | undefined;
+        metadata?: Record<string, unknown> | undefined;
+        scorerAISystemPrompt?: string | undefined;
+        scorerAISystemPromptId?: string | undefined;
+        scorerAIProvider?: string | undefined;
+        scorerAIModelSlug?: string | undefined;
+        scorerAIInputTokensUsed?: number | undefined;
+        scorerAIOutputTokensUsed?: number | undefined;
+        scorerAIInputCost?: string | undefined;
+        scorerAIOutputCost?: string | undefined;
+    };
+    newWithId(input: Omit<{
+        id: string;
+        value: number;
+        responseId: string;
+        scoringMethod: "ai" | "human" | "algo";
+        extractedAnswers: string[];
+        namespace: "peerbench.ai";
+        kind: "llm/mcq.sc";
+        schemaVersion: 1;
+        explanation?: string | undefined;
+        metadata?: Record<string, unknown> | undefined;
+        scorerAISystemPrompt?: string | undefined;
+        scorerAISystemPromptId?: string | undefined;
+        scorerAIProvider?: string | undefined;
+        scorerAIModelSlug?: string | undefined;
+        scorerAIInputTokensUsed?: number | undefined;
+        scorerAIOutputTokensUsed?: number | undefined;
+        scorerAIInputCost?: string | undefined;
+        scorerAIOutputCost?: string | undefined;
+    }, "id" | "kind" | "namespace" | "schemaVersion">, generator: IdGenerator): Promise<{
+        id: string;
+        value: number;
+        responseId: string;
+        scoringMethod: "ai" | "human" | "algo";
+        extractedAnswers: string[];
+        namespace: "peerbench.ai";
+        kind: "llm/mcq.sc";
+        schemaVersion: 1;
+        explanation?: string | undefined;
+        metadata?: Record<string, unknown> | undefined;
+        scorerAISystemPrompt?: string | undefined;
+        scorerAISystemPromptId?: string | undefined;
+        scorerAIProvider?: string | undefined;
+        scorerAIModelSlug?: string | undefined;
+        scorerAIInputTokensUsed?: number | undefined;
+        scorerAIOutputTokensUsed?: number | undefined;
+        scorerAIInputCost?: string | undefined;
+        scorerAIOutputCost?: string | undefined;
+    }>;
+};
+type MCQScoreV1 = z.infer<typeof MCQScoreSchemaV1>;
+
+declare const MultiTurnKind: "llm/multi-turn";
+declare const MultiTurnTestCaseSchemaV1: z.ZodObject<Omit<{
+    id: z.ZodString;
+    namespace: z.ZodString;
+    schemaVersion: z.ZodNumber;
+    kind: z.ZodString;
+    metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+}, "kind" | "namespace" | "schemaVersion"> & {
+    messages: z.ZodArray<z.ZodObject<{
+        role: z.ZodString;
+        content: z.ZodString;
+        goodAnswers: z.ZodOptional<z.ZodArray<z.ZodString>>;
+        badAnswers: z.ZodOptional<z.ZodArray<z.ZodString>>;
+    }, z.core.$strip>>;
+    maxTurns: z.ZodOptional<z.ZodNumber>;
+    expectedOutcome: z.ZodOptional<z.ZodString>;
+} & {
+    namespace: z.ZodLiteral<"peerbench.ai">;
+    kind: z.ZodLiteral<"llm/multi-turn.tc">;
+    schemaVersion: z.ZodLiteral<1>;
+}, z.core.$strip> & {
+    new: (input: Omit<{
+        id: string;
+        messages: {
+            role: string;
+            content: string;
+            goodAnswers?: string[] | undefined;
+            badAnswers?: string[] | undefined;
+        }[];
+        namespace: "peerbench.ai";
+        kind: "llm/multi-turn.tc";
+        schemaVersion: 1;
+        metadata?: Record<string, unknown> | undefined;
+        maxTurns?: number | undefined;
+        expectedOutcome?: string | undefined;
+    }, "kind" | "namespace" | "schemaVersion">) => {
+        id: string;
+        messages: {
+            role: string;
+            content: string;
+            goodAnswers?: string[] | undefined;
+            badAnswers?: string[] | undefined;
+        }[];
+        namespace: "peerbench.ai";
+        kind: "llm/multi-turn.tc";
+        schemaVersion: 1;
+        metadata?: Record<string, unknown> | undefined;
+        maxTurns?: number | undefined;
+        expectedOutcome?: string | undefined;
+    };
+    newWithId(input: Omit<{
+        id: string;
+        messages: {
+            role: string;
+            content: string;
+            goodAnswers?: string[] | undefined;
+            badAnswers?: string[] | undefined;
+        }[];
+        namespace: "peerbench.ai";
+        kind: "llm/multi-turn.tc";
+        schemaVersion: 1;
+        metadata?: Record<string, unknown> | undefined;
+        maxTurns?: number | undefined;
+        expectedOutcome?: string | undefined;
+    }, "id" | "kind" | "namespace" | "schemaVersion">, generator: IdGenerator): Promise<{
+        id: string;
+        messages: {
+            role: string;
+            content: string;
+            goodAnswers?: string[] | undefined;
+            badAnswers?: string[] | undefined;
+        }[];
+        namespace: "peerbench.ai";
+        kind: "llm/multi-turn.tc";
+        schemaVersion: 1;
+        metadata?: Record<string, unknown> | undefined;
+        maxTurns?: number | undefined;
+        expectedOutcome?: string | undefined;
+    }>;
+};
+type MultiTurnTestCaseV1 = z.infer<typeof MultiTurnTestCaseSchemaV1>;
+declare const MultiTurnResponseSchemaV1: z.ZodObject<Omit<{
+    id: z.ZodString;
+    namespace: z.ZodString;
+    schemaVersion: z.ZodNumber;
+    kind: z.ZodString;
+    startedAt: z.ZodNumber;
+    completedAt: z.ZodNumber;
+    testCaseId: z.ZodString;
+    metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+}, "kind" | "namespace" | "schemaVersion"> & {
+    replies: z.ZodArray<z.ZodObject<{
+        messageIndex: z.ZodNumber;
+        startedAt: z.ZodNumber;
+        completedAt: z.ZodNumber;
+        data: z.ZodString;
+        inputTokensUsed: z.ZodOptional<z.ZodNumber>;
+        outputTokensUsed: z.ZodOptional<z.ZodNumber>;
+        inputCost: z.ZodOptional<z.ZodString>;
+        outputCost: z.ZodOptional<z.ZodString>;
+    }, z.core.$strip>>;
+    data: z.ZodString;
+    modelSlug: z.ZodString;
+    provider: z.ZodString;
+    systemPromptId: z.ZodOptional<z.ZodString>;
+    inputTokensUsed: z.ZodOptional<z.ZodNumber>;
+    outputTokensUsed: z.ZodOptional<z.ZodNumber>;
+    inputCost: z.ZodOptional<z.ZodString>;
+    outputCost: z.ZodOptional<z.ZodString>;
+} & {
+    namespace: z.ZodLiteral<"peerbench.ai">;
+    kind: z.ZodLiteral<"llm/multi-turn.rs">;
+    schemaVersion: z.ZodLiteral<1>;
+}, z.core.$strip> & {
+    new: (input: Omit<{
+        startedAt: number;
+        completedAt: number;
+        id: string;
+        testCaseId: string;
+        replies: {
+            messageIndex: number;
+            startedAt: number;
+            completedAt: number;
+            data: string;
+            inputTokensUsed?: number | undefined;
+            outputTokensUsed?: number | undefined;
+            inputCost?: string | undefined;
+            outputCost?: string | undefined;
+        }[];
+        data: string;
+        modelSlug: string;
+        provider: string;
+        namespace: "peerbench.ai";
+        kind: "llm/multi-turn.rs";
+        schemaVersion: 1;
+        metadata?: Record<string, unknown> | undefined;
+        systemPromptId?: string | undefined;
+        inputTokensUsed?: number | undefined;
+        outputTokensUsed?: number | undefined;
+        inputCost?: string | undefined;
+        outputCost?: string | undefined;
+    }, "kind" | "namespace" | "schemaVersion">) => {
+        startedAt: number;
+        completedAt: number;
+        id: string;
+        testCaseId: string;
+        replies: {
+            messageIndex: number;
+            startedAt: number;
+            completedAt: number;
+            data: string;
+            inputTokensUsed?: number | undefined;
+            outputTokensUsed?: number | undefined;
+            inputCost?: string | undefined;
+            outputCost?: string | undefined;
+        }[];
+        data: string;
+        modelSlug: string;
+        provider: string;
+        namespace: "peerbench.ai";
+        kind: "llm/multi-turn.rs";
+        schemaVersion: 1;
+        metadata?: Record<string, unknown> | undefined;
+        systemPromptId?: string | undefined;
+        inputTokensUsed?: number | undefined;
+        outputTokensUsed?: number | undefined;
+        inputCost?: string | undefined;
+        outputCost?: string | undefined;
+    };
+    newWithId(input: Omit<{
+        startedAt: number;
+        completedAt: number;
+        id: string;
+        testCaseId: string;
+        replies: {
+            messageIndex: number;
+            startedAt: number;
+            completedAt: number;
+            data: string;
+            inputTokensUsed?: number | undefined;
+            outputTokensUsed?: number | undefined;
+            inputCost?: string | undefined;
+            outputCost?: string | undefined;
+        }[];
+        data: string;
+        modelSlug: string;
+        provider: string;
+        namespace: "peerbench.ai";
+        kind: "llm/multi-turn.rs";
+        schemaVersion: 1;
+        metadata?: Record<string, unknown> | undefined;
+        systemPromptId?: string | undefined;
+        inputTokensUsed?: number | undefined;
+        outputTokensUsed?: number | undefined;
+        inputCost?: string | undefined;
+        outputCost?: string | undefined;
+    }, "id" | "kind" | "namespace" | "schemaVersion">, generator: IdGenerator): Promise<{
+        startedAt: number;
+        completedAt: number;
+        id: string;
+        testCaseId: string;
+        replies: {
+            messageIndex: number;
+            startedAt: number;
+            completedAt: number;
+            data: string;
+            inputTokensUsed?: number | undefined;
+            outputTokensUsed?: number | undefined;
+            inputCost?: string | undefined;
+            outputCost?: string | undefined;
+        }[];
+        data: string;
+        modelSlug: string;
+        provider: string;
+        namespace: "peerbench.ai";
+        kind: "llm/multi-turn.rs";
+        schemaVersion: 1;
+        metadata?: Record<string, unknown> | undefined;
+        systemPromptId?: string | undefined;
+        inputTokensUsed?: number | undefined;
+        outputTokensUsed?: number | undefined;
+        inputCost?: string | undefined;
+        outputCost?: string | undefined;
+    }>;
+};
+type MultiTurnResponseV1 = z.infer<typeof MultiTurnResponseSchemaV1>;
+declare const MultiTurnScoreSchemaV1: z.ZodObject<Omit<{
+    id: z.ZodString;
+    namespace: z.ZodString;
+    kind: z.ZodString;
+    schemaVersion: z.ZodNumber;
+    value: z.ZodNumber;
+    responseId: z.ZodString;
+    explanation: z.ZodOptional<z.ZodString>;
+    metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+    scoringMethod: z.ZodEnum<{
+        readonly ai: "ai";
+        readonly human: "human";
+        readonly algo: "algo";
+    }>;
+}, "kind" | "namespace" | "schemaVersion"> & {
+    individualScores: z.ZodArray<z.ZodObject<{
+        replyIndex: z.ZodNumber;
+        value: z.ZodNumber;
+    }, z.core.$strip>>;
+    scorerAISystemPrompt: z.ZodOptional<z.ZodString>;
+    scorerAISystemPromptId: z.ZodOptional<z.ZodString>;
+    scorerAIProvider: z.ZodOptional<z.ZodString>;
+    scorerAIModelSlug: z.ZodOptional<z.ZodString>;
+    scorerAIInputTokensUsed: z.ZodOptional<z.ZodNumber>;
+    scorerAIOutputTokensUsed: z.ZodOptional<z.ZodNumber>;
+    scorerAIInputCost: z.ZodOptional<z.ZodString>;
+    scorerAIOutputCost: z.ZodOptional<z.ZodString>;
+} & {
+    namespace: z.ZodLiteral<"peerbench.ai">;
+    kind: z.ZodLiteral<"llm/multi-turn.sc">;
+    schemaVersion: z.ZodLiteral<1>;
+}, z.core.$strip> & {
+    new: (input: Omit<{
+        id: string;
+        value: number;
+        responseId: string;
+        scoringMethod: "ai" | "human" | "algo";
+        individualScores: {
+            replyIndex: number;
+            value: number;
+        }[];
+        namespace: "peerbench.ai";
+        kind: "llm/multi-turn.sc";
+        schemaVersion: 1;
+        explanation?: string | undefined;
+        metadata?: Record<string, unknown> | undefined;
+        scorerAISystemPrompt?: string | undefined;
+        scorerAISystemPromptId?: string | undefined;
+        scorerAIProvider?: string | undefined;
+        scorerAIModelSlug?: string | undefined;
+        scorerAIInputTokensUsed?: number | undefined;
+        scorerAIOutputTokensUsed?: number | undefined;
+        scorerAIInputCost?: string | undefined;
+        scorerAIOutputCost?: string | undefined;
+    }, "kind" | "namespace" | "schemaVersion">) => {
+        id: string;
+        value: number;
+        responseId: string;
+        scoringMethod: "ai" | "human" | "algo";
+        individualScores: {
+            replyIndex: number;
+            value: number;
+        }[];
+        namespace: "peerbench.ai";
+        kind: "llm/multi-turn.sc";
+        schemaVersion: 1;
+        explanation?: string | undefined;
+        metadata?: Record<string, unknown> | undefined;
+        scorerAISystemPrompt?: string | undefined;
+        scorerAISystemPromptId?: string | undefined;
+        scorerAIProvider?: string | undefined;
+        scorerAIModelSlug?: string | undefined;
+        scorerAIInputTokensUsed?: number | undefined;
+        scorerAIOutputTokensUsed?: number | undefined;
+        scorerAIInputCost?: string | undefined;
+        scorerAIOutputCost?: string | undefined;
+    };
+    newWithId(input: Omit<{
+        id: string;
+        value: number;
+        responseId: string;
+        scoringMethod: "ai" | "human" | "algo";
+        individualScores: {
+            replyIndex: number;
+            value: number;
+        }[];
+        namespace: "peerbench.ai";
+        kind: "llm/multi-turn.sc";
+        schemaVersion: 1;
+        explanation?: string | undefined;
+        metadata?: Record<string, unknown> | undefined;
+        scorerAISystemPrompt?: string | undefined;
+        scorerAISystemPromptId?: string | undefined;
+        scorerAIProvider?: string | undefined;
+        scorerAIModelSlug?: string | undefined;
+        scorerAIInputTokensUsed?: number | undefined;
+        scorerAIOutputTokensUsed?: number | undefined;
+        scorerAIInputCost?: string | undefined;
+        scorerAIOutputCost?: string | undefined;
+    }, "id" | "kind" | "namespace" | "schemaVersion">, generator: IdGenerator): Promise<{
+        id: string;
+        value: number;
+        responseId: string;
+        scoringMethod: "ai" | "human" | "algo";
+        individualScores: {
+            replyIndex: number;
+            value: number;
+        }[];
+        namespace: "peerbench.ai";
+        kind: "llm/multi-turn.sc";
+        schemaVersion: 1;
+        explanation?: string | undefined;
+        metadata?: Record<string, unknown> | undefined;
+        scorerAISystemPrompt?: string | undefined;
+        scorerAISystemPromptId?: string | undefined;
+        scorerAIProvider?: string | undefined;
+        scorerAIModelSlug?: string | undefined;
+        scorerAIInputTokensUsed?: number | undefined;
+        scorerAIOutputTokensUsed?: number | undefined;
+        scorerAIInputCost?: string | undefined;
+        scorerAIOutputCost?: string | undefined;
+    }>;
+};
+type MultiTurnScoreV1 = z.infer<typeof MultiTurnScoreSchemaV1>;
+
+declare const QAKind: "llm/qa";
+declare const QATestCaseSchemaV1: z.ZodObject<Omit<{
+    id: z.ZodString;
+    namespace: z.ZodString;
+    schemaVersion: z.ZodNumber;
+    kind: z.ZodString;
+    metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+}, "kind" | "namespace" | "schemaVersion"> & {
+    question: z.ZodString;
+    goodAnswers: z.ZodArray<z.ZodString>;
+    badAnswers: z.ZodArray<z.ZodString>;
+} & {
+    namespace: z.ZodLiteral<"peerbench.ai">;
+    kind: z.ZodLiteral<"llm/qa.tc">;
+    schemaVersion: z.ZodLiteral<1>;
+}, z.core.$strip> & {
+    new: (input: Omit<{
+        id: string;
+        question: string;
+        goodAnswers: string[];
+        badAnswers: string[];
+        namespace: "peerbench.ai";
+        kind: "llm/qa.tc";
+        schemaVersion: 1;
+        metadata?: Record<string, unknown> | undefined;
+    }, "kind" | "namespace" | "schemaVersion">) => {
+        id: string;
+        question: string;
+        goodAnswers: string[];
+        badAnswers: string[];
+        namespace: "peerbench.ai";
+        kind: "llm/qa.tc";
+        schemaVersion: 1;
+        metadata?: Record<string, unknown> | undefined;
+    };
+    newWithId(input: Omit<{
+        id: string;
+        question: string;
+        goodAnswers: string[];
+        badAnswers: string[];
+        namespace: "peerbench.ai";
+        kind: "llm/qa.tc";
+        schemaVersion: 1;
+        metadata?: Record<string, unknown> | undefined;
+    }, "id" | "kind" | "namespace" | "schemaVersion">, generator: IdGenerator): Promise<{
+        id: string;
+        question: string;
+        goodAnswers: string[];
+        badAnswers: string[];
+        namespace: "peerbench.ai";
+        kind: "llm/qa.tc";
+        schemaVersion: 1;
+        metadata?: Record<string, unknown> | undefined;
+    }>;
+};
+type QATestCaseV1 = z.infer<typeof QATestCaseSchemaV1>;
+declare const QAResponseSchemaV1: z.ZodObject<Omit<{
+    id: z.ZodString;
+    namespace: z.ZodString;
+    schemaVersion: z.ZodNumber;
+    kind: z.ZodString;
+    startedAt: z.ZodNumber;
+    completedAt: z.ZodNumber;
+    testCaseId: z.ZodString;
+    metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+}, "kind" | "namespace" | "schemaVersion"> & {
+    data: z.ZodString;
+    modelSlug: z.ZodString;
+    provider: z.ZodString;
+    systemPromptId: z.ZodOptional<z.ZodString>;
+    inputTokensUsed: z.ZodOptional<z.ZodNumber>;
+    outputTokensUsed: z.ZodOptional<z.ZodNumber>;
+    inputCost: z.ZodOptional<z.ZodString>;
+    outputCost: z.ZodOptional<z.ZodString>;
+} & {
+    namespace: z.ZodLiteral<"peerbench.ai">;
+    kind: z.ZodLiteral<"llm/qa.rs">;
+    schemaVersion: z.ZodLiteral<1>;
+}, z.core.$strip> & {
+    new: (input: Omit<{
+        startedAt: number;
+        completedAt: number;
+        id: string;
+        testCaseId: string;
+        data: string;
+        modelSlug: string;
+        provider: string;
+        namespace: "peerbench.ai";
+        kind: "llm/qa.rs";
+        schemaVersion: 1;
+        metadata?: Record<string, unknown> | undefined;
+        systemPromptId?: string | undefined;
+        inputTokensUsed?: number | undefined;
+        outputTokensUsed?: number | undefined;
+        inputCost?: string | undefined;
+        outputCost?: string | undefined;
+    }, "kind" | "namespace" | "schemaVersion">) => {
+        startedAt: number;
+        completedAt: number;
+        id: string;
+        testCaseId: string;
+        data: string;
+        modelSlug: string;
+        provider: string;
+        namespace: "peerbench.ai";
+        kind: "llm/qa.rs";
+        schemaVersion: 1;
+        metadata?: Record<string, unknown> | undefined;
+        systemPromptId?: string | undefined;
+        inputTokensUsed?: number | undefined;
+        outputTokensUsed?: number | undefined;
+        inputCost?: string | undefined;
+        outputCost?: string | undefined;
+    };
+    newWithId(input: Omit<{
+        startedAt: number;
+        completedAt: number;
+        id: string;
+        testCaseId: string;
+        data: string;
+        modelSlug: string;
+        provider: string;
+        namespace: "peerbench.ai";
+        kind: "llm/qa.rs";
+        schemaVersion: 1;
+        metadata?: Record<string, unknown> | undefined;
+        systemPromptId?: string | undefined;
+        inputTokensUsed?: number | undefined;
+        outputTokensUsed?: number | undefined;
+        inputCost?: string | undefined;
+        outputCost?: string | undefined;
+    }, "id" | "kind" | "namespace" | "schemaVersion">, generator: IdGenerator): Promise<{
+        startedAt: number;
+        completedAt: number;
+        id: string;
+        testCaseId: string;
+        data: string;
+        modelSlug: string;
+        provider: string;
+        namespace: "peerbench.ai";
+        kind: "llm/qa.rs";
+        schemaVersion: 1;
+        metadata?: Record<string, unknown> | undefined;
+        systemPromptId?: string | undefined;
+        inputTokensUsed?: number | undefined;
+        outputTokensUsed?: number | undefined;
+        inputCost?: string | undefined;
+        outputCost?: string | undefined;
+    }>;
+};
+type QAResponseV1 = z.infer<typeof QAResponseSchemaV1>;
+declare const QAScoreSchemaV1: z.ZodObject<Omit<{
+    id: z.ZodString;
+    namespace: z.ZodString;
+    kind: z.ZodString;
+    schemaVersion: z.ZodNumber;
+    value: z.ZodNumber;
+    responseId: z.ZodString;
+    explanation: z.ZodOptional<z.ZodString>;
+    metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+    scoringMethod: z.ZodEnum<{
+        readonly ai: "ai";
+        readonly human: "human";
+        readonly algo: "algo";
+    }>;
+}, "kind" | "namespace" | "schemaVersion"> & {
+    scorerAISystemPrompt: z.ZodOptional<z.ZodString>;
+    scorerAISystemPromptId: z.ZodOptional<z.ZodString>;
+    scorerAIProvider: z.ZodOptional<z.ZodString>;
+    scorerAIModelSlug: z.ZodOptional<z.ZodString>;
+    scorerAIInputTokensUsed: z.ZodOptional<z.ZodNumber>;
+    scorerAIOutputTokensUsed: z.ZodOptional<z.ZodNumber>;
+    scorerAIInputCost: z.ZodOptional<z.ZodString>;
+    scorerAIOutputCost: z.ZodOptional<z.ZodString>;
+} & {
+    namespace: z.ZodLiteral<"peerbench.ai">;
+    kind: z.ZodLiteral<"llm/qa.sc">;
+    schemaVersion: z.ZodLiteral<1>;
+}, z.core.$strip> & {
+    new: (input: Omit<{
+        id: string;
+        value: number;
+        responseId: string;
+        scoringMethod: "ai" | "human" | "algo";
+        namespace: "peerbench.ai";
+        kind: "llm/qa.sc";
+        schemaVersion: 1;
+        explanation?: string | undefined;
+        metadata?: Record<string, unknown> | undefined;
+        scorerAISystemPrompt?: string | undefined;
+        scorerAISystemPromptId?: string | undefined;
+        scorerAIProvider?: string | undefined;
+        scorerAIModelSlug?: string | undefined;
+        scorerAIInputTokensUsed?: number | undefined;
+        scorerAIOutputTokensUsed?: number | undefined;
+        scorerAIInputCost?: string | undefined;
+        scorerAIOutputCost?: string | undefined;
+    }, "kind" | "namespace" | "schemaVersion">) => {
+        id: string;
+        value: number;
+        responseId: string;
+        scoringMethod: "ai" | "human" | "algo";
+        namespace: "peerbench.ai";
+        kind: "llm/qa.sc";
+        schemaVersion: 1;
+        explanation?: string | undefined;
+        metadata?: Record<string, unknown> | undefined;
+        scorerAISystemPrompt?: string | undefined;
+        scorerAISystemPromptId?: string | undefined;
+        scorerAIProvider?: string | undefined;
+        scorerAIModelSlug?: string | undefined;
+        scorerAIInputTokensUsed?: number | undefined;
+        scorerAIOutputTokensUsed?: number | undefined;
+        scorerAIInputCost?: string | undefined;
+        scorerAIOutputCost?: string | undefined;
+    };
+    newWithId(input: Omit<{
+        id: string;
+        value: number;
+        responseId: string;
+        scoringMethod: "ai" | "human" | "algo";
+        namespace: "peerbench.ai";
+        kind: "llm/qa.sc";
+        schemaVersion: 1;
+        explanation?: string | undefined;
+        metadata?: Record<string, unknown> | undefined;
+        scorerAISystemPrompt?: string | undefined;
+        scorerAISystemPromptId?: string | undefined;
+        scorerAIProvider?: string | undefined;
+        scorerAIModelSlug?: string | undefined;
+        scorerAIInputTokensUsed?: number | undefined;
+        scorerAIOutputTokensUsed?: number | undefined;
+        scorerAIInputCost?: string | undefined;
+        scorerAIOutputCost?: string | undefined;
+    }, "id" | "kind" | "namespace" | "schemaVersion">, generator: IdGenerator): Promise<{
+        id: string;
+        value: number;
+        responseId: string;
+        scoringMethod: "ai" | "human" | "algo";
+        namespace: "peerbench.ai";
+        kind: "llm/qa.sc";
+        schemaVersion: 1;
+        explanation?: string | undefined;
+        metadata?: Record<string, unknown> | undefined;
+        scorerAISystemPrompt?: string | undefined;
+        scorerAISystemPromptId?: string | undefined;
+        scorerAIProvider?: string | undefined;
+        scorerAIModelSlug?: string | undefined;
+        scorerAIInputTokensUsed?: number | undefined;
+        scorerAIOutputTokensUsed?: number | undefined;
+        scorerAIInputCost?: string | undefined;
+        scorerAIOutputCost?: string | undefined;
+    }>;
+};
+type QAScoreV1 = z.infer<typeof QAScoreSchemaV1>;
+
+declare const peerbenchRunner: (params: {
+    testCase: {
+        id: string;
+        question: string;
+        options: Record<string, string>;
+        correctAnswerKeys: string[];
+        namespace: "peerbench.ai";
+        kind: "llm/mcq.tc";
+        schemaVersion: 1;
+        metadata?: Record<string, unknown> | undefined;
+    } | {
+        id: string;
+        question: string;
+        goodAnswers: string[];
+        badAnswers: string[];
+        namespace: "peerbench.ai";
+        kind: "llm/qa.tc";
+        schemaVersion: 1;
+        metadata?: Record<string, unknown> | undefined;
+    };
+    provider: AbstractLLMProvider;
+    scorer?: MCQScorer | LLMAsAJudgeScorer | undefined;
+    runConfig: {
+        model: string;
+        llmJudgeModel?: string | undefined;
+        llmJudgeSystemPrompt?: {
+            id: string;
+            version: number;
+            content: string;
+            namespace: "peerbench.ai";
+            kind: `${string}/simple.sys-prompt`;
+            schemaVersion: 1;
+            metadata?: Record<string, unknown> | undefined;
+        } | undefined;
+        systemPrompt?: {
+            id: string;
+            version: number;
+            content: string;
+            namespace: "peerbench.ai";
+            kind: `${string}/simple.sys-prompt`;
+            schemaVersion: 1;
+            metadata?: Record<string, unknown> | undefined;
+        } | undefined;
+        templateVariables?: Record<string, string> | undefined;
+    };
+    idGenerators?: {
+        response?: IdGenerator;
+        score?: IdGenerator;
+    };
+}) => Promise<{
+    response: {
+        startedAt: number;
+        completedAt: number;
+        id: string;
+        testCaseId: string;
+        data: string;
+        modelSlug: string;
+        provider: string;
+        namespace: "peerbench.ai";
+        kind: "llm/mcq.rs";
+        schemaVersion: 1;
+        metadata?: Record<string, unknown> | undefined;
+        systemPromptId?: string | undefined;
+        inputTokensUsed?: number | undefined;
+        outputTokensUsed?: number | undefined;
+        inputCost?: string | undefined;
+        outputCost?: string | undefined;
+    } | {
+        startedAt: number;
+        completedAt: number;
+        id: string;
+        testCaseId: string;
+        data: string;
+        modelSlug: string;
+        provider: string;
+        namespace: "peerbench.ai";
+        kind: "llm/qa.rs";
+        schemaVersion: 1;
+        metadata?: Record<string, unknown> | undefined;
+        systemPromptId?: string | undefined;
+        inputTokensUsed?: number | undefined;
+        outputTokensUsed?: number | undefined;
+        inputCost?: string | undefined;
+        outputCost?: string | undefined;
+    };
+    score?: {
+        id: string;
+        value: number;
+        responseId: string;
+        scoringMethod: "ai" | "human" | "algo";
+        extractedAnswers: string[];
+        namespace: "peerbench.ai";
+        kind: "llm/mcq.sc";
+        schemaVersion: 1;
+        explanation?: string | undefined;
+        metadata?: Record<string, unknown> | undefined;
+        scorerAISystemPrompt?: string | undefined;
+        scorerAISystemPromptId?: string | undefined;
+        scorerAIProvider?: string | undefined;
+        scorerAIModelSlug?: string | undefined;
+        scorerAIInputTokensUsed?: number | undefined;
+        scorerAIOutputTokensUsed?: number | undefined;
+        scorerAIInputCost?: string | undefined;
+        scorerAIOutputCost?: string | undefined;
+    } | {
+        id: string;
+        value: number;
+        responseId: string;
+        scoringMethod: "ai" | "human" | "algo";
+        namespace: "peerbench.ai";
+        kind: "llm/qa.sc";
+        schemaVersion: 1;
+        explanation?: string | undefined;
+        metadata?: Record<string, unknown> | undefined;
+        scorerAISystemPrompt?: string | undefined;
+        scorerAISystemPromptId?: string | undefined;
+        scorerAIProvider?: string | undefined;
+        scorerAIModelSlug?: string | undefined;
+        scorerAIInputTokensUsed?: number | undefined;
+        scorerAIOutputTokensUsed?: number | undefined;
+        scorerAIInputCost?: string | undefined;
+        scorerAIOutputCost?: string | undefined;
+    } | undefined;
+}>;
+
+declare class PeerbenchJSONStorage extends JSONFileStorage<MCQTestCaseV1 | MCQResponseV1 | MCQScoreV1 | QATestCaseV1 | QAResponseV1 | QAScoreV1 | MultiTurnTestCaseV1 | MultiTurnResponseV1 | MultiTurnScoreV1> {
+    constructor(config: {
+        path: string;
+        chunkSize?: number;
+    });
+}
+
+declare const index_MCQKind: typeof MCQKind;
+declare const index_MCQResponseSchemaV1: typeof MCQResponseSchemaV1;
+type index_MCQResponseV1 = MCQResponseV1;
+declare const index_MCQScoreSchemaV1: typeof MCQScoreSchemaV1;
+type index_MCQScoreV1 = MCQScoreV1;
+declare const index_MCQTestCaseSchemaV1: typeof MCQTestCaseSchemaV1;
+type index_MCQTestCaseV1 = MCQTestCaseV1;
+declare const index_MultiTurnKind: typeof MultiTurnKind;
+declare const index_MultiTurnResponseSchemaV1: typeof MultiTurnResponseSchemaV1;
+type index_MultiTurnResponseV1 = MultiTurnResponseV1;
+declare const index_MultiTurnScoreSchemaV1: typeof MultiTurnScoreSchemaV1;
+type index_MultiTurnScoreV1 = MultiTurnScoreV1;
+declare const index_MultiTurnTestCaseSchemaV1: typeof MultiTurnTestCaseSchemaV1;
+type index_MultiTurnTestCaseV1 = MultiTurnTestCaseV1;
+type index_PeerbenchJSONStorage = PeerbenchJSONStorage;
+declare const index_PeerbenchJSONStorage: typeof PeerbenchJSONStorage;
+declare const index_QAKind: typeof QAKind;
+declare const index_QAResponseSchemaV1: typeof QAResponseSchemaV1;
+type index_QAResponseV1 = QAResponseV1;
+declare const index_QAScoreSchemaV1: typeof QAScoreSchemaV1;
+type index_QAScoreV1 = QAScoreV1;
+declare const index_QATestCaseSchemaV1: typeof QATestCaseSchemaV1;
+type index_QATestCaseV1 = QATestCaseV1;
+declare const index_peerbenchRunner: typeof peerbenchRunner;
+declare namespace index {
+  export { index_MCQKind as MCQKind, index_MCQResponseSchemaV1 as MCQResponseSchemaV1, type index_MCQResponseV1 as MCQResponseV1, index_MCQScoreSchemaV1 as MCQScoreSchemaV1, type index_MCQScoreV1 as MCQScoreV1, index_MCQTestCaseSchemaV1 as MCQTestCaseSchemaV1, type index_MCQTestCaseV1 as MCQTestCaseV1, index_MultiTurnKind as MultiTurnKind, index_MultiTurnResponseSchemaV1 as MultiTurnResponseSchemaV1, type index_MultiTurnResponseV1 as MultiTurnResponseV1, index_MultiTurnScoreSchemaV1 as MultiTurnScoreSchemaV1, type index_MultiTurnScoreV1 as MultiTurnScoreV1, index_MultiTurnTestCaseSchemaV1 as MultiTurnTestCaseSchemaV1, type index_MultiTurnTestCaseV1 as MultiTurnTestCaseV1, index_PeerbenchJSONStorage as PeerbenchJSONStorage, index_QAKind as QAKind, index_QAResponseSchemaV1 as QAResponseSchemaV1, type index_QAResponseV1 as QAResponseV1, index_QAScoreSchemaV1 as QAScoreSchemaV1, type index_QAScoreV1 as QAScoreV1, index_QATestCaseSchemaV1 as QATestCaseSchemaV1, type index_QATestCaseV1 as QATestCaseV1, index_peerbenchRunner as peerbenchRunner };
+}
+
+export { index as peerbench };