peerbench 0.0.4 → 0.0.6

package/dist/benchmarks/index.d.ts

@@ -878,7 +878,7 @@ declare const QAScoreSchemaV1: z.ZodObject<Omit<{
 };
 type QAScoreV1 = z.infer<typeof QAScoreSchemaV1>;
 
-declare const peerbenchRunner: (params: {
+declare const peerbenchRunner: ((params: {
     testCase: {
         id: string;
         question: string;
@@ -1002,7 +1002,632 @@ declare const peerbenchRunner: (params: {
         scorerAIInputCost?: string | undefined;
         scorerAIOutputCost?: string | undefined;
     } | undefined;
-}>;
+}>) & {
+    config: {
+        runConfigSchema: z__default.ZodObject<{
+            model: z__default.ZodString;
+            llmJudgeModel: z__default.ZodOptional<z__default.ZodString>;
+            llmJudgeSystemPrompt: z__default.ZodOptional<z__default.ZodObject<Omit<{
+                id: z__default.ZodString;
+                namespace: z__default.ZodString;
+                kind: z__default.ZodString;
+                schemaVersion: z__default.ZodNumber;
+                version: z__default.ZodNumber;
+                metadata: z__default.ZodOptional<z__default.ZodRecord<z__default.ZodString, z__default.ZodUnknown>>;
+            }, "kind" | "namespace" | "schemaVersion"> & {
+                content: z__default.ZodString;
+            } & {
+                namespace: z__default.ZodLiteral<"peerbench.ai">;
+                kind: z__default.ZodLiteral<`${string}/simple.sys-prompt`>;
+                schemaVersion: z__default.ZodLiteral<1>;
+            }, z__default.core.$strip> & {
+                new: (input: Omit<{
+                    id: string;
+                    version: number;
+                    content: string;
+                    namespace: "peerbench.ai";
+                    kind: `${string}/simple.sys-prompt`;
+                    schemaVersion: 1;
+                    metadata?: Record<string, unknown> | undefined;
+                }, "kind" | "namespace" | "schemaVersion">) => {
+                    id: string;
+                    version: number;
+                    content: string;
+                    namespace: "peerbench.ai";
+                    kind: `${string}/simple.sys-prompt`;
+                    schemaVersion: 1;
+                    metadata?: Record<string, unknown> | undefined;
+                };
+                newWithId(input: Omit<{
+                    id: string;
+                    version: number;
+                    content: string;
+                    namespace: "peerbench.ai";
+                    kind: `${string}/simple.sys-prompt`;
+                    schemaVersion: 1;
+                    metadata?: Record<string, unknown> | undefined;
+                }, "id" | "kind" | "namespace" | "schemaVersion">, generator: IdGenerator): Promise<{
+                    id: string;
+                    version: number;
+                    content: string;
+                    namespace: "peerbench.ai";
+                    kind: `${string}/simple.sys-prompt`;
+                    schemaVersion: 1;
+                    metadata?: Record<string, unknown> | undefined;
+                }>;
+            }>;
+            llmJudgeFieldsToExtract: z__default.ZodOptional<z__default.ZodRecord<z__default.ZodString, z__default.ZodCustom<z__default.ZodType<unknown, unknown, z__default.core.$ZodTypeInternals<unknown, unknown>>, z__default.ZodType<unknown, unknown, z__default.core.$ZodTypeInternals<unknown, unknown>>>>>;
+            systemPrompt: z__default.ZodOptional<z__default.ZodObject<Omit<{
+                id: z__default.ZodString;
+                namespace: z__default.ZodString;
+                kind: z__default.ZodString;
+                schemaVersion: z__default.ZodNumber;
+                version: z__default.ZodNumber;
+                metadata: z__default.ZodOptional<z__default.ZodRecord<z__default.ZodString, z__default.ZodUnknown>>;
+            }, "kind" | "namespace" | "schemaVersion"> & {
+                content: z__default.ZodString;
+            } & {
+                namespace: z__default.ZodLiteral<"peerbench.ai">;
+                kind: z__default.ZodLiteral<`${string}/simple.sys-prompt`>;
+                schemaVersion: z__default.ZodLiteral<1>;
+            }, z__default.core.$strip> & {
+                new: (input: Omit<{
+                    id: string;
+                    version: number;
+                    content: string;
+                    namespace: "peerbench.ai";
+                    kind: `${string}/simple.sys-prompt`;
+                    schemaVersion: 1;
+                    metadata?: Record<string, unknown> | undefined;
+                }, "kind" | "namespace" | "schemaVersion">) => {
+                    id: string;
+                    version: number;
+                    content: string;
+                    namespace: "peerbench.ai";
+                    kind: `${string}/simple.sys-prompt`;
+                    schemaVersion: 1;
+                    metadata?: Record<string, unknown> | undefined;
+                };
+                newWithId(input: Omit<{
+                    id: string;
+                    version: number;
+                    content: string;
+                    namespace: "peerbench.ai";
+                    kind: `${string}/simple.sys-prompt`;
+                    schemaVersion: 1;
+                    metadata?: Record<string, unknown> | undefined;
+                }, "id" | "kind" | "namespace" | "schemaVersion">, generator: IdGenerator): Promise<{
+                    id: string;
+                    version: number;
+                    content: string;
+                    namespace: "peerbench.ai";
+                    kind: `${string}/simple.sys-prompt`;
+                    schemaVersion: 1;
+                    metadata?: Record<string, unknown> | undefined;
+                }>;
+            }>;
+            templateVariables: z__default.ZodOptional<z__default.ZodRecord<z__default.ZodString, z__default.ZodString>>;
+        }, z__default.core.$strip>;
+        schemaSets: [{
+            readonly testCase: z__default.ZodObject<Omit<{
+                id: z__default.ZodString;
+                namespace: z__default.ZodString;
+                schemaVersion: z__default.ZodNumber;
+                kind: z__default.ZodString;
+                metadata: z__default.ZodOptional<z__default.ZodRecord<z__default.ZodString, z__default.ZodUnknown>>;
+            }, "kind" | "namespace" | "schemaVersion"> & {
+                question: z__default.ZodString;
+                options: z__default.ZodRecord<z__default.ZodString, z__default.ZodString>;
+                correctAnswerKeys: z__default.ZodArray<z__default.ZodString>;
+            } & {
+                namespace: z__default.ZodLiteral<"peerbench.ai">;
+                kind: z__default.ZodLiteral<"llm/mcq.tc">;
+                schemaVersion: z__default.ZodLiteral<1>;
+            }, z__default.core.$strip> & {
+                new: (input: Omit<{
+                    id: string;
+                    question: string;
+                    options: Record<string, string>;
+                    correctAnswerKeys: string[];
+                    namespace: "peerbench.ai";
+                    kind: "llm/mcq.tc";
+                    schemaVersion: 1;
+                    metadata?: Record<string, unknown> | undefined;
+                }, "kind" | "namespace" | "schemaVersion">) => {
+                    id: string;
+                    question: string;
+                    options: Record<string, string>;
+                    correctAnswerKeys: string[];
+                    namespace: "peerbench.ai";
+                    kind: "llm/mcq.tc";
+                    schemaVersion: 1;
+                    metadata?: Record<string, unknown> | undefined;
+                };
+                newWithId(input: Omit<{
+                    id: string;
+                    question: string;
+                    options: Record<string, string>;
+                    correctAnswerKeys: string[];
+                    namespace: "peerbench.ai";
+                    kind: "llm/mcq.tc";
+                    schemaVersion: 1;
+                    metadata?: Record<string, unknown> | undefined;
+                }, "id" | "kind" | "namespace" | "schemaVersion">, generator: IdGenerator): Promise<{
+                    id: string;
+                    question: string;
+                    options: Record<string, string>;
+                    correctAnswerKeys: string[];
+                    namespace: "peerbench.ai";
+                    kind: "llm/mcq.tc";
+                    schemaVersion: 1;
+                    metadata?: Record<string, unknown> | undefined;
+                }>;
+            };
+            readonly response: z__default.ZodObject<Omit<{
+                id: z__default.ZodString;
+                namespace: z__default.ZodString;
+                schemaVersion: z__default.ZodNumber;
+                kind: z__default.ZodString;
+                startedAt: z__default.ZodNumber;
+                completedAt: z__default.ZodNumber;
+                testCaseId: z__default.ZodString;
+                metadata: z__default.ZodOptional<z__default.ZodRecord<z__default.ZodString, z__default.ZodUnknown>>;
+            }, "kind" | "namespace" | "schemaVersion"> & {
+                data: z__default.ZodString;
+                modelSlug: z__default.ZodString;
+                provider: z__default.ZodString;
+                systemPromptId: z__default.ZodOptional<z__default.ZodString>;
+                inputTokensUsed: z__default.ZodOptional<z__default.ZodNumber>;
+                outputTokensUsed: z__default.ZodOptional<z__default.ZodNumber>;
+                inputCost: z__default.ZodOptional<z__default.ZodString>;
+                outputCost: z__default.ZodOptional<z__default.ZodString>;
+            } & {
+                namespace: z__default.ZodLiteral<"peerbench.ai">;
+                kind: z__default.ZodLiteral<"llm/mcq.rs">;
+                schemaVersion: z__default.ZodLiteral<1>;
+            }, z__default.core.$strip> & {
+                new: (input: Omit<{
+                    startedAt: number;
+                    completedAt: number;
+                    id: string;
+                    testCaseId: string;
+                    data: string;
+                    modelSlug: string;
+                    provider: string;
+                    namespace: "peerbench.ai";
+                    kind: "llm/mcq.rs";
+                    schemaVersion: 1;
+                    metadata?: Record<string, unknown> | undefined;
+                    systemPromptId?: string | undefined;
+                    inputTokensUsed?: number | undefined;
+                    outputTokensUsed?: number | undefined;
+                    inputCost?: string | undefined;
+                    outputCost?: string | undefined;
+                }, "kind" | "namespace" | "schemaVersion">) => {
+                    startedAt: number;
+                    completedAt: number;
+                    id: string;
+                    testCaseId: string;
+                    data: string;
+                    modelSlug: string;
+                    provider: string;
+                    namespace: "peerbench.ai";
+                    kind: "llm/mcq.rs";
+                    schemaVersion: 1;
+                    metadata?: Record<string, unknown> | undefined;
+                    systemPromptId?: string | undefined;
+                    inputTokensUsed?: number | undefined;
+                    outputTokensUsed?: number | undefined;
+                    inputCost?: string | undefined;
+                    outputCost?: string | undefined;
+                };
+                newWithId(input: Omit<{
+                    startedAt: number;
+                    completedAt: number;
+                    id: string;
+                    testCaseId: string;
+                    data: string;
+                    modelSlug: string;
+                    provider: string;
+                    namespace: "peerbench.ai";
+                    kind: "llm/mcq.rs";
+                    schemaVersion: 1;
+                    metadata?: Record<string, unknown> | undefined;
+                    systemPromptId?: string | undefined;
+                    inputTokensUsed?: number | undefined;
+                    outputTokensUsed?: number | undefined;
+                    inputCost?: string | undefined;
+                    outputCost?: string | undefined;
+                }, "id" | "kind" | "namespace" | "schemaVersion">, generator: IdGenerator): Promise<{
+                    startedAt: number;
+                    completedAt: number;
+                    id: string;
+                    testCaseId: string;
+                    data: string;
+                    modelSlug: string;
+                    provider: string;
+                    namespace: "peerbench.ai";
+                    kind: "llm/mcq.rs";
+                    schemaVersion: 1;
+                    metadata?: Record<string, unknown> | undefined;
+                    systemPromptId?: string | undefined;
+                    inputTokensUsed?: number | undefined;
+                    outputTokensUsed?: number | undefined;
+                    inputCost?: string | undefined;
+                    outputCost?: string | undefined;
+                }>;
+            };
+            readonly score: z__default.ZodObject<Omit<{
+                id: z__default.ZodString;
+                namespace: z__default.ZodString;
+                kind: z__default.ZodString;
+                schemaVersion: z__default.ZodNumber;
+                value: z__default.ZodNumber;
+                responseId: z__default.ZodString;
+                explanation: z__default.ZodOptional<z__default.ZodString>;
+                metadata: z__default.ZodOptional<z__default.ZodRecord<z__default.ZodString, z__default.ZodUnknown>>;
+                scoringMethod: z__default.ZodEnum<{
+                    readonly ai: "ai";
+                    readonly human: "human";
+                    readonly algo: "algo";
+                }>;
+            }, "kind" | "namespace" | "schemaVersion"> & {
+                extractedAnswers: z__default.ZodArray<z__default.ZodString>;
+                scorerAISystemPrompt: z__default.ZodOptional<z__default.ZodString>;
+                scorerAISystemPromptId: z__default.ZodOptional<z__default.ZodString>;
+                scorerAIProvider: z__default.ZodOptional<z__default.ZodString>;
+                scorerAIModelSlug: z__default.ZodOptional<z__default.ZodString>;
+                scorerAIInputTokensUsed: z__default.ZodOptional<z__default.ZodNumber>;
+                scorerAIOutputTokensUsed: z__default.ZodOptional<z__default.ZodNumber>;
+                scorerAIInputCost: z__default.ZodOptional<z__default.ZodString>;
+                scorerAIOutputCost: z__default.ZodOptional<z__default.ZodString>;
+            } & {
+                namespace: z__default.ZodLiteral<"peerbench.ai">;
+                kind: z__default.ZodLiteral<"llm/mcq.sc">;
+                schemaVersion: z__default.ZodLiteral<1>;
+            }, z__default.core.$strip> & {
+                new: (input: Omit<{
+                    id: string;
+                    value: number;
+                    responseId: string;
+                    scoringMethod: "ai" | "human" | "algo";
+                    extractedAnswers: string[];
+                    namespace: "peerbench.ai";
+                    kind: "llm/mcq.sc";
+                    schemaVersion: 1;
+                    metadata?: Record<string, unknown> | undefined;
+                    explanation?: string | undefined;
+                    scorerAISystemPrompt?: string | undefined;
+                    scorerAISystemPromptId?: string | undefined;
+                    scorerAIProvider?: string | undefined;
+                    scorerAIModelSlug?: string | undefined;
+                    scorerAIInputTokensUsed?: number | undefined;
+                    scorerAIOutputTokensUsed?: number | undefined;
+                    scorerAIInputCost?: string | undefined;
+                    scorerAIOutputCost?: string | undefined;
+                }, "kind" | "namespace" | "schemaVersion">) => {
+                    id: string;
+                    value: number;
+                    responseId: string;
+                    scoringMethod: "ai" | "human" | "algo";
+                    extractedAnswers: string[];
+                    namespace: "peerbench.ai";
+                    kind: "llm/mcq.sc";
+                    schemaVersion: 1;
+                    metadata?: Record<string, unknown> | undefined;
+                    explanation?: string | undefined;
+                    scorerAISystemPrompt?: string | undefined;
+                    scorerAISystemPromptId?: string | undefined;
+                    scorerAIProvider?: string | undefined;
+                    scorerAIModelSlug?: string | undefined;
+                    scorerAIInputTokensUsed?: number | undefined;
+                    scorerAIOutputTokensUsed?: number | undefined;
+                    scorerAIInputCost?: string | undefined;
+                    scorerAIOutputCost?: string | undefined;
+                };
+                newWithId(input: Omit<{
+                    id: string;
+                    value: number;
+                    responseId: string;
+                    scoringMethod: "ai" | "human" | "algo";
+                    extractedAnswers: string[];
+                    namespace: "peerbench.ai";
+                    kind: "llm/mcq.sc";
+                    schemaVersion: 1;
+                    metadata?: Record<string, unknown> | undefined;
+                    explanation?: string | undefined;
+                    scorerAISystemPrompt?: string | undefined;
+                    scorerAISystemPromptId?: string | undefined;
+                    scorerAIProvider?: string | undefined;
+                    scorerAIModelSlug?: string | undefined;
+                    scorerAIInputTokensUsed?: number | undefined;
+                    scorerAIOutputTokensUsed?: number | undefined;
+                    scorerAIInputCost?: string | undefined;
+                    scorerAIOutputCost?: string | undefined;
+                }, "id" | "kind" | "namespace" | "schemaVersion">, generator: IdGenerator): Promise<{
+                    id: string;
+                    value: number;
+                    responseId: string;
+                    scoringMethod: "ai" | "human" | "algo";
+                    extractedAnswers: string[];
+                    namespace: "peerbench.ai";
+                    kind: "llm/mcq.sc";
+                    schemaVersion: 1;
+                    metadata?: Record<string, unknown> | undefined;
+                    explanation?: string | undefined;
+                    scorerAISystemPrompt?: string | undefined;
+                    scorerAISystemPromptId?: string | undefined;
+                    scorerAIProvider?: string | undefined;
+                    scorerAIModelSlug?: string | undefined;
+                    scorerAIInputTokensUsed?: number | undefined;
+                    scorerAIOutputTokensUsed?: number | undefined;
+                    scorerAIInputCost?: string | undefined;
+                    scorerAIOutputCost?: string | undefined;
+                }>;
+            };
+        }, {
+            readonly testCase: z__default.ZodObject<Omit<{
+                id: z__default.ZodString;
+                namespace: z__default.ZodString;
+                schemaVersion: z__default.ZodNumber;
+                kind: z__default.ZodString;
+                metadata: z__default.ZodOptional<z__default.ZodRecord<z__default.ZodString, z__default.ZodUnknown>>;
+            }, "kind" | "namespace" | "schemaVersion"> & {
+                question: z__default.ZodString;
+                goodAnswers: z__default.ZodArray<z__default.ZodString>;
+                badAnswers: z__default.ZodArray<z__default.ZodString>;
+            } & {
+                namespace: z__default.ZodLiteral<"peerbench.ai">;
+                kind: z__default.ZodLiteral<"llm/qa.tc">;
+                schemaVersion: z__default.ZodLiteral<1>;
+            }, z__default.core.$strip> & {
+                new: (input: Omit<{
+                    id: string;
+                    question: string;
+                    goodAnswers: string[];
+                    badAnswers: string[];
+                    namespace: "peerbench.ai";
+                    kind: "llm/qa.tc";
+                    schemaVersion: 1;
+                    metadata?: Record<string, unknown> | undefined;
+                }, "kind" | "namespace" | "schemaVersion">) => {
+                    id: string;
+                    question: string;
+                    goodAnswers: string[];
+                    badAnswers: string[];
+                    namespace: "peerbench.ai";
+                    kind: "llm/qa.tc";
+                    schemaVersion: 1;
+                    metadata?: Record<string, unknown> | undefined;
+                };
+                newWithId(input: Omit<{
+                    id: string;
+                    question: string;
+                    goodAnswers: string[];
+                    badAnswers: string[];
+                    namespace: "peerbench.ai";
+                    kind: "llm/qa.tc";
+                    schemaVersion: 1;
+                    metadata?: Record<string, unknown> | undefined;
+                }, "id" | "kind" | "namespace" | "schemaVersion">, generator: IdGenerator): Promise<{
+                    id: string;
+                    question: string;
+                    goodAnswers: string[];
+                    badAnswers: string[];
+                    namespace: "peerbench.ai";
+                    kind: "llm/qa.tc";
+                    schemaVersion: 1;
+                    metadata?: Record<string, unknown> | undefined;
+                }>;
+            };
+            readonly response: z__default.ZodObject<Omit<{
+                id: z__default.ZodString;
+                namespace: z__default.ZodString;
+                schemaVersion: z__default.ZodNumber;
+                kind: z__default.ZodString;
+                startedAt: z__default.ZodNumber;
+                completedAt: z__default.ZodNumber;
+                testCaseId: z__default.ZodString;
+                metadata: z__default.ZodOptional<z__default.ZodRecord<z__default.ZodString, z__default.ZodUnknown>>;
+            }, "kind" | "namespace" | "schemaVersion"> & {
+                data: z__default.ZodString;
+                modelSlug: z__default.ZodString;
+                provider: z__default.ZodString;
+                systemPromptId: z__default.ZodOptional<z__default.ZodString>;
+                inputTokensUsed: z__default.ZodOptional<z__default.ZodNumber>;
+                outputTokensUsed: z__default.ZodOptional<z__default.ZodNumber>;
+                inputCost: z__default.ZodOptional<z__default.ZodString>;
+                outputCost: z__default.ZodOptional<z__default.ZodString>;
+            } & {
+                namespace: z__default.ZodLiteral<"peerbench.ai">;
+                kind: z__default.ZodLiteral<"llm/qa.rs">;
+                schemaVersion: z__default.ZodLiteral<1>;
+            }, z__default.core.$strip> & {
+                new: (input: Omit<{
+                    startedAt: number;
+                    completedAt: number;
+                    id: string;
+                    testCaseId: string;
+                    data: string;
+                    modelSlug: string;
+                    provider: string;
+                    namespace: "peerbench.ai";
+                    kind: "llm/qa.rs";
+                    schemaVersion: 1;
+                    metadata?: Record<string, unknown> | undefined;
+                    systemPromptId?: string | undefined;
+                    inputTokensUsed?: number | undefined;
+                    outputTokensUsed?: number | undefined;
+                    inputCost?: string | undefined;
+                    outputCost?: string | undefined;
+                }, "kind" | "namespace" | "schemaVersion">) => {
+                    startedAt: number;
+                    completedAt: number;
+                    id: string;
+                    testCaseId: string;
+                    data: string;
+                    modelSlug: string;
+                    provider: string;
+                    namespace: "peerbench.ai";
+                    kind: "llm/qa.rs";
+                    schemaVersion: 1;
+                    metadata?: Record<string, unknown> | undefined;
+                    systemPromptId?: string | undefined;
+                    inputTokensUsed?: number | undefined;
+                    outputTokensUsed?: number | undefined;
+                    inputCost?: string | undefined;
+                    outputCost?: string | undefined;
+                };
+                newWithId(input: Omit<{
+                    startedAt: number;
+                    completedAt: number;
+                    id: string;
+                    testCaseId: string;
+                    data: string;
+                    modelSlug: string;
+                    provider: string;
+                    namespace: "peerbench.ai";
+                    kind: "llm/qa.rs";
+                    schemaVersion: 1;
+                    metadata?: Record<string, unknown> | undefined;
+                    systemPromptId?: string | undefined;
+                    inputTokensUsed?: number | undefined;
+                    outputTokensUsed?: number | undefined;
+                    inputCost?: string | undefined;
+                    outputCost?: string | undefined;
+                }, "id" | "kind" | "namespace" | "schemaVersion">, generator: IdGenerator): Promise<{
+                    startedAt: number;
+                    completedAt: number;
+                    id: string;
+                    testCaseId: string;
+                    data: string;
+                    modelSlug: string;
+                    provider: string;
+                    namespace: "peerbench.ai";
+                    kind: "llm/qa.rs";
+                    schemaVersion: 1;
+                    metadata?: Record<string, unknown> | undefined;
+                    systemPromptId?: string | undefined;
+                    inputTokensUsed?: number | undefined;
+                    outputTokensUsed?: number | undefined;
+                    inputCost?: string | undefined;
+                    outputCost?: string | undefined;
+                }>;
+            };
+            readonly score: z__default.ZodObject<Omit<{
+                id: z__default.ZodString;
+                namespace: z__default.ZodString;
+                kind: z__default.ZodString;
+                schemaVersion: z__default.ZodNumber;
+                value: z__default.ZodNumber;
+                responseId: z__default.ZodString;
+                explanation: z__default.ZodOptional<z__default.ZodString>;
+                metadata: z__default.ZodOptional<z__default.ZodRecord<z__default.ZodString, z__default.ZodUnknown>>;
+                scoringMethod: z__default.ZodEnum<{
+                    readonly ai: "ai";
+                    readonly human: "human";
+                    readonly algo: "algo";
+                }>;
+            }, "kind" | "namespace" | "schemaVersion"> & {
+                scorerAISystemPrompt: z__default.ZodOptional<z__default.ZodString>;
+                scorerAISystemPromptId: z__default.ZodOptional<z__default.ZodString>;
+                scorerAIProvider: z__default.ZodOptional<z__default.ZodString>;
+                scorerAIModelSlug: z__default.ZodOptional<z__default.ZodString>;
+                scorerAIInputTokensUsed: z__default.ZodOptional<z__default.ZodNumber>;
+                scorerAIOutputTokensUsed: z__default.ZodOptional<z__default.ZodNumber>;
+                scorerAIInputCost: z__default.ZodOptional<z__default.ZodString>;
+                scorerAIOutputCost: z__default.ZodOptional<z__default.ZodString>;
+            } & {
+                namespace: z__default.ZodLiteral<"peerbench.ai">;
+                kind: z__default.ZodLiteral<"llm/qa.sc">;
+                schemaVersion: z__default.ZodLiteral<1>;
+            }, z__default.core.$strip> & {
+                new: (input: Omit<{
+                    id: string;
+                    value: number;
+                    responseId: string;
+                    scoringMethod: "ai" | "human" | "algo";
+                    namespace: "peerbench.ai";
+                    kind: "llm/qa.sc";
+                    schemaVersion: 1;
+                    metadata?: Record<string, unknown> | undefined;
+                    explanation?: string | undefined;
+                    scorerAISystemPrompt?: string | undefined;
+                    scorerAISystemPromptId?: string | undefined;
+                    scorerAIProvider?: string | undefined;
+                    scorerAIModelSlug?: string | undefined;
+                    scorerAIInputTokensUsed?: number | undefined;
+                    scorerAIOutputTokensUsed?: number | undefined;
+                    scorerAIInputCost?: string | undefined;
+                    scorerAIOutputCost?: string | undefined;
+                }, "kind" | "namespace" | "schemaVersion">) => {
+                    id: string;
+                    value: number;
+                    responseId: string;
+                    scoringMethod: "ai" | "human" | "algo";
+                    namespace: "peerbench.ai";
+                    kind: "llm/qa.sc";
+                    schemaVersion: 1;
+                    metadata?: Record<string, unknown> | undefined;
+                    explanation?: string | undefined;
+                    scorerAISystemPrompt?: string | undefined;
+                    scorerAISystemPromptId?: string | undefined;
+                    scorerAIProvider?: string | undefined;
+                    scorerAIModelSlug?: string | undefined;
+                    scorerAIInputTokensUsed?: number | undefined;
+                    scorerAIOutputTokensUsed?: number | undefined;
+                    scorerAIInputCost?: string | undefined;
+                    scorerAIOutputCost?: string | undefined;
+                };
+                newWithId(input: Omit<{
+                    id: string;
+                    value: number;
+                    responseId: string;
+                    scoringMethod: "ai" | "human" | "algo";
+                    namespace: "peerbench.ai";
+                    kind: "llm/qa.sc";
+                    schemaVersion: 1;
+                    metadata?: Record<string, unknown> | undefined;
+                    explanation?: string | undefined;
+                    scorerAISystemPrompt?: string | undefined;
+                    scorerAISystemPromptId?: string | undefined;
+                    scorerAIProvider?: string | undefined;
+                    scorerAIModelSlug?: string | undefined;
+                    scorerAIInputTokensUsed?: number | undefined;
+                    scorerAIOutputTokensUsed?: number | undefined;
+                    scorerAIInputCost?: string | undefined;
+                    scorerAIOutputCost?: string | undefined;
+                }, "id" | "kind" | "namespace" | "schemaVersion">, generator: IdGenerator): Promise<{
+                    id: string;
+                    value: number;
+                    responseId: string;
+                    scoringMethod: "ai" | "human" | "algo";
+                    namespace: "peerbench.ai";
+                    kind: "llm/qa.sc";
+                    schemaVersion: 1;
+                    metadata?: Record<string, unknown> | undefined;
+                    explanation?: string | undefined;
+                    scorerAISystemPrompt?: string | undefined;
+                    scorerAISystemPromptId?: string | undefined;
+                    scorerAIProvider?: string | undefined;
+                    scorerAIModelSlug?: string | undefined;
+                    scorerAIInputTokensUsed?: number | undefined;
+                    scorerAIOutputTokensUsed?: number | undefined;
+                    scorerAIInputCost?: string | undefined;
+                    scorerAIOutputCost?: string | undefined;
+                }>;
+            };
+        }];
+        providers: [typeof AbstractLLMProvider];
+        scorers: [typeof LLMAsAJudgeScorer, typeof MCQScorer];
+        parseRunConfig?: boolean;
+        defaults?: {
+            scorer?: MCQScorer | LLMAsAJudgeScorer | undefined;
+            responseIdGenerator?: IdGenerator;
+            scoreIdGenerator?: IdGenerator;
+        } | undefined;
+    };
+};
 
 declare class PeerbenchJSONStorage extends JSONFileStorage<MCQTestCaseV1 | MCQResponseV1 | MCQScoreV1 | QATestCaseV1 | QAResponseV1 | QAScoreV1 | MultiTurnTestCaseV1 | MultiTurnResponseV1 | MultiTurnScoreV1> {
     constructor(config: {

package/dist/benchmarks/index.js

@@ -3,11 +3,10 @@ import {
 } from "../chunk-Q6GSOHOP.js";
 import {
   defineRunner
-} from "../chunk-QY5MPNNB.js";
+} from "../chunk-RTEAK4II.js";
 import {
-  LLMAsAJudgeScorer,
-  MCQScorer
-} from "../chunk-DNGT4SJC.js";
+  AbstractLLMProvider
+} from "../chunk-3JF7SHLC.js";
 import {
   BaseResponseSchemaV1,
   BaseScoreSchemaV1,
@@ -21,14 +20,15 @@ import {
   ScoringMethod
 } from "../chunk-HMQYGCKI.js";
 import {
-  JSONFileStorage
-} from "../chunk-JFLUJLGT.js";
-import {
-  AbstractLLMProvider
-} from "../chunk-HPPCDSJ3.js";
+  LLMAsAJudgeScorer,
+  MCQScorer
+} from "../chunk-DNGT4SJC.js";
 import {
   PEERBENCH_NAMESPACE
 } from "../chunk-UHHHSYVE.js";
+import {
+  JSONFileStorage
+} from "../chunk-JFLUJLGT.js";
 import {
   idGeneratorUUIDv7
 } from "../chunk-4UBK6452.js";

package/dist/{chunk-HPPCDSJ3.js → chunk-3JF7SHLC.js}

@@ -44,7 +44,7 @@ var MastraProvider = class extends AbstractLLMProvider {
       {
         messages: apiMessages,
         runtimeContext: {
-          "model-id": args.model
+          "model-id": args.modelName
         }
       },
       { memory: args.memory ?? this.memory }
@@ -232,4 +232,4 @@ export {
   OpenAIProvider,
   OpenRouterProvider
 };
-//# sourceMappingURL=chunk-HPPCDSJ3.js.map
+//# sourceMappingURL=chunk-3JF7SHLC.js.map

package/dist/chunk-3JF7SHLC.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/providers/abstract/provider.ts","../src/providers/abstract/llm.ts","../src/providers/mastra.ts","../src/providers/openai.ts","../src/providers/openrouter.ts"],"sourcesContent":["export abstract class AbstractProvider {\n  abstract readonly kind: string;\n}\n\nexport type ProviderResponse<TData = unknown> = {\n  startedAt: number;\n  completedAt: number;\n  data: TData;\n};\n","import { AbstractProvider, ProviderResponse } from \"./provider\";\nimport {\n  ResponseFormatJSONObject,\n  ResponseFormatJSONSchema,\n  ResponseFormatText,\n} from \"openai/resources/shared\";\nimport { ChatCompletionMessageParam } from \"openai/resources/chat/completions\";\n\nexport abstract class AbstractLLMProvider extends AbstractProvider {\n  abstract forward(args: LLMProviderForwardArgs): Promise<ChatResponse>;\n}\n\nexport type LLMProviderForwardArgs = {\n  messages: ChatCompletionMessageParam[];\n  model: string;\n  abortSignal?: AbortSignal;\n  temperature?: number;\n  responseFormat?:\n    | ResponseFormatText\n    | ResponseFormatJSONSchema\n    | ResponseFormatJSONObject;\n};\n\nexport type ChatResponse = ProviderResponse<string> & {\n  inputTokensUsed?: number;\n  outputTokensUsed?: number;\n  inputCost?: string;\n  outputCost?: string;\n\n  metadata?: Record<string, unknown>;\n};\n","import {\n  AbstractLLMProvider,\n  type ChatResponse,\n  type LLMProviderForwardArgs,\n} from \"./abstract/llm\";\nimport { MastraClient, type GetAgentResponse } from \"@mastra/client-js\";\n\nexport class MastraProvider extends AbstractLLMProvider {\n  override readonly kind = \"mastra\";\n\n  private readonly endpoint: string;\n  private readonly authToken?: string;\n  private client: MastraClient;\n  private memory?: AgentMemoryOption;\n\n  constructor(params: {\n    endpoint: string;\n    authToken?: string;\n    memory?: AgentMemoryOption;\n  }) {\n    super();\n    this.endpoint = params.endpoint;\n    this.authToken = params.authToken;\n    this.client = new MastraClient({\n      baseUrl: this.endpoint,\n      headers: this.authToken\n        ? {\n          Authorization: `Bearer ${this.authToken}`,\n        }\n        : undefined,\n    });\n    this.memory = params.memory;\n  }\n\n  override async forward(\n    args: LLMProviderForwardArgs & {\n      memory?: AgentMemoryOption;\n\n      /**\n       * The model that will be used as the brain for the agent.\n       */\n      modelName?: string\n    }\n  ): Promise<ChatResponse> {\n    const apiMessages = args.messages\n      .filter((m) => m.role === \"user\" || m.role === \"assistant\")\n      .map((m) => ({\n        role: m.role as \"user\" | \"assistant\",\n        content: String((m as any).content ?? \"\"),\n      }));\n\n    const agent = this.client.getAgent(args.model);\n\n    const startedAt = Date.now();\n    const response = await agent.generate(\n      {\n        messages: apiMessages,\n        runtimeContext: {\n          \"model-id\": args.modelName,\n        },\n      },\n      { memory: args.memory ?? this.memory }\n    );\n\n    return {\n      data: response.text,\n      startedAt,\n      completedAt: Date.now(),\n    };\n  }\n\n  async getAgentInfo(args: {\n    agentId: string;\n    runtimeContext?: MastraRuntimeContext;\n  }) {\n    return await this.client\n      .getAgent(args.agentId)\n      .details(args.runtimeContext);\n  }\n\n  async getAgents(args?: {\n    runtimeContext?: MastraRuntimeContext;\n    partial?: boolean;\n  }): Promise<Record<string, GetAgentResponse>> {\n    return this.client.getAgents(args?.runtimeContext, args?.partial);\n  }\n}\n\n// NOTE: Mastra client does not export these types\nexport type AgentMemoryOption = Parameters<\n  Parameters<MastraClient[\"getAgent\"]>[\"0\"] extends string\n  ? ReturnType<MastraClient[\"getAgent\"]>[\"generate\"]\n  : never\n>[0] extends { memory?: infer M }\n  ? M\n  : never;\n\ntype MastraRuntimeContext = Parameters<\n  Parameters<MastraClient[\"getAgent\"]>[\"0\"] extends string\n  ? ReturnType<MastraClient[\"getAgent\"]>[\"generate\"]\n  : never\n>[0] extends { runtimeContext?: infer R }\n  ? R\n  : never;\n","import { RateLimiter } from \"@/utils\";\nimport { ChatCompletionMessageParam } from \"openai/resources/chat/completions\";\nimport {\n  ResponseFormatJSONObject,\n  ResponseFormatJSONSchema,\n  ResponseFormatText,\n} from \"openai/resources/shared\";\nimport OpenAI, { APIError } from \"openai\";\nimport { AbstractLLMProvider, ChatResponse } from \"./abstract/llm\";\nimport { PEERBENCH_NAMESPACE } from \"@/constants\";\n\nexport class OpenAIProvider extends AbstractLLMProvider {\n  override readonly kind = `${PEERBENCH_NAMESPACE}/llm/openai` as const;\n\n  private client: OpenAI;\n  private rateLimiter: RateLimiter;\n  private maxRetries: number;\n\n  constructor(config: {\n    apiKey: string;\n    baseURL: string;\n    maxRetries?: number;\n    timeout?: number;\n    rateLimiter?: RateLimiter;\n  }) {\n    super();\n    this.maxRetries = config.maxRetries ?? 3;\n    this.rateLimiter =\n      config.rateLimiter ??\n      new RateLimiter({\n        maxWeight: 20,\n        timeWindow: 3_000,\n      });\n\n    this.client = new OpenAI({\n      baseURL: config.baseURL,\n      apiKey: config.apiKey,\n      timeout: config.timeout,\n      dangerouslyAllowBrowser: true,\n    });\n  }\n\n  async forward(args: {\n    messages: ChatCompletionMessageParam[];\n    model: string;\n    abortSignal?: AbortSignal;\n    temperature?: number;\n    responseFormat?:\n      | ResponseFormatText\n      | ResponseFormatJSONSchema\n      | ResponseFormatJSONObject;\n  }): Promise<ChatResponse> {\n    let retryCount = this.maxRetries;\n    while (retryCount > 0) {\n      let startedAt: Date = new Date();\n\n      try {\n        const response = await this.rateLimiter.execute(\n          async () => {\n            // Capture the start time of the request\n            startedAt = new Date();\n            return await this.client.chat.completions.create(\n              {\n                model: args.model,\n                messages: args.messages,\n                temperature: args.temperature,\n                response_format: args.responseFormat,\n              },\n              // Signal for request\n              { signal: args.abortSignal }\n            );\n          },\n          // Signal for rate limiting\n          { signal: args.abortSignal }\n        );\n\n        if (\"error\" in response) {\n          const err = response.error as any;\n          throw new Error(\n            `${err.message} - Code ${err.code} - ${JSON.stringify(err)}`\n          );\n        }\n\n        if (!response?.choices?.[0]?.message?.content) {\n          throw new Error(\"No content returned from the model\");\n        }\n\n        return {\n          data: response.choices[0].message.content,\n\n          inputTokensUsed: response?.usage?.prompt_tokens,\n          outputTokensUsed: response?.usage?.completion_tokens,\n\n          startedAt: startedAt.getTime(),\n          completedAt: Date.now(),\n        };\n      } catch (err) {\n        if (err instanceof APIError && err.status === 401) {\n          throw new Error(`Invalid credentials provided`, { cause: err });\n        }\n\n        retryCount--;\n\n        // More likely an empty HTTP response returned by the Provider\n        // and it couldn't be parsed as JSON by the OpenAI SDK. We need to retry the request\n        // More info can be found in the following links:\n        // https://www.reddit.com/r/SillyTavernAI/comments/1ik95vr/deepseek_r1_on_openrouter_returning_blank_messages/\n        // https://github.com/cline/cline/issues/60\n        if (err instanceof SyntaxError) {\n          console.debug(err);\n          continue;\n        }\n\n        // If it was another error, just continue until we run out of retries\n        if (retryCount !== 0) {\n          continue;\n        }\n\n        throw new Error(\n          `Failed to forward prompt to the model: ${err instanceof Error ? err.message : err}`,\n          { cause: err }\n        );\n      }\n    }\n\n    throw new Error(\n      `Failed to forward prompt to the model: Max retries reached`,\n      { cause: new Error(\"Max retries reached\") }\n    );\n  }\n}\n","import {\n  AbstractLLMProvider,\n  ChatResponse,\n  LLMProviderForwardArgs,\n} from \"./abstract/llm\";\nimport { RateLimiter } from \"@/utils\";\nimport { OpenAIProvider } from \"./openai\";\nimport { PEERBENCH_NAMESPACE } from \"@/constants\";\nimport Decimal from \"decimal.js\";\nimport axios from \"axios\";\n\nconst baseURL = \"https://openrouter.ai/api/v1\";\nconst MODELS_CACHE_TTL = 1000 * 60 * 60 * 24; // 24 hours\n\nexport class OpenRouterProvider extends AbstractLLMProvider {\n  override readonly kind = `${PEERBENCH_NAMESPACE}/llm/openrouter.ai` as const;\n\n  private models: ModelsResponse | undefined = undefined;\n  private modelsCachePromise: Promise<ModelsResponse | undefined> =\n    Promise.resolve(undefined);\n  private modelsUpdatedAt = 0;\n  private openAIProvider: OpenAIProvider;\n\n  constructor(config: {\n    apiKey: string;\n    maxRetries?: number;\n    timeout?: number;\n    rateLimiter?: RateLimiter;\n  }) {\n    super();\n    this.openAIProvider = new OpenAIProvider({\n      baseURL,\n      apiKey: config.apiKey,\n      maxRetries: config.maxRetries,\n      timeout: config.timeout,\n      rateLimiter: config.rateLimiter,\n    });\n  }\n\n  override async forward(args: LLMProviderForwardArgs): Promise<ChatResponse> {\n    // Update models cache concurrently (non-blocking)\n    const [response] = await Promise.all([\n      this.openAIProvider.forward(args),\n      this.updateModelsCache().catch(() => {\n        // Silently fail if cache update fails so we won't have cost info in the result\n      }),\n    ]);\n\n    // Get the model info from the cache\n    const modelInfo = this.models?.data.find((m) => m.id === args.model);\n    let inputCost: string | undefined = undefined;\n    let outputCost: string | undefined = undefined;\n\n    if (modelInfo !== undefined) {\n      // Use Decimal.js for more accurate calculation\n      if (response.inputTokensUsed !== undefined) {\n        inputCost = new Decimal(modelInfo.pricing.prompt)\n          .mul(response.inputTokensUsed)\n          .toFixed(10);\n      }\n      if (response.outputTokensUsed !== undefined) {\n        outputCost = new Decimal(modelInfo.pricing.completion)\n          .mul(response.outputTokensUsed)\n          .toFixed(10);\n      }\n    }\n\n    return {\n      ...response,\n      inputCost,\n      outputCost,\n    };\n  }\n\n  /**\n   * Updates the cache that holds information about OpenRouter models\n   * including pricing information. It will be valid for 24 hours as\n   * long as the instance of this Provider object is alive.\n   */\n  private async updateModelsCache() {\n    // Chain each update method call to the promise.\n    // This approach prevents race conditions between multiple calls.\n    // Since each call is chained to the end of the previous one,\n    // each promise makes a request only if the models cache is not updated\n    // in the last call. Otherwise it simply resolves to the cached value.\n    this.modelsCachePromise = this.modelsCachePromise\n      .then(async () => {\n        if (\n          // The data presented in the cache\n          this.models !== undefined &&\n          // The cache is still valid\n          Date.now() - this.modelsUpdatedAt < MODELS_CACHE_TTL\n        ) {\n          return this.models;\n        }\n\n        // If the cache is not valid, update it\n        return axios\n          .get<ModelsResponse>(`${baseURL}/models`)\n          .then((res) => res.data)\n          .then((data) => {\n            // Only get the models that supports text input and output\n            data = {\n              data: data.data.filter(\n                (m) =>\n                  m.architecture.input_modalities.includes(\"text\") &&\n                  m.architecture.output_modalities.includes(\"text\") &&\n                  // These models are \"fast apply model\" and don't support multi turn conversations so don't include them\n                  ![\n                    \"morph/morph-v3-large\",\n                    \"morph/morph-v3-fast\",\n                    \"relace/relace-apply-3\",\n                  ].includes(m.id)\n              ),\n            };\n\n            this.models = data;\n            this.modelsUpdatedAt = Date.now();\n\n            return data;\n          });\n      })\n      .catch(() => undefined);\n\n    // Wait for the promise chain to resolve\n    await this.modelsCachePromise;\n  }\n}\n\ntype PutModality = \"text\" | \"image\" | \"file\" | \"audio\";\ntype Modality = \"text->text\" | \"text+image->text\" | \"text+image->text+image\";\ntype ModelsResponse = {\n  data: {\n    readonly id: string;\n    readonly canonical_slug: string;\n    readonly hugging_face_id: null | string;\n    readonly name: string;\n    readonly created: number;\n    readonly description: string;\n    readonly context_length: number;\n    readonly architecture: {\n      readonly modality: Modality;\n      readonly input_modalities: PutModality[];\n      readonly output_modalities: PutModality[];\n      readonly instruct_type: null | string;\n    };\n    readonly pricing: {\n      readonly prompt: string;\n      readonly completion: string;\n      readonly request?: string;\n      readonly image?: string;\n      readonly web_search?: string;\n      readonly internal_reasoning?: string;\n      readonly input_cache_read?: string;\n      readonly input_cache_write?: string;\n      readonly audio?: string;\n    };\n  }[];\n};\n"],"mappings":";;;;;;;;AAAO,IAAe,mBAAf,MAAgC;AAEvC;;;ACMO,IAAe,sBAAf,cAA2C,iBAAiB;AAEnE;;;ACLA,SAAS,oBAA2C;AAE7C,IAAM,iBAAN,cAA6B,oBAAoB;AAAA,EACpC,OAAO;AAAA,EAER;AAAA,EACA;AAAA,EACT;AAAA,EACA;AAAA,EAER,YAAY,QAIT;AACD,UAAM;AACN,SAAK,WAAW,OAAO;AACvB,SAAK,YAAY,OAAO;AACxB,SAAK,SAAS,IAAI,aAAa;AAAA,MAC7B,SAAS,KAAK;AAAA,MACd,SAAS,KAAK,YACV;AAAA,QACA,eAAe,UAAU,KAAK,SAAS;AAAA,MACzC,IACE;AAAA,IACN,CAAC;AACD,SAAK,SAAS,OAAO;AAAA,EACvB;AAAA,EAEA,MAAe,QACb,MAQuB;AACvB,UAAM,cAAc,KAAK,SACtB,OAAO,CAAC,MAAM,EAAE,SAAS,UAAU,EAAE,SAAS,WAAW,EACzD,IAAI,CAAC,OAAO;AAAA,MACX,MAAM,EAAE;AAAA,MACR,SAAS,OAAQ,EAAU,WAAW,EAAE;AAAA,IAC1C,EAAE;AAEJ,UAAM,QAAQ,KAAK,OAAO,SAAS,KAAK,KAAK;AAE7C,UAAM,YAAY,KAAK,IAAI;AAC3B,UAAM,WAAW,MAAM,MAAM;AAAA,MAC3B;AAAA,QACE,UAAU;AAAA,QACV,gBAAgB;AAAA,UACd,YAAY,KAAK;AAAA,QACnB;AAAA,MACF;AAAA,MACA,EAAE,QAAQ,KAAK,UAAU,KAAK,OAAO;AAAA,IACvC;AAEA,WAAO;AAAA,MACL,MAAM,SAAS;AAAA,MACf;AAAA,MACA,aAAa,KAAK,IAAI;AAAA,IACxB;AAAA,EACF;AAAA,EAEA,MAAM,aAAa,MAGhB;AACD,WAAO,MAAM,KAAK,OACf,SAAS,KAAK,OAAO,EACrB,QAAQ,KAAK,cAAc;AAAA,EAChC;AAAA,EAEA,MAAM,UAAU,MAG8B;AAC5C,WAAO,KAAK,OAAO,UAAU,MAAM,gBAAgB,MAAM,OAAO;AAAA,EAClE;AACF;;;AC/EA,OAAO,UAAU,gBAAgB;AAI1B,IAAM,iBAAN,cAA6B,oBAAoB;AAAA,EACpC,OAAO,GAAG,mBAAmB;AAAA,EAEvC;AAAA,EACA;AAAA,EACA;AAAA,EAER,YAAY,QAMT;AACD,UAAM;AACN,SAAK,aAAa,OAAO,cAAc;AACvC,SAAK,cACH,OAAO,eACP,IAAI,YAAY;AAAA,MACd,WAAW;AAAA,MACX,YAAY;AAAA,IACd,CAAC;AAEH,SAAK,SAAS,IAAI,OAAO;AAAA,MACvB,SAAS,OAAO;AAAA,MAChB,QAAQ,OAAO;AAAA,MACf,SAAS,OAAO;AAAA,MAChB,yBAAyB;AAAA,IAC3B,CAAC;AAAA,EACH;AAAA,EAEA,MAAM,QAAQ,MASY;AACxB,QAAI,aAAa,KAAK;AACtB,WAAO,aAAa,GAAG;AACrB,UAAI,YAAkB,oBAAI,KAAK;AAE/B,UAAI;AACF,cAAM,WAAW,MAAM,KAAK,YAAY;AAAA,UACtC,YAAY;AAEV,wBAAY,oBAAI,KAAK;AACrB,mBAAO,MAAM,KAAK,OAAO,KAAK,YAAY;AAAA,cACxC;AAAA,gBACE,OAAO,KAAK;AAAA,gBACZ,UAAU,KAAK;AAAA,gBACf,aAAa,KAAK;AAAA,gBAClB,iBAAiB,KAAK;AAAA,cACxB;AAAA;AAAA,cAEA,EAAE,QAAQ,KAAK,YAAY;AAAA,YAC7B;AAAA,UACF;AAAA;AAAA,UAEA,EAAE,QAAQ,KAAK,YAAY;AAAA,QAC7B;AAEA,YAAI,WAAW,UAAU;AACvB,gBAAM,MAAM,SAAS;AACrB,gBAAM,IAAI;AAAA,YACR,GAAG,IAAI,OAAO,WAAW,IAAI,IAAI,MAAM,KAAK,UAAU,GAAG,CAAC;AAAA,UAC5D;AAAA,QACF;AAEA,YAAI,CAAC,UAAU,UAAU,CAAC,GAAG,SAAS,SAAS;AAC7C,gBAAM,IAAI,MAAM,oCAAoC;AAAA,QACtD;AAEA,eAAO;AAAA,UACL,MAAM,SAAS,QAAQ,CAAC,EAAE,QAAQ;AAAA,UAElC,iBAAiB,UAAU,OAAO;AAAA,UAClC,kBAAkB,UAAU,OAAO;AAAA,UAEnC,WAAW,UAAU,QAAQ;AAAA,UAC7B,aAAa,KAAK,IAAI;AAAA,QACxB;AAAA,MACF,SAAS,KAAK;AACZ,YAAI,eAAe,YAAY,IAAI,WAAW,KAAK;AACjD,gBAAM,IAAI,MAAM,gCAAgC,EAAE,OAAO,IAAI,CAAC;AAAA,QAChE;AAEA;AAOA,YAAI,eAAe,aAAa;AAC9B,kBAAQ,MAAM,GAAG;AACjB;AAAA,QACF;AAGA,YAAI,eAAe,GAAG;AACpB;AAAA,QACF;AAEA,cAAM,IAAI;AAAA,UACR,0CAA0C,eAAe,QAAQ,IAAI,UAAU,GAAG;AAAA,UAClF,EAAE,OAAO,IAAI;AAAA,QACf;AAAA,MACF;AAAA,IACF;AAEA,UAAM,IAAI;AAAA,MACR;AAAA,MACA,EAAE,OAAO,IAAI,MAAM,qBAAqB,EAAE;AAAA,IAC5C;AAAA,EACF;AACF;;;AC1HA,OAAO,aAAa;AACpB,OAAO,WAAW;AAElB,IAAM,UAAU;AAChB,IAAM,mBAAmB,MAAO,KAAK,KAAK;AAEnC,IAAM,qBAAN,cAAiC,oBAAoB;AAAA,EACxC,OAAO,GAAG,mBAAmB;AAAA,EAEvC,SAAqC;AAAA,EACrC,qBACN,QAAQ,QAAQ,MAAS;AAAA,EACnB,kBAAkB;AAAA,EAClB;AAAA,EAER,YAAY,QAKT;AACD,UAAM;AACN,SAAK,iBAAiB,IAAI,eAAe;AAAA,MACvC;AAAA,MACA,QAAQ,OAAO;AAAA,MACf,YAAY,OAAO;AAAA,MACnB,SAAS,OAAO;AAAA,MAChB,aAAa,OAAO;AAAA,IACtB,CAAC;AAAA,EACH;AAAA,EAEA,MAAe,QAAQ,MAAqD;AAE1E,UAAM,CAAC,QAAQ,IAAI,MAAM,QAAQ,IAAI;AAAA,MACnC,KAAK,eAAe,QAAQ,IAAI;AAAA,MAChC,KAAK,kBAAkB,EAAE,MAAM,MAAM;AAAA,MAErC,CAAC;AAAA,IACH,CAAC;AAGD,UAAM,YAAY,KAAK,QAAQ,KAAK,KAAK,CAAC,MAAM,EAAE,OAAO,KAAK,KAAK;AACnE,QAAI,YAAgC;AACpC,QAAI,aAAiC;AAErC,QAAI,cAAc,QAAW;AAE3B,UAAI,SAAS,oBAAoB,QAAW;AAC1C,oBAAY,IAAI,QAAQ,UAAU,QAAQ,MAAM,EAC7C,IAAI,SAAS,eAAe,EAC5B,QAAQ,EAAE;AAAA,MACf;AACA,UAAI,SAAS,qBAAqB,QAAW;AAC3C,qBAAa,IAAI,QAAQ,UAAU,QAAQ,UAAU,EAClD,IAAI,SAAS,gBAAgB,EAC7B,QAAQ,EAAE;AAAA,MACf;AAAA,IACF;AAEA,WAAO;AAAA,MACL,GAAG;AAAA,MACH;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAc,oBAAoB;AAMhC,SAAK,qBAAqB,KAAK,mBAC5B,KAAK,YAAY;AAChB;AAAA;AAAA,QAEE,KAAK,WAAW;AAAA,QAEhB,KAAK,IAAI,IAAI,KAAK,kBAAkB;AAAA,QACpC;AACA,eAAO,KAAK;AAAA,MACd;AAGA,aAAO,MACJ,IAAoB,GAAG,OAAO,SAAS,EACvC,KAAK,CAAC,QAAQ,IAAI,IAAI,EACtB,KAAK,CAAC,SAAS;AAEd,eAAO;AAAA,UACL,MAAM,KAAK,KAAK;AAAA,YACd,CAAC,MACC,EAAE,aAAa,iBAAiB,SAAS,MAAM,KAC/C,EAAE,aAAa,kBAAkB,SAAS,MAAM;AAAA,YAEhD,CAAC;AAAA,cACC;AAAA,cACA;AAAA,cACA;AAAA,YACF,EAAE,SAAS,EAAE,EAAE;AAAA,UACnB;AAAA,QACF;AAEA,aAAK,SAAS;AACd,aAAK,kBAAkB,KAAK,IAAI;AAEhC,eAAO;AAAA,MACT,CAAC;AAAA,IACL,CAAC,EACA,MAAM,MAAM,MAAS;AAGxB,UAAM,KAAK;AAAA,EACb;AACF;","names":[]}

package/dist/{chunk-QY5MPNNB.js → chunk-RTEAK4II.js}

@@ -5,7 +5,7 @@ import {
 // src/helpers/define-runner.ts
 import z from "zod";
 function defineRunner(config, fn) {
-  return async (params) => {
+  const func = async (params) => {
     if (config.runConfigSchema && config.parseRunConfig !== false) {
       z.object(config.runConfigSchema).parse(params.runConfig);
     }
@@ -20,9 +20,18 @@ function defineRunner(config, fn) {
     }
     return await fn(params);
   };
+  return Object.assign(func, {
+    /**
+     * The configuration that was used to define the runner.
+     */
+    config: {
+      ...config,
+      runConfigSchema: z.object(config.runConfigSchema)
+    }
+  });
 }
 
 export {
   defineRunner
 };
-//# sourceMappingURL=chunk-QY5MPNNB.js.map
+//# sourceMappingURL=chunk-RTEAK4II.js.map

package/dist/chunk-RTEAK4II.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/helpers/define-runner.ts"],"sourcesContent":["import {\n  IdGenerator,\n  InferRunConfig,\n  ProviderCtor,\n  Runner,\n  ScorerCtor,\n} from \"@/types\";\nimport { idGeneratorUUIDv7 } from \"@/utils\";\nimport z from \"zod\";\n\nexport function defineRunner<\n  const TProviders extends ProviderCtor[],\n  const TScorers extends ScorerCtor[],\n  const TSchemaSets extends SchemaSetDefinition[],\n  const TRunConfigSchema extends z.ZodRawShape = {},\n>(\n  config: {\n    schemaSets: TSchemaSets;\n    providers: TProviders;\n    scorers: TScorers;\n    runConfigSchema?: TRunConfigSchema;\n\n    /**\n     * @default true\n     */\n    parseRunConfig?: boolean;\n    defaults?: {\n      scorer?: InstanceType<TScorers[number]>;\n      responseIdGenerator?: IdGenerator;\n      scoreIdGenerator?: IdGenerator;\n    };\n  },\n  fn: Runner<\n    TSchemaSets[number][\"testCase\"],\n    TSchemaSets[number][\"response\"],\n    TSchemaSets[number][\"score\"],\n    InstanceType<TProviders[number]>,\n    InstanceType<TScorers[number]>,\n    InferRunConfig<TRunConfigSchema>\n  >\n) {\n  const func = async (params: Parameters<typeof fn>[0]) => {\n    if (config.runConfigSchema && config.parseRunConfig !== false) {\n      z.object(config.runConfigSchema).parse(params.runConfig);\n    }\n\n    if (params.idGenerators && !params.idGenerators.response) {\n      params.idGenerators.response =\n        config.defaults?.responseIdGenerator ?? idGeneratorUUIDv7;\n    }\n\n    if (params.idGenerators && !params.idGenerators.score) {\n      params.idGenerators.score =\n        config.defaults?.scoreIdGenerator ?? idGeneratorUUIDv7;\n    }\n\n    if (params.scorer === undefined) {\n      params.scorer = config.defaults?.scorer ?? undefined;\n    }\n\n    return await fn(params);\n  };\n\n  return Object.assign(func, {\n    /**\n     * The configuration that was used to define the runner.\n     */\n    config: {\n      ...config,\n      runConfigSchema: z.object(config.runConfigSchema),\n    },\n  });\n}\n\ntype SchemaSetDefinition<\n  TTestCase extends z.ZodObject = z.ZodObject,\n  TResponse extends z.ZodObject = z.ZodObject,\n  TScore extends z.ZodObject = z.ZodObject,\n> = {\n  testCase: TTestCase;\n  response: TResponse;\n  score: TScore;\n};\n"],"mappings":";;;;;AAQA,OAAO,OAAO;AAEP,SAAS,aAMd,QAgBA,IAQA;AACA,QAAM,OAAO,OAAO,WAAqC;AACvD,QAAI,OAAO,mBAAmB,OAAO,mBAAmB,OAAO;AAC7D,QAAE,OAAO,OAAO,eAAe,EAAE,MAAM,OAAO,SAAS;AAAA,IACzD;AAEA,QAAI,OAAO,gBAAgB,CAAC,OAAO,aAAa,UAAU;AACxD,aAAO,aAAa,WAClB,OAAO,UAAU,uBAAuB;AAAA,IAC5C;AAEA,QAAI,OAAO,gBAAgB,CAAC,OAAO,aAAa,OAAO;AACrD,aAAO,aAAa,QAClB,OAAO,UAAU,oBAAoB;AAAA,IACzC;AAEA,QAAI,OAAO,WAAW,QAAW;AAC/B,aAAO,SAAS,OAAO,UAAU,UAAU;AAAA,IAC7C;AAEA,WAAO,MAAM,GAAG,MAAM;AAAA,EACxB;AAEA,SAAO,OAAO,OAAO,MAAM;AAAA;AAAA;AAAA;AAAA,IAIzB,QAAQ;AAAA,MACN,GAAG;AAAA,MACH,iBAAiB,EAAE,OAAO,OAAO,eAAe;AAAA,IAClD;AAAA,EACF,CAAC;AACH;","names":[]}

package/dist/index.d.ts

@@ -69,10 +69,29 @@ declare function defineRunner<const TProviders extends ProviderCtor[], const TSc
         responseIdGenerator?: IdGenerator;
         scoreIdGenerator?: IdGenerator;
     };
-}, fn: Runner<TSchemaSets[number]["testCase"], TSchemaSets[number]["response"], TSchemaSets[number]["score"], InstanceType<TProviders[number]>, InstanceType<TScorers[number]>, InferRunConfig<TRunConfigSchema>>): (params: Parameters<typeof fn>[0]) => Promise<{
+}, fn: Runner<TSchemaSets[number]["testCase"], TSchemaSets[number]["response"], TSchemaSets[number]["score"], InstanceType<TProviders[number]>, InstanceType<TScorers[number]>, InferRunConfig<TRunConfigSchema>>): ((params: Parameters<typeof fn>[0]) => Promise<{
     response: z__default.core.output<TSchemaSets[number]["response"]>;
     score?: z__default.core.output<TSchemaSets[number]["score"]> | undefined;
-}>;
+}>) & {
+    /**
+     * The configuration that was used to define the runner.
+     */
+    config: {
+        runConfigSchema: z__default.ZodObject<{ -readonly [P in keyof TRunConfigSchema]: TRunConfigSchema[P]; }, z__default.core.$strip>;
+        schemaSets: TSchemaSets;
+        providers: TProviders;
+        scorers: TScorers;
+        /**
+         * @default true
+         */
+        parseRunConfig?: boolean;
+        defaults?: {
+            scorer?: InstanceType<TScorers[number]>;
+            responseIdGenerator?: IdGenerator;
+            scoreIdGenerator?: IdGenerator;
+        };
+    };
+};
 type SchemaSetDefinition<TTestCase extends z__default.ZodObject = z__default.ZodObject, TResponse extends z__default.ZodObject = z__default.ZodObject, TScore extends z__default.ZodObject = z__default.ZodObject> = {
     testCase: TTestCase;
     response: TResponse;

package/dist/index.js

@@ -1,6 +1,6 @@
 import {
   defineRunner
-} from "./chunk-QY5MPNNB.js";
+} from "./chunk-RTEAK4II.js";
 import {
   ScoringMethod
 } from "./chunk-HMQYGCKI.js";

package/dist/providers/index.js

@@ -4,7 +4,7 @@ import {
   MastraProvider,
   OpenAIProvider,
   OpenRouterProvider
-} from "../chunk-HPPCDSJ3.js";
+} from "../chunk-3JF7SHLC.js";
 import "../chunk-UHHHSYVE.js";
 import "../chunk-4UBK6452.js";
 import "../chunk-PZ5AY32C.js";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "peerbench",
-  "version": "0.0.4",
+  "version": "0.0.6",
   "description": "Tooling help work with AI benchmarking data and create trustworthy AI",
   "keywords": [
     "ai",

package/dist/chunk-HPPCDSJ3.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../src/providers/abstract/provider.ts","../src/providers/abstract/llm.ts","../src/providers/mastra.ts","../src/providers/openai.ts","../src/providers/openrouter.ts"],"sourcesContent":["export abstract class AbstractProvider {\n  abstract readonly kind: string;\n}\n\nexport type ProviderResponse<TData = unknown> = {\n  startedAt: number;\n  completedAt: number;\n  data: TData;\n};\n","import { AbstractProvider, ProviderResponse } from \"./provider\";\nimport {\n  ResponseFormatJSONObject,\n  ResponseFormatJSONSchema,\n  ResponseFormatText,\n} from \"openai/resources/shared\";\nimport { ChatCompletionMessageParam } from \"openai/resources/chat/completions\";\n\nexport abstract class AbstractLLMProvider extends AbstractProvider {\n  abstract forward(args: LLMProviderForwardArgs): Promise<ChatResponse>;\n}\n\nexport type LLMProviderForwardArgs = {\n  messages: ChatCompletionMessageParam[];\n  model: string;\n  abortSignal?: AbortSignal;\n  temperature?: number;\n  responseFormat?:\n    | ResponseFormatText\n    | ResponseFormatJSONSchema\n    | ResponseFormatJSONObject;\n};\n\nexport type ChatResponse = ProviderResponse<string> & {\n  inputTokensUsed?: number;\n  outputTokensUsed?: number;\n  inputCost?: string;\n  outputCost?: string;\n\n  metadata?: Record<string, unknown>;\n};\n","import {\n  AbstractLLMProvider,\n  type ChatResponse,\n  type LLMProviderForwardArgs,\n} from \"./abstract/llm\";\nimport { MastraClient, type GetAgentResponse } from \"@mastra/client-js\";\n\nexport class MastraProvider extends AbstractLLMProvider {\n  override readonly kind = \"mastra\";\n\n  private readonly endpoint: string;\n  private readonly authToken?: string;\n  private client: MastraClient;\n  private memory?: AgentMemoryOption;\n\n  constructor(params: {\n    endpoint: string;\n    authToken?: string;\n    memory?: AgentMemoryOption;\n  }) {\n    super();\n    this.endpoint = params.endpoint;\n    this.authToken = params.authToken;\n    this.client = new MastraClient({\n      baseUrl: this.endpoint,\n      headers: this.authToken\n        ? {\n          Authorization: `Bearer ${this.authToken}`,\n        }\n        : undefined,\n    });\n    this.memory = params.memory;\n  }\n\n  override async forward(\n    args: LLMProviderForwardArgs & {\n      memory?: AgentMemoryOption;\n\n      /**\n       * The model that will be used as the brain for the agent.\n       */\n      modelName?: string\n    }\n  ): Promise<ChatResponse> {\n    const apiMessages = args.messages\n      .filter((m) => m.role === \"user\" || m.role === \"assistant\")\n      .map((m) => ({\n        role: m.role as \"user\" | \"assistant\",\n        content: String((m as any).content ?? \"\"),\n      }));\n\n    const agent = this.client.getAgent(args.model);\n\n    const startedAt = Date.now();\n    const response = await agent.generate(\n      {\n        messages: apiMessages,\n        runtimeContext: {\n          \"model-id\": args.model,\n        },\n      },\n      { memory: args.memory ?? this.memory }\n    );\n\n    return {\n      data: response.text,\n      startedAt,\n      completedAt: Date.now(),\n    };\n  }\n\n  async getAgentInfo(args: {\n    agentId: string;\n    runtimeContext?: MastraRuntimeContext;\n  }) {\n    return await this.client\n      .getAgent(args.agentId)\n      .details(args.runtimeContext);\n  }\n\n  async getAgents(args?: {\n    runtimeContext?: MastraRuntimeContext;\n    partial?: boolean;\n  }): Promise<Record<string, GetAgentResponse>> {\n    return this.client.getAgents(args?.runtimeContext, args?.partial);\n  }\n}\n\n// NOTE: Mastra client does not export these types\nexport type AgentMemoryOption = Parameters<\n  Parameters<MastraClient[\"getAgent\"]>[\"0\"] extends string\n  ? ReturnType<MastraClient[\"getAgent\"]>[\"generate\"]\n  : never\n>[0] extends { memory?: infer M }\n  ? M\n  : never;\n\ntype MastraRuntimeContext = Parameters<\n  Parameters<MastraClient[\"getAgent\"]>[\"0\"] extends string\n  ? ReturnType<MastraClient[\"getAgent\"]>[\"generate\"]\n  : never\n>[0] extends { runtimeContext?: infer R }\n  ? R\n  : never;\n","import { RateLimiter } from \"@/utils\";\nimport { ChatCompletionMessageParam } from \"openai/resources/chat/completions\";\nimport {\n  ResponseFormatJSONObject,\n  ResponseFormatJSONSchema,\n  ResponseFormatText,\n} from \"openai/resources/shared\";\nimport OpenAI, { APIError } from \"openai\";\nimport { AbstractLLMProvider, ChatResponse } from \"./abstract/llm\";\nimport { PEERBENCH_NAMESPACE } from \"@/constants\";\n\nexport class OpenAIProvider extends AbstractLLMProvider {\n  override readonly kind = `${PEERBENCH_NAMESPACE}/llm/openai` as const;\n\n  private client: OpenAI;\n  private rateLimiter: RateLimiter;\n  private maxRetries: number;\n\n  constructor(config: {\n    apiKey: string;\n    baseURL: string;\n    maxRetries?: number;\n    timeout?: number;\n    rateLimiter?: RateLimiter;\n  }) {\n    super();\n    this.maxRetries = config.maxRetries ?? 3;\n    this.rateLimiter =\n      config.rateLimiter ??\n      new RateLimiter({\n        maxWeight: 20,\n        timeWindow: 3_000,\n      });\n\n    this.client = new OpenAI({\n      baseURL: config.baseURL,\n      apiKey: config.apiKey,\n      timeout: config.timeout,\n      dangerouslyAllowBrowser: true,\n    });\n  }\n\n  async forward(args: {\n    messages: ChatCompletionMessageParam[];\n    model: string;\n    abortSignal?: AbortSignal;\n    temperature?: number;\n    responseFormat?:\n      | ResponseFormatText\n      | ResponseFormatJSONSchema\n      | ResponseFormatJSONObject;\n  }): Promise<ChatResponse> {\n    let retryCount = this.maxRetries;\n    while (retryCount > 0) {\n      let startedAt: Date = new Date();\n\n      try {\n        const response = await this.rateLimiter.execute(\n          async () => {\n            // Capture the start time of the request\n            startedAt = new Date();\n            return await this.client.chat.completions.create(\n              {\n                model: args.model,\n                messages: args.messages,\n                temperature: args.temperature,\n                response_format: args.responseFormat,\n              },\n              // Signal for request\n              { signal: args.abortSignal }\n            );\n          },\n          // Signal for rate limiting\n          { signal: args.abortSignal }\n        );\n\n        if (\"error\" in response) {\n          const err = response.error as any;\n          throw new Error(\n            `${err.message} - Code ${err.code} - ${JSON.stringify(err)}`\n          );\n        }\n\n        if (!response?.choices?.[0]?.message?.content) {\n          throw new Error(\"No content returned from the model\");\n        }\n\n        return {\n          data: response.choices[0].message.content,\n\n          inputTokensUsed: response?.usage?.prompt_tokens,\n          outputTokensUsed: response?.usage?.completion_tokens,\n\n          startedAt: startedAt.getTime(),\n          completedAt: Date.now(),\n        };\n      } catch (err) {\n        if (err instanceof APIError && err.status === 401) {\n          throw new Error(`Invalid credentials provided`, { cause: err });\n        }\n\n        retryCount--;\n\n        // More likely an empty HTTP response returned by the Provider\n        // and it couldn't be parsed as JSON by the OpenAI SDK. We need to retry the request\n        // More info can be found in the following links:\n        // https://www.reddit.com/r/SillyTavernAI/comments/1ik95vr/deepseek_r1_on_openrouter_returning_blank_messages/\n        // https://github.com/cline/cline/issues/60\n        if (err instanceof SyntaxError) {\n          console.debug(err);\n          continue;\n        }\n\n        // If it was another error, just continue until we run out of retries\n        if (retryCount !== 0) {\n          continue;\n        }\n\n        throw new Error(\n          `Failed to forward prompt to the model: ${err instanceof Error ? err.message : err}`,\n          { cause: err }\n        );\n      }\n    }\n\n    throw new Error(\n      `Failed to forward prompt to the model: Max retries reached`,\n      { cause: new Error(\"Max retries reached\") }\n    );\n  }\n}\n","import {\n  AbstractLLMProvider,\n  ChatResponse,\n  LLMProviderForwardArgs,\n} from \"./abstract/llm\";\nimport { RateLimiter } from \"@/utils\";\nimport { OpenAIProvider } from \"./openai\";\nimport { PEERBENCH_NAMESPACE } from \"@/constants\";\nimport Decimal from \"decimal.js\";\nimport axios from \"axios\";\n\nconst baseURL = \"https://openrouter.ai/api/v1\";\nconst MODELS_CACHE_TTL = 1000 * 60 * 60 * 24; // 24 hours\n\nexport class OpenRouterProvider extends AbstractLLMProvider {\n  override readonly kind = `${PEERBENCH_NAMESPACE}/llm/openrouter.ai` as const;\n\n  private models: ModelsResponse | undefined = undefined;\n  private modelsCachePromise: Promise<ModelsResponse | undefined> =\n    Promise.resolve(undefined);\n  private modelsUpdatedAt = 0;\n  private openAIProvider: OpenAIProvider;\n\n  constructor(config: {\n    apiKey: string;\n    maxRetries?: number;\n    timeout?: number;\n    rateLimiter?: RateLimiter;\n  }) {\n    super();\n    this.openAIProvider = new OpenAIProvider({\n      baseURL,\n      apiKey: config.apiKey,\n      maxRetries: config.maxRetries,\n      timeout: config.timeout,\n      rateLimiter: config.rateLimiter,\n    });\n  }\n\n  override async forward(args: LLMProviderForwardArgs): Promise<ChatResponse> {\n    // Update models cache concurrently (non-blocking)\n    const [response] = await Promise.all([\n      this.openAIProvider.forward(args),\n      this.updateModelsCache().catch(() => {\n        // Silently fail if cache update fails so we won't have cost info in the result\n      }),\n    ]);\n\n    // Get the model info from the cache\n    const modelInfo = this.models?.data.find((m) => m.id === args.model);\n    let inputCost: string | undefined = undefined;\n    let outputCost: string | undefined = undefined;\n\n    if (modelInfo !== undefined) {\n      // Use Decimal.js for more accurate calculation\n      if (response.inputTokensUsed !== undefined) {\n        inputCost = new Decimal(modelInfo.pricing.prompt)\n          .mul(response.inputTokensUsed)\n          .toFixed(10);\n      }\n      if (response.outputTokensUsed !== undefined) {\n        outputCost = new Decimal(modelInfo.pricing.completion)\n          .mul(response.outputTokensUsed)\n          .toFixed(10);\n      }\n    }\n\n    return {\n      ...response,\n      inputCost,\n      outputCost,\n    };\n  }\n\n  /**\n   * Updates the cache that holds information about OpenRouter models\n   * including pricing information. It will be valid for 24 hours as\n   * long as the instance of this Provider object is alive.\n   */\n  private async updateModelsCache() {\n    // Chain each update method call to the promise.\n    // This approach prevents race conditions between multiple calls.\n    // Since each call is chained to the end of the previous one,\n    // each promise makes a request only if the models cache is not updated\n    // in the last call. Otherwise it simply resolves to the cached value.\n    this.modelsCachePromise = this.modelsCachePromise\n      .then(async () => {\n        if (\n          // The data presented in the cache\n          this.models !== undefined &&\n          // The cache is still valid\n          Date.now() - this.modelsUpdatedAt < MODELS_CACHE_TTL\n        ) {\n          return this.models;\n        }\n\n        // If the cache is not valid, update it\n        return axios\n          .get<ModelsResponse>(`${baseURL}/models`)\n          .then((res) => res.data)\n          .then((data) => {\n            // Only get the models that supports text input and output\n            data = {\n              data: data.data.filter(\n                (m) =>\n                  m.architecture.input_modalities.includes(\"text\") &&\n                  m.architecture.output_modalities.includes(\"text\") &&\n                  // These models are \"fast apply model\" and don't support multi turn conversations so don't include them\n                  ![\n                    \"morph/morph-v3-large\",\n                    \"morph/morph-v3-fast\",\n                    \"relace/relace-apply-3\",\n                  ].includes(m.id)\n              ),\n            };\n\n            this.models = data;\n            this.modelsUpdatedAt = Date.now();\n\n            return data;\n          });\n      })\n      .catch(() => undefined);\n\n    // Wait for the promise chain to resolve\n    await this.modelsCachePromise;\n  }\n}\n\ntype PutModality = \"text\" | \"image\" | \"file\" | \"audio\";\ntype Modality = \"text->text\" | \"text+image->text\" | \"text+image->text+image\";\ntype ModelsResponse = {\n  data: {\n    readonly id: string;\n    readonly canonical_slug: string;\n    readonly hugging_face_id: null | string;\n    readonly name: string;\n    readonly created: number;\n    readonly description: string;\n    readonly context_length: number;\n    readonly architecture: {\n      readonly modality: Modality;\n      readonly input_modalities: PutModality[];\n      readonly output_modalities: PutModality[];\n      readonly instruct_type: null | string;\n    };\n    readonly pricing: {\n      readonly prompt: string;\n      readonly completion: string;\n      readonly request?: string;\n      readonly image?: string;\n      readonly web_search?: string;\n      readonly internal_reasoning?: string;\n      readonly input_cache_read?: string;\n      readonly input_cache_write?: string;\n      readonly audio?: string;\n    };\n  }[];\n};\n"],"mappings":";;;;;;;;AAAO,IAAe,mBAAf,MAAgC;AAEvC;;;ACMO,IAAe,sBAAf,cAA2C,iBAAiB;AAEnE;;;ACLA,SAAS,oBAA2C;AAE7C,IAAM,iBAAN,cAA6B,oBAAoB;AAAA,EACpC,OAAO;AAAA,EAER;AAAA,EACA;AAAA,EACT;AAAA,EACA;AAAA,EAER,YAAY,QAIT;AACD,UAAM;AACN,SAAK,WAAW,OAAO;AACvB,SAAK,YAAY,OAAO;AACxB,SAAK,SAAS,IAAI,aAAa;AAAA,MAC7B,SAAS,KAAK;AAAA,MACd,SAAS,KAAK,YACV;AAAA,QACA,eAAe,UAAU,KAAK,SAAS;AAAA,MACzC,IACE;AAAA,IACN,CAAC;AACD,SAAK,SAAS,OAAO;AAAA,EACvB;AAAA,EAEA,MAAe,QACb,MAQuB;AACvB,UAAM,cAAc,KAAK,SACtB,OAAO,CAAC,MAAM,EAAE,SAAS,UAAU,EAAE,SAAS,WAAW,EACzD,IAAI,CAAC,OAAO;AAAA,MACX,MAAM,EAAE;AAAA,MACR,SAAS,OAAQ,EAAU,WAAW,EAAE;AAAA,IAC1C,EAAE;AAEJ,UAAM,QAAQ,KAAK,OAAO,SAAS,KAAK,KAAK;AAE7C,UAAM,YAAY,KAAK,IAAI;AAC3B,UAAM,WAAW,MAAM,MAAM;AAAA,MAC3B;AAAA,QACE,UAAU;AAAA,QACV,gBAAgB;AAAA,UACd,YAAY,KAAK;AAAA,QACnB;AAAA,MACF;AAAA,MACA,EAAE,QAAQ,KAAK,UAAU,KAAK,OAAO;AAAA,IACvC;AAEA,WAAO;AAAA,MACL,MAAM,SAAS;AAAA,MACf;AAAA,MACA,aAAa,KAAK,IAAI;AAAA,IACxB;AAAA,EACF;AAAA,EAEA,MAAM,aAAa,MAGhB;AACD,WAAO,MAAM,KAAK,OACf,SAAS,KAAK,OAAO,EACrB,QAAQ,KAAK,cAAc;AAAA,EAChC;AAAA,EAEA,MAAM,UAAU,MAG8B;AAC5C,WAAO,KAAK,OAAO,UAAU,MAAM,gBAAgB,MAAM,OAAO;AAAA,EAClE;AACF;;;AC/EA,OAAO,UAAU,gBAAgB;AAI1B,IAAM,iBAAN,cAA6B,oBAAoB;AAAA,EACpC,OAAO,GAAG,mBAAmB;AAAA,EAEvC;AAAA,EACA;AAAA,EACA;AAAA,EAER,YAAY,QAMT;AACD,UAAM;AACN,SAAK,aAAa,OAAO,cAAc;AACvC,SAAK,cACH,OAAO,eACP,IAAI,YAAY;AAAA,MACd,WAAW;AAAA,MACX,YAAY;AAAA,IACd,CAAC;AAEH,SAAK,SAAS,IAAI,OAAO;AAAA,MACvB,SAAS,OAAO;AAAA,MAChB,QAAQ,OAAO;AAAA,MACf,SAAS,OAAO;AAAA,MAChB,yBAAyB;AAAA,IAC3B,CAAC;AAAA,EACH;AAAA,EAEA,MAAM,QAAQ,MASY;AACxB,QAAI,aAAa,KAAK;AACtB,WAAO,aAAa,GAAG;AACrB,UAAI,YAAkB,oBAAI,KAAK;AAE/B,UAAI;AACF,cAAM,WAAW,MAAM,KAAK,YAAY;AAAA,UACtC,YAAY;AAEV,wBAAY,oBAAI,KAAK;AACrB,mBAAO,MAAM,KAAK,OAAO,KAAK,YAAY;AAAA,cACxC;AAAA,gBACE,OAAO,KAAK;AAAA,gBACZ,UAAU,KAAK;AAAA,gBACf,aAAa,KAAK;AAAA,gBAClB,iBAAiB,KAAK;AAAA,cACxB;AAAA;AAAA,cAEA,EAAE,QAAQ,KAAK,YAAY;AAAA,YAC7B;AAAA,UACF;AAAA;AAAA,UAEA,EAAE,QAAQ,KAAK,YAAY;AAAA,QAC7B;AAEA,YAAI,WAAW,UAAU;AACvB,gBAAM,MAAM,SAAS;AACrB,gBAAM,IAAI;AAAA,YACR,GAAG,IAAI,OAAO,WAAW,IAAI,IAAI,MAAM,KAAK,UAAU,GAAG,CAAC;AAAA,UAC5D;AAAA,QACF;AAEA,YAAI,CAAC,UAAU,UAAU,CAAC,GAAG,SAAS,SAAS;AAC7C,gBAAM,IAAI,MAAM,oCAAoC;AAAA,QACtD;AAEA,eAAO;AAAA,UACL,MAAM,SAAS,QAAQ,CAAC,EAAE,QAAQ;AAAA,UAElC,iBAAiB,UAAU,OAAO;AAAA,UAClC,kBAAkB,UAAU,OAAO;AAAA,UAEnC,WAAW,UAAU,QAAQ;AAAA,UAC7B,aAAa,KAAK,IAAI;AAAA,QACxB;AAAA,MACF,SAAS,KAAK;AACZ,YAAI,eAAe,YAAY,IAAI,WAAW,KAAK;AACjD,gBAAM,IAAI,MAAM,gCAAgC,EAAE,OAAO,IAAI,CAAC;AAAA,QAChE;AAEA;AAOA,YAAI,eAAe,aAAa;AAC9B,kBAAQ,MAAM,GAAG;AACjB;AAAA,QACF;AAGA,YAAI,eAAe,GAAG;AACpB;AAAA,QACF;AAEA,cAAM,IAAI;AAAA,UACR,0CAA0C,eAAe,QAAQ,IAAI,UAAU,GAAG;AAAA,UAClF,EAAE,OAAO,IAAI;AAAA,QACf;AAAA,MACF;AAAA,IACF;AAEA,UAAM,IAAI;AAAA,MACR;AAAA,MACA,EAAE,OAAO,IAAI,MAAM,qBAAqB,EAAE;AAAA,IAC5C;AAAA,EACF;AACF;;;AC1HA,OAAO,aAAa;AACpB,OAAO,WAAW;AAElB,IAAM,UAAU;AAChB,IAAM,mBAAmB,MAAO,KAAK,KAAK;AAEnC,IAAM,qBAAN,cAAiC,oBAAoB;AAAA,EACxC,OAAO,GAAG,mBAAmB;AAAA,EAEvC,SAAqC;AAAA,EACrC,qBACN,QAAQ,QAAQ,MAAS;AAAA,EACnB,kBAAkB;AAAA,EAClB;AAAA,EAER,YAAY,QAKT;AACD,UAAM;AACN,SAAK,iBAAiB,IAAI,eAAe;AAAA,MACvC;AAAA,MACA,QAAQ,OAAO;AAAA,MACf,YAAY,OAAO;AAAA,MACnB,SAAS,OAAO;AAAA,MAChB,aAAa,OAAO;AAAA,IACtB,CAAC;AAAA,EACH;AAAA,EAEA,MAAe,QAAQ,MAAqD;AAE1E,UAAM,CAAC,QAAQ,IAAI,MAAM,QAAQ,IAAI;AAAA,MACnC,KAAK,eAAe,QAAQ,IAAI;AAAA,MAChC,KAAK,kBAAkB,EAAE,MAAM,MAAM;AAAA,MAErC,CAAC;AAAA,IACH,CAAC;AAGD,UAAM,YAAY,KAAK,QAAQ,KAAK,KAAK,CAAC,MAAM,EAAE,OAAO,KAAK,KAAK;AACnE,QAAI,YAAgC;AACpC,QAAI,aAAiC;AAErC,QAAI,cAAc,QAAW;AAE3B,UAAI,SAAS,oBAAoB,QAAW;AAC1C,oBAAY,IAAI,QAAQ,UAAU,QAAQ,MAAM,EAC7C,IAAI,SAAS,eAAe,EAC5B,QAAQ,EAAE;AAAA,MACf;AACA,UAAI,SAAS,qBAAqB,QAAW;AAC3C,qBAAa,IAAI,QAAQ,UAAU,QAAQ,UAAU,EAClD,IAAI,SAAS,gBAAgB,EAC7B,QAAQ,EAAE;AAAA,MACf;AAAA,IACF;AAEA,WAAO;AAAA,MACL,GAAG;AAAA,MACH;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAc,oBAAoB;AAMhC,SAAK,qBAAqB,KAAK,mBAC5B,KAAK,YAAY;AAChB;AAAA;AAAA,QAEE,KAAK,WAAW;AAAA,QAEhB,KAAK,IAAI,IAAI,KAAK,kBAAkB;AAAA,QACpC;AACA,eAAO,KAAK;AAAA,MACd;AAGA,aAAO,MACJ,IAAoB,GAAG,OAAO,SAAS,EACvC,KAAK,CAAC,QAAQ,IAAI,IAAI,EACtB,KAAK,CAAC,SAAS;AAEd,eAAO;AAAA,UACL,MAAM,KAAK,KAAK;AAAA,YACd,CAAC,MACC,EAAE,aAAa,iBAAiB,SAAS,MAAM,KAC/C,EAAE,aAAa,kBAAkB,SAAS,MAAM;AAAA,YAEhD,CAAC;AAAA,cACC;AAAA,cACA;AAAA,cACA;AAAA,YACF,EAAE,SAAS,EAAE,EAAE;AAAA,UACnB;AAAA,QACF;AAEA,aAAK,SAAS;AACd,aAAK,kBAAkB,KAAK,IAAI;AAEhC,eAAO;AAAA,MACT,CAAC;AAAA,IACL,CAAC,EACA,MAAM,MAAM,MAAS;AAGxB,UAAM,KAAK;AAAA,EACb;AACF;","names":[]}

package/dist/chunk-QY5MPNNB.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../src/helpers/define-runner.ts"],"sourcesContent":["import {\n  IdGenerator,\n  InferRunConfig,\n  ProviderCtor,\n  Runner,\n  ScorerCtor,\n} from \"@/types\";\nimport { idGeneratorUUIDv7 } from \"@/utils\";\nimport z from \"zod\";\n\nexport function defineRunner<\n  const TProviders extends ProviderCtor[],\n  const TScorers extends ScorerCtor[],\n  const TSchemaSets extends SchemaSetDefinition[],\n  const TRunConfigSchema extends z.ZodRawShape = {},\n>(\n  config: {\n    schemaSets: TSchemaSets;\n    providers: TProviders;\n    scorers: TScorers;\n    runConfigSchema?: TRunConfigSchema;\n\n    /**\n     * @default true\n     */\n    parseRunConfig?: boolean;\n    defaults?: {\n      scorer?: InstanceType<TScorers[number]>;\n      responseIdGenerator?: IdGenerator;\n      scoreIdGenerator?: IdGenerator;\n    };\n  },\n  fn: Runner<\n    TSchemaSets[number][\"testCase\"],\n    TSchemaSets[number][\"response\"],\n    TSchemaSets[number][\"score\"],\n    InstanceType<TProviders[number]>,\n    InstanceType<TScorers[number]>,\n    InferRunConfig<TRunConfigSchema>\n  >\n) {\n  return async (params: Parameters<typeof fn>[0]) => {\n    if (config.runConfigSchema && config.parseRunConfig !== false) {\n      z.object(config.runConfigSchema).parse(params.runConfig);\n    }\n\n    if (params.idGenerators && !params.idGenerators.response) {\n      params.idGenerators.response =\n        config.defaults?.responseIdGenerator ?? idGeneratorUUIDv7;\n    }\n\n    if (params.idGenerators && !params.idGenerators.score) {\n      params.idGenerators.score =\n        config.defaults?.scoreIdGenerator ?? idGeneratorUUIDv7;\n    }\n\n    if (params.scorer === undefined) {\n      params.scorer = config.defaults?.scorer ?? undefined;\n    }\n\n    return await fn(params);\n  };\n}\n\ntype SchemaSetDefinition<\n  TTestCase extends z.ZodObject = z.ZodObject,\n  TResponse extends z.ZodObject = z.ZodObject,\n  TScore extends z.ZodObject = z.ZodObject,\n> = {\n  testCase: TTestCase;\n  response: TResponse;\n  score: TScore;\n};\n"],"mappings":";;;;;AAQA,OAAO,OAAO;AAEP,SAAS,aAMd,QAgBA,IAQA;AACA,SAAO,OAAO,WAAqC;AACjD,QAAI,OAAO,mBAAmB,OAAO,mBAAmB,OAAO;AAC7D,QAAE,OAAO,OAAO,eAAe,EAAE,MAAM,OAAO,SAAS;AAAA,IACzD;AAEA,QAAI,OAAO,gBAAgB,CAAC,OAAO,aAAa,UAAU;AACxD,aAAO,aAAa,WAClB,OAAO,UAAU,uBAAuB;AAAA,IAC5C;AAEA,QAAI,OAAO,gBAAgB,CAAC,OAAO,aAAa,OAAO;AACrD,aAAO,aAAa,QAClB,OAAO,UAAU,oBAAoB;AAAA,IACzC;AAEA,QAAI,OAAO,WAAW,QAAW;AAC/B,aAAO,SAAS,OAAO,UAAU,UAAU;AAAA,IAC7C;AAEA,WAAO,MAAM,GAAG,MAAM;AAAA,EACxB;AACF;","names":[]}