peerbench 0.0.1 → 0.0.2-alpha-dev.0

package/dist/index.d.ts

@@ -1,3864 +1,82 @@
-import { z, TypeOf } from 'zod';
-import { XMLParser } from 'fast-xml-parser';
-import { ChatCompletionMessageParam } from 'openai/resources/chat/completions';
-import OpenAI from 'openai';
-import { ChatCompletionMessageParam as ChatCompletionMessageParam$1 } from 'openai/resources/chat';
-import { Account } from 'viem';
-import { CID } from 'multiformats/cid';
+import { I as IdGenerator, P as ProviderCtor, S as ScorerCtor } from './index-BAioQhp2.js';
+export { A as AbstractClassConstructor, C as ClassConstructor, b as Id, a as InferExtension, M as MaybePromise, c as ScoringMethod, W as WidenZodObject } from './index-BAioQhp2.js';
+export { R as RateLimiter, b as RateLimiterCallOptions, a as RateLimiterOptions } from './rate-limiter-CSmVIRsM.js';
+import z__default from 'zod';
+import { A as AbstractProvider } from './provider-BDjGp2y-.js';
+import { A as AbstractScorer } from './abstract-Dec9Sc5O.js';
 
-declare const PromptTypes: {
-    readonly MultipleChoice: "multiple-choice";
-    readonly OrderSentences: "order-sentences";
-    readonly TextReplacement: "text-replacement";
-    readonly Typo: "typo";
-    readonly OpenEnded: "open-ended";
-    readonly OpenEndedWithDocs: "open-ended-with-docs";
-};
-type PromptType = (typeof PromptTypes)[keyof typeof PromptTypes];
-declare const ScoringMethods: {
-    /**
-     * Scored by a human
-     */
-    readonly human: "human";
-    /**
-     * Scored using an AI model
-     */
-    readonly ai: "ai";
-    /**
-     * Scored using an algorithm
-     */
-    readonly algo: "algo";
-};
-type ScoringMethod = (typeof ScoringMethods)[keyof typeof ScoringMethods];
-declare const PromptSchema: z.ZodEffects<z.ZodObject<{
-    /**
-     * Decentralized identifier of the Prompt
-     */
-    did: z.ZodPipeline<z.ZodEffects<z.ZodString, string, string>, z.ZodType<string, z.ZodTypeDef, string>>;
-    /**
-     * The question that is going to be asked to the model
-     */
-    question: z.ZodObject<{
-        /**
-         * Question data itself
-         */
-        data: z.ZodString;
-        /**
-         * CID v1 calculation of the question string
-         */
-        cid: z.ZodString;
-        /**
-         * SHA256 hash of the question string
-         */
-        sha256: z.ZodString;
-    }, "strip", z.ZodTypeAny, {
-        sha256: string;
-        data: string;
-        cid: string;
-    }, {
-        sha256: string;
-        data: string;
-        cid: string;
-    }>;
-    /**
-     * Multiple choice answers for the question where the keys are letters and the values are the answers.
-     */
-    options: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
-    /**
-     * Full prompt that is going to be sent to the model
-     */
-    fullPrompt: z.ZodObject<{
-        /**
-         * Full prompt itself
-         */
-        data: z.ZodString;
-        /**
-         * CID v1 calculation of the full prompt string
-         */
-        cid: z.ZodString;
-        /**
-         * SHA256 hash of the full prompt string
-         */
-        sha256: z.ZodString;
-    }, "strip", z.ZodTypeAny, {
-        sha256: string;
-        data: string;
-        cid: string;
-    }, {
-        sha256: string;
-        data: string;
-        cid: string;
-    }>;
-    /**
-     * Type of the Prompt
-     */
-    type: z.ZodNativeEnum<{
-        readonly MultipleChoice: "multiple-choice";
-        readonly OrderSentences: "order-sentences";
-        readonly TextReplacement: "text-replacement";
-        readonly Typo: "typo";
-        readonly OpenEnded: "open-ended";
-        readonly OpenEndedWithDocs: "open-ended-with-docs";
-    }>;
-    /**
-     * Expected option value for the question
-     */
-    answer: z.ZodOptional<z.ZodString>;
-    /**
-     * Expected letter of the answer (e.g "A", "B" or "C")
-     */
-    answerKey: z.ZodOptional<z.ZodString>;
-    /**
-     * Additional metadata related to the Prompt
-     */
-    metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodAny>>;
-    /**
-     * Expected Scorer identifiers that can be used to
-     * score the Responses for this Prompt
-     */
-    scorers: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
-}, "strip", z.ZodTypeAny, {
-    type: "multiple-choice" | "order-sentences" | "text-replacement" | "typo" | "open-ended" | "open-ended-with-docs";
-    did: string;
-    question: {
-        sha256: string;
-        data: string;
-        cid: string;
-    };
-    fullPrompt: {
-        sha256: string;
-        data: string;
-        cid: string;
-    };
-    answer?: string | undefined;
-    metadata?: Record<string, any> | undefined;
-    options?: Record<string, string> | undefined;
-    answerKey?: string | undefined;
-    scorers?: string[] | undefined;
-}, {
-    type: "multiple-choice" | "order-sentences" | "text-replacement" | "typo" | "open-ended" | "open-ended-with-docs";
-    did: string;
-    question: {
-        sha256: string;
-        data: string;
-        cid: string;
-    };
-    fullPrompt: {
-        sha256: string;
-        data: string;
-        cid: string;
-    };
-    answer?: string | undefined;
-    metadata?: Record<string, any> | undefined;
-    options?: Record<string, string> | undefined;
-    answerKey?: string | undefined;
-    scorers?: string[] | undefined;
-}>, {
-    type: "multiple-choice" | "order-sentences" | "text-replacement" | "typo" | "open-ended" | "open-ended-with-docs";
-    did: string;
-    question: {
-        sha256: string;
-        data: string;
-        cid: string;
-    };
-    fullPrompt: {
-        sha256: string;
-        data: string;
-        cid: string;
-    };
-    answer?: string | undefined;
-    metadata?: Record<string, any> | undefined;
-    options?: Record<string, string> | undefined;
-    answerKey?: string | undefined;
-    scorers?: string[] | undefined;
-}, {
-    type: "multiple-choice" | "order-sentences" | "text-replacement" | "typo" | "open-ended" | "open-ended-with-docs";
-    did: string;
-    question: {
-        sha256: string;
-        data: string;
-        cid: string;
-    };
-    fullPrompt: {
-        sha256: string;
-        data: string;
-        cid: string;
-    };
-    answer?: string | undefined;
-    metadata?: Record<string, any> | undefined;
-    options?: Record<string, string> | undefined;
-    answerKey?: string | undefined;
-    scorers?: string[] | undefined;
-}>;
-/**
- * PeerBench Prompt object
- */
-type Prompt = z.infer<typeof PromptSchema>;
-declare const TaskSchema: z.ZodObject<{
-    /**
-     * Decentralized identifier of the Task
-     */
-    did: z.ZodString;
-    /**
-     * The Prompts that the Task has
-     */
-    prompts: z.ZodArray<z.ZodEffects<z.ZodObject<{
-        /**
-         * Decentralized identifier of the Prompt
-         */
-        did: z.ZodPipeline<z.ZodEffects<z.ZodString, string, string>, z.ZodType<string, z.ZodTypeDef, string>>;
-        /**
-         * The question that is going to be asked to the model
-         */
-        question: z.ZodObject<{
-            /**
-             * Question data itself
-             */
-            data: z.ZodString;
-            /**
-             * CID v1 calculation of the question string
-             */
-            cid: z.ZodString;
-            /**
-             * SHA256 hash of the question string
-             */
-            sha256: z.ZodString;
-        }, "strip", z.ZodTypeAny, {
-            sha256: string;
-            data: string;
-            cid: string;
-        }, {
-            sha256: string;
-            data: string;
-            cid: string;
-        }>;
-        /**
-         * Multiple choice answers for the question where the keys are letters and the values are the answers.
-         */
-        options: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
-        /**
-         * Full prompt that is going to be sent to the model
-         */
-        fullPrompt: z.ZodObject<{
-            /**
-             * Full prompt itself
-             */
-            data: z.ZodString;
-            /**
-             * CID v1 calculation of the full prompt string
-             */
-            cid: z.ZodString;
-            /**
-             * SHA256 hash of the full prompt string
-             */
-            sha256: z.ZodString;
-        }, "strip", z.ZodTypeAny, {
-            sha256: string;
-            data: string;
-            cid: string;
-        }, {
-            sha256: string;
-            data: string;
-            cid: string;
-        }>;
-        /**
-         * Type of the Prompt
-         */
-        type: z.ZodNativeEnum<{
-            readonly MultipleChoice: "multiple-choice";
-            readonly OrderSentences: "order-sentences";
-            readonly TextReplacement: "text-replacement";
-            readonly Typo: "typo";
-            readonly OpenEnded: "open-ended";
-            readonly OpenEndedWithDocs: "open-ended-with-docs";
-        }>;
-        /**
-         * Expected option value for the question
-         */
-        answer: z.ZodOptional<z.ZodString>;
-        /**
-         * Expected letter of the answer (e.g "A", "B" or "C")
-         */
-        answerKey: z.ZodOptional<z.ZodString>;
-        /**
-         * Additional metadata related to the Prompt
-         */
-        metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodAny>>;
-        /**
-         * Expected Scorer identifiers that can be used to
-         * score the Responses for this Prompt
-         */
-        scorers: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
-    }, "strip", z.ZodTypeAny, {
-        type: "multiple-choice" | "order-sentences" | "text-replacement" | "typo" | "open-ended" | "open-ended-with-docs";
-        did: string;
-        question: {
-            sha256: string;
-            data: string;
-            cid: string;
-        };
-        fullPrompt: {
-            sha256: string;
-            data: string;
-            cid: string;
-        };
-        answer?: string | undefined;
-        metadata?: Record<string, any> | undefined;
-        options?: Record<string, string> | undefined;
-        answerKey?: string | undefined;
-        scorers?: string[] | undefined;
-    }, {
-        type: "multiple-choice" | "order-sentences" | "text-replacement" | "typo" | "open-ended" | "open-ended-with-docs";
-        did: string;
-        question: {
-            sha256: string;
-            data: string;
-            cid: string;
-        };
-        fullPrompt: {
-            sha256: string;
-            data: string;
-            cid: string;
-        };
-        answer?: string | undefined;
-        metadata?: Record<string, any> | undefined;
-        options?: Record<string, string> | undefined;
-        answerKey?: string | undefined;
-        scorers?: string[] | undefined;
-    }>, {
-        type: "multiple-choice" | "order-sentences" | "text-replacement" | "typo" | "open-ended" | "open-ended-with-docs";
-        did: string;
-        question: {
-            sha256: string;
-            data: string;
-            cid: string;
-        };
-        fullPrompt: {
-            sha256: string;
-            data: string;
-            cid: string;
-        };
-        answer?: string | undefined;
-        metadata?: Record<string, any> | undefined;
-        options?: Record<string, string> | undefined;
-        answerKey?: string | undefined;
-        scorers?: string[] | undefined;
-    }, {
-        type: "multiple-choice" | "order-sentences" | "text-replacement" | "typo" | "open-ended" | "open-ended-with-docs";
-        did: string;
-        question: {
-            sha256: string;
-            data: string;
-            cid: string;
-        };
-        fullPrompt: {
-            sha256: string;
-            data: string;
-            cid: string;
-        };
-        answer?: string | undefined;
-        metadata?: Record<string, any> | undefined;
-        options?: Record<string, string> | undefined;
-        answerKey?: string | undefined;
-        scorers?: string[] | undefined;
-    }>, "many">;
-    /**
-     * CID v1 calculation of the Task file
-     */
-    cid: z.ZodString;
-    /**
-     * SHA256 calculation of the Task file
-     */
-    sha256: z.ZodString;
-    /**
-     * Basename of the Task file
-     */
-    fileName: z.ZodString;
-    /**
-     * Full path of the Task file
-     */
-    path: z.ZodString;
-}, "strip", z.ZodTypeAny, {
-    sha256: string;
-    path: string;
-    did: string;
-    cid: string;
-    prompts: {
-        type: "multiple-choice" | "order-sentences" | "text-replacement" | "typo" | "open-ended" | "open-ended-with-docs";
-        did: string;
-        question: {
-            sha256: string;
-            data: string;
-            cid: string;
-        };
-        fullPrompt: {
-            sha256: string;
-            data: string;
-            cid: string;
-        };
-        answer?: string | undefined;
-        metadata?: Record<string, any> | undefined;
-        options?: Record<string, string> | undefined;
-        answerKey?: string | undefined;
-        scorers?: string[] | undefined;
-    }[];
-    fileName: string;
-}, {
-    sha256: string;
-    path: string;
-    did: string;
-    cid: string;
-    prompts: {
-        type: "multiple-choice" | "order-sentences" | "text-replacement" | "typo" | "open-ended" | "open-ended-with-docs";
-        did: string;
-        question: {
-            sha256: string;
-            data: string;
-            cid: string;
-        };
-        fullPrompt: {
-            sha256: string;
-            data: string;
-            cid: string;
-        };
-        answer?: string | undefined;
-        metadata?: Record<string, any> | undefined;
-        options?: Record<string, string> | undefined;
-        answerKey?: string | undefined;
-        scorers?: string[] | undefined;
-    }[];
-    fileName: string;
-}>;
-/**
- * Task object that includes the prompts and the Task file metadata
- */
-type Task = z.infer<typeof TaskSchema>;
-declare const PromptResponseSchema: z.ZodObject<{
-    /**
-     * Unique identifier of the Response
-     */
-    did: z.ZodPipeline<z.ZodEffects<z.ZodString, string, string>, z.ZodType<string, z.ZodTypeDef, string>>;
-    /**
-     * Name of the Provider that the Response comes from
-     */
-    provider: z.ZodString;
-    /**
-     * ID of the Model that was used by the Provider
-     */
-    modelId: z.ZodString;
-    /**
-     * Known name of the model by peerBench
-     */
-    modelName: z.ZodString;
-    /**
-     * Owner of the model
-     */
-    modelOwner: z.ZodString;
-    /**
-     * The entity that responsible for hosting the model
-     */
-    modelHost: z.ZodString;
-    /**
-     * The Prompt that used to achieve this Response.
-     */
-    prompt: z.ZodEffects<z.ZodObject<{
-        /**
-         * Decentralized identifier of the Prompt
-         */
-        did: z.ZodPipeline<z.ZodEffects<z.ZodString, string, string>, z.ZodType<string, z.ZodTypeDef, string>>;
-        /**
-         * The question that is going to be asked to the model
-         */
-        question: z.ZodObject<{
-            /**
-             * Question data itself
-             */
-            data: z.ZodString;
-            /**
-             * CID v1 calculation of the question string
-             */
-            cid: z.ZodString;
-            /**
-             * SHA256 hash of the question string
-             */
-            sha256: z.ZodString;
-        }, "strip", z.ZodTypeAny, {
-            sha256: string;
-            data: string;
-            cid: string;
-        }, {
-            sha256: string;
-            data: string;
-            cid: string;
-        }>;
-        /**
-         * Multiple choice answers for the question where the keys are letters and the values are the answers.
-         */
-        options: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
-        /**
-         * Full prompt that is going to be sent to the model
-         */
-        fullPrompt: z.ZodObject<{
-            /**
-             * Full prompt itself
-             */
-            data: z.ZodString;
-            /**
-             * CID v1 calculation of the full prompt string
-             */
-            cid: z.ZodString;
-            /**
-             * SHA256 hash of the full prompt string
-             */
-            sha256: z.ZodString;
-        }, "strip", z.ZodTypeAny, {
-            sha256: string;
-            data: string;
-            cid: string;
-        }, {
-            sha256: string;
-            data: string;
-            cid: string;
-        }>;
-        /**
-         * Type of the Prompt
-         */
-        type: z.ZodNativeEnum<{
-            readonly MultipleChoice: "multiple-choice";
-            readonly OrderSentences: "order-sentences";
-            readonly TextReplacement: "text-replacement";
-            readonly Typo: "typo";
-            readonly OpenEnded: "open-ended";
-            readonly OpenEndedWithDocs: "open-ended-with-docs";
-        }>;
-        /**
-         * Expected option value for the question
-         */
-        answer: z.ZodOptional<z.ZodString>;
-        /**
-         * Expected letter of the answer (e.g "A", "B" or "C")
-         */
-        answerKey: z.ZodOptional<z.ZodString>;
-        /**
-         * Additional metadata related to the Prompt
-         */
-        metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodAny>>;
-        /**
-         * Expected Scorer identifiers that can be used to
-         * score the Responses for this Prompt
-         */
-        scorers: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
-    }, "strip", z.ZodTypeAny, {
-        type: "multiple-choice" | "order-sentences" | "text-replacement" | "typo" | "open-ended" | "open-ended-with-docs";
-        did: string;
-        question: {
-            sha256: string;
-            data: string;
-            cid: string;
-        };
-        fullPrompt: {
-            sha256: string;
-            data: string;
-            cid: string;
-        };
-        answer?: string | undefined;
-        metadata?: Record<string, any> | undefined;
-        options?: Record<string, string> | undefined;
-        answerKey?: string | undefined;
-        scorers?: string[] | undefined;
-    }, {
-        type: "multiple-choice" | "order-sentences" | "text-replacement" | "typo" | "open-ended" | "open-ended-with-docs";
-        did: string;
-        question: {
-            sha256: string;
-            data: string;
-            cid: string;
-        };
-        fullPrompt: {
-            sha256: string;
-            data: string;
-            cid: string;
-        };
-        answer?: string | undefined;
-        metadata?: Record<string, any> | undefined;
-        options?: Record<string, string> | undefined;
-        answerKey?: string | undefined;
-        scorers?: string[] | undefined;
-    }>, {
-        type: "multiple-choice" | "order-sentences" | "text-replacement" | "typo" | "open-ended" | "open-ended-with-docs";
-        did: string;
-        question: {
-            sha256: string;
-            data: string;
-            cid: string;
-        };
-        fullPrompt: {
-            sha256: string;
-            data: string;
-            cid: string;
-        };
-        answer?: string | undefined;
-        metadata?: Record<string, any> | undefined;
-        options?: Record<string, string> | undefined;
-        answerKey?: string | undefined;
-        scorers?: string[] | undefined;
-    }, {
-        type: "multiple-choice" | "order-sentences" | "text-replacement" | "typo" | "open-ended" | "open-ended-with-docs";
-        did: string;
-        question: {
-            sha256: string;
-            data: string;
-            cid: string;
-        };
-        fullPrompt: {
-            sha256: string;
-            data: string;
-            cid: string;
-        };
-        answer?: string | undefined;
-        metadata?: Record<string, any> | undefined;
-        options?: Record<string, string> | undefined;
-        answerKey?: string | undefined;
-        scorers?: string[] | undefined;
-    }>;
-    /**
-     * CID v1 calculation of the Response data.
-     */
-    cid: z.ZodString;
-    /**
-     * SHA256 calculation of the Response data.
-     */
-    sha256: z.ZodString;
-    /**
-     * Response data itself.
-     */
-    data: z.ZodString;
-    /**
-     * Timestamp when the Prompt sent to the Model
-     */
-    startedAt: z.ZodNumber;
-    /**
-     * Timestamp when the Model responded this particular Prompt.
-     */
-    finishedAt: z.ZodNumber;
-    /**
-     * Unique identifier of which run this Response belongs to
-     */
-    runId: z.ZodString;
-    inputTokensUsed: z.ZodOptional<z.ZodNumber>;
-    outputTokensUsed: z.ZodOptional<z.ZodNumber>;
-    inputCost: z.ZodOptional<z.ZodString>;
-    outputCost: z.ZodOptional<z.ZodString>;
-    metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodAny>>;
-}, "strip", z.ZodTypeAny, {
-    sha256: string;
-    startedAt: number;
-    data: string;
-    modelName: string;
-    did: string;
-    cid: string;
-    provider: string;
-    modelId: string;
-    modelOwner: string;
-    modelHost: string;
-    prompt: {
-        type: "multiple-choice" | "order-sentences" | "text-replacement" | "typo" | "open-ended" | "open-ended-with-docs";
-        did: string;
-        question: {
-            sha256: string;
-            data: string;
-            cid: string;
-        };
-        fullPrompt: {
-            sha256: string;
-            data: string;
-            cid: string;
-        };
-        answer?: string | undefined;
-        metadata?: Record<string, any> | undefined;
-        options?: Record<string, string> | undefined;
-        answerKey?: string | undefined;
-        scorers?: string[] | undefined;
-    };
-    finishedAt: number;
-    runId: string;
-    metadata?: Record<string, any> | undefined;
-    inputTokensUsed?: number | undefined;
-    outputTokensUsed?: number | undefined;
-    inputCost?: string | undefined;
-    outputCost?: string | undefined;
-}, {
-    sha256: string;
-    startedAt: number;
-    data: string;
-    modelName: string;
-    did: string;
-    cid: string;
-    provider: string;
-    modelId: string;
-    modelOwner: string;
-    modelHost: string;
-    prompt: {
-        type: "multiple-choice" | "order-sentences" | "text-replacement" | "typo" | "open-ended" | "open-ended-with-docs";
-        did: string;
-        question: {
-            sha256: string;
-            data: string;
-            cid: string;
-        };
-        fullPrompt: {
-            sha256: string;
-            data: string;
-            cid: string;
-        };
-        answer?: string | undefined;
-        metadata?: Record<string, any> | undefined;
-        options?: Record<string, string> | undefined;
-        answerKey?: string | undefined;
-        scorers?: string[] | undefined;
-    };
-    finishedAt: number;
-    runId: string;
-    metadata?: Record<string, any> | undefined;
-    inputTokensUsed?: number | undefined;
-    outputTokensUsed?: number | undefined;
-    inputCost?: string | undefined;
-    outputCost?: string | undefined;
-}>;
-type PromptResponse = z.infer<typeof PromptResponseSchema>;
-declare const PromptScoreSchema: z.ZodObject<{
-    did: z.ZodPipeline<z.ZodEffects<z.ZodString, string, string>, z.ZodType<string, z.ZodTypeDef, string>>;
-    provider: z.ZodString;
-    modelId: z.ZodString;
-    modelName: z.ZodString;
-    modelOwner: z.ZodString;
-    modelHost: z.ZodString;
-    cid: z.ZodString;
-    sha256: z.ZodString;
-    startedAt: z.ZodNumber;
-    finishedAt: z.ZodNumber;
-    runId: z.ZodString;
-    inputTokensUsed: z.ZodOptional<z.ZodNumber>;
-    outputTokensUsed: z.ZodOptional<z.ZodNumber>;
-    inputCost: z.ZodOptional<z.ZodString>;
-    outputCost: z.ZodOptional<z.ZodString>;
-    metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodAny>>;
-} & {
-    prompt: z.ZodOptional<z.ZodObject<{
-        did: z.ZodPipeline<z.ZodEffects<z.ZodString, string, string>, z.ZodType<string, z.ZodTypeDef, string>>;
-        metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodAny>>;
-        scorers: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
-    } & {
-        options: z.ZodOptional<z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>>;
-        question: z.ZodObject<{
-            cid: z.ZodString;
-            sha256: z.ZodString;
-        } & {
-            data: z.ZodOptional<z.ZodString>;
-        }, "strip", z.ZodTypeAny, {
-            sha256: string;
-            cid: string;
-            data?: string | undefined;
-        }, {
-            sha256: string;
-            cid: string;
-            data?: string | undefined;
-        }>;
-        fullPrompt: z.ZodObject<{
-            cid: z.ZodString;
-            sha256: z.ZodString;
-        } & {
-            data: z.ZodOptional<z.ZodString>;
-        }, "strip", z.ZodTypeAny, {
-            sha256: string;
-            cid: string;
-            data?: string | undefined;
-        }, {
-            sha256: string;
-            cid: string;
-            data?: string | undefined;
-        }>;
-        type: z.ZodOptional<z.ZodNativeEnum<{
-            readonly MultipleChoice: "multiple-choice";
-            readonly OrderSentences: "order-sentences";
-            readonly TextReplacement: "text-replacement";
-            readonly Typo: "typo";
-            readonly OpenEnded: "open-ended";
-            readonly OpenEndedWithDocs: "open-ended-with-docs";
-        }>>;
-        answer: z.ZodOptional<z.ZodOptional<z.ZodString>>;
-        answerKey: z.ZodOptional<z.ZodOptional<z.ZodString>>;
-    }, "strip", z.ZodTypeAny, {
-        did: string;
-        question: {
-            sha256: string;
-            cid: string;
-            data?: string | undefined;
-        };
-        fullPrompt: {
-            sha256: string;
-            cid: string;
-            data?: string | undefined;
-        };
-        answer?: string | undefined;
-        metadata?: Record<string, any> | undefined;
-        options?: Record<string, string> | undefined;
-        type?: "multiple-choice" | "order-sentences" | "text-replacement" | "typo" | "open-ended" | "open-ended-with-docs" | undefined;
-        answerKey?: string | undefined;
-        scorers?: string[] | undefined;
-    }, {
-        did: string;
-        question: {
-            sha256: string;
-            cid: string;
-            data?: string | undefined;
-        };
-        fullPrompt: {
-            sha256: string;
-            cid: string;
-            data?: string | undefined;
-        };
-        answer?: string | undefined;
-        metadata?: Record<string, any> | undefined;
-        options?: Record<string, string> | undefined;
-        type?: "multiple-choice" | "order-sentences" | "text-replacement" | "typo" | "open-ended" | "open-ended-with-docs" | undefined;
-        answerKey?: string | undefined;
-        scorers?: string[] | undefined;
-    }>>;
-    data: z.ZodOptional<z.ZodString>;
-    scoreDID: z.ZodPipeline<z.ZodEffects<z.ZodString, string, string>, z.ZodType<string, z.ZodTypeDef, string>>;
-    scoreMetadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodAny>>;
-    score: z.ZodNumber;
-    method: z.ZodNativeEnum<{
-        /**
-         * Scored by a human
-         */
-        readonly human: "human";
-        /**
-         * Scored using an AI model
-         */
-        readonly ai: "ai";
-        /**
-         * Scored using an algorithm
-         */
-        readonly algo: "algo";
-    }>;
-    explanation: z.ZodOptional<z.ZodString>;
-    scorerAI: z.ZodOptional<z.ZodObject<{
-        provider: z.ZodString;
-        modelName: z.ZodString;
-        modelHost: z.ZodString;
-        modelOwner: z.ZodString;
-        modelId: z.ZodString;
-        inputTokensUsed: z.ZodOptional<z.ZodNumber>;
-        outputTokensUsed: z.ZodOptional<z.ZodNumber>;
-        inputCost: z.ZodOptional<z.ZodString>;
-        outputCost: z.ZodOptional<z.ZodString>;
-    }, "strip", z.ZodTypeAny, {
-        modelName: string;
-        provider: string;
-        modelId: string;
-        modelOwner: string;
-        modelHost: string;
-        inputTokensUsed?: number | undefined;
-        outputTokensUsed?: number | undefined;
-        inputCost?: string | undefined;
-        outputCost?: string | undefined;
-    }, {
-        modelName: string;
-        provider: string;
-        modelId: string;
-        modelOwner: string;
-        modelHost: string;
-        inputTokensUsed?: number | undefined;
-        outputTokensUsed?: number | undefined;
-        inputCost?: string | undefined;
-        outputCost?: string | undefined;
-    }>>;
-}, "strip", z.ZodTypeAny, {
-    sha256: string;
-    startedAt: number;
-    modelName: string;
-    did: string;
-    cid: string;
-    provider: string;
-    modelId: string;
-    modelOwner: string;
-    modelHost: string;
-    finishedAt: number;
-    runId: string;
-    scoreDID: string;
-    score: number;
-    method: "human" | "ai" | "algo";
-    metadata?: Record<string, any> | undefined;
-    inputTokensUsed?: number | undefined;
-    outputTokensUsed?: number | undefined;
-    data?: string | undefined;
-    inputCost?: string | undefined;
-    outputCost?: string | undefined;
-    prompt?: {
-        did: string;
-        question: {
-            sha256: string;
-            cid: string;
-            data?: string | undefined;
-        };
-        fullPrompt: {
-            sha256: string;
-            cid: string;
-            data?: string | undefined;
-        };
-        answer?: string | undefined;
-        metadata?: Record<string, any> | undefined;
-        options?: Record<string, string> | undefined;
-        type?: "multiple-choice" | "order-sentences" | "text-replacement" | "typo" | "open-ended" | "open-ended-with-docs" | undefined;
-        answerKey?: string | undefined;
-        scorers?: string[] | undefined;
-    } | undefined;
-    scoreMetadata?: Record<string, any> | undefined;
-    explanation?: string | undefined;
-    scorerAI?: {
-        modelName: string;
-        provider: string;
-        modelId: string;
-        modelOwner: string;
-        modelHost: string;
-        inputTokensUsed?: number | undefined;
-        outputTokensUsed?: number | undefined;
-        inputCost?: string | undefined;
-        outputCost?: string | undefined;
-    } | undefined;
-}, {
-    sha256: string;
-    startedAt: number;
-    modelName: string;
-    did: string;
-    cid: string;
-    provider: string;
-    modelId: string;
-    modelOwner: string;
-    modelHost: string;
-    finishedAt: number;
-    runId: string;
-    scoreDID: string;
-    score: number;
-    method: "human" | "ai" | "algo";
-    metadata?: Record<string, any> | undefined;
-    inputTokensUsed?: number | undefined;
-    outputTokensUsed?: number | undefined;
-    data?: string | undefined;
-    inputCost?: string | undefined;
-    outputCost?: string | undefined;
-    prompt?: {
-        did: string;
-        question: {
-            sha256: string;
-            cid: string;
-            data?: string | undefined;
-        };
-        fullPrompt: {
-            sha256: string;
-            cid: string;
-            data?: string | undefined;
-        };
-        answer?: string | undefined;
-        metadata?: Record<string, any> | undefined;
-        options?: Record<string, string> | undefined;
-        type?: "multiple-choice" | "order-sentences" | "text-replacement" | "typo" | "open-ended" | "open-ended-with-docs" | undefined;
-        answerKey?: string | undefined;
-        scorers?: string[] | undefined;
-    } | undefined;
-    scoreMetadata?: Record<string, any> | undefined;
-    explanation?: string | undefined;
-    scorerAI?: {
-        modelName: string;
-        provider: string;
-        modelId: string;
-        modelOwner: string;
-        modelHost: string;
-        inputTokensUsed?: number | undefined;
-        outputTokensUsed?: number | undefined;
-        inputCost?: string | undefined;
-        outputCost?: string | undefined;
-    } | undefined;
-}>;
-type PromptScore = z.infer<typeof PromptScoreSchema>;
-type MaybePromise<T> = T | Promise<T>;
-type PromptOptions = Record<string, string>;
-
-declare abstract class AbstractCollector<TOutput> {
-    /**
-     * String identifier of the collector.
-     */
-    abstract readonly identifier: string;
-    /**
-     * Collects data from a source
-     * @param sourceUrl The URL or identifier of the source
-     * @param options Additional options for collection (optional)
-     * @returns Promise resolving to the collected data or null if failed
-     */
-    abstract collect(source: unknown, options?: Record<string, any>): Promise<TOutput | undefined>;
-    /**
-     * Initializes the collector (depends on the implementation)
-     */
-    initialize(...args: any[]): Promise<void>;
-}
-
-declare abstract class AbstractRSSCollector<TOutput> extends AbstractCollector<TOutput> {
-    protected parser: XMLParser;
-    abstract feedSchema: z.ZodSchema<unknown>;
-    constructor();
-    /**
-     * Helper function to fetch the feed from the URL
-     * @param url The URL of the feed
-     * @returns Raw XML string of the feed
-     */
-    protected fetchFeed(url: string): Promise<string>;
-    /**
-     * Parses the raw XML string and validates it
-     * against the defined `FeedStructure` schema
-     * @param xml The raw XML string
-     * @returns Structured data of the RSS feed
-     */
-    protected parseFeedXML(xml: string): Promise<z.infer<(typeof this)["feedSchema"]>>;
-}
+declare function sleep(ms: number, signal?: AbortSignal): Promise<void>;
 
 /**
- * TODO: Write a description for this collector
- */
-declare class PubMedCollector extends AbstractRSSCollector<PubMedCollectedData> {
-    readonly identifier = "pubmed";
-    feedSchema: z.ZodObject<{
-        rss: z.ZodObject<{
-            channel: z.ZodObject<{
-                title: z.ZodString;
-                link: z.ZodString;
-                description: z.ZodString;
-                pubDate: z.ZodString;
-                lastBuildDate: z.ZodOptional<z.ZodString>;
-                language: z.ZodOptional<z.ZodString>;
-                item: z.ZodPipeline<z.ZodEffects<z.ZodAny, any[], any>, z.ZodArray<z.ZodObject<{
-                    title: z.ZodString;
-                    link: z.ZodString;
-                    description: z.ZodString;
-                    guid: z.ZodString;
-                    pubDate: z.ZodString;
-                    "content:encoded": z.ZodString;
-                }, "strip", z.ZodTypeAny, {
-                    link: string;
-                    title: string;
-                    description: string;
-                    pubDate: string;
-                    guid: string;
-                    "content:encoded": string;
-                }, {
-                    link: string;
-                    title: string;
-                    description: string;
-                    pubDate: string;
-                    guid: string;
-                    "content:encoded": string;
-                }>, "many">>;
-            }, "strip", z.ZodTypeAny, {
-                link: string;
-                title: string;
-                description: string;
-                pubDate: string;
-                item: {
-                    link: string;
-                    title: string;
-                    description: string;
-                    pubDate: string;
-                    guid: string;
-                    "content:encoded": string;
-                }[];
-                lastBuildDate?: string | undefined;
-                language?: string | undefined;
-            }, {
-                link: string;
-                title: string;
-                description: string;
-                pubDate: string;
-                lastBuildDate?: string | undefined;
-                language?: string | undefined;
-                item?: any;
-            }>;
-        }, "strip", z.ZodTypeAny, {
-            channel: {
-                link: string;
-                title: string;
-                description: string;
-                pubDate: string;
-                item: {
-                    link: string;
-                    title: string;
-                    description: string;
-                    pubDate: string;
-                    guid: string;
-                    "content:encoded": string;
-                }[];
-                lastBuildDate?: string | undefined;
-                language?: string | undefined;
-            };
-        }, {
-            channel: {
-                link: string;
-                title: string;
-                description: string;
-                pubDate: string;
-                lastBuildDate?: string | undefined;
-                language?: string | undefined;
-                item?: any;
-            };
-        }>;
-    }, "strip", z.ZodTypeAny, {
-        rss: {
-            channel: {
-                link: string;
-                title: string;
-                description: string;
-                pubDate: string;
-                item: {
-                    link: string;
-                    title: string;
-                    description: string;
-                    pubDate: string;
-                    guid: string;
-                    "content:encoded": string;
-                }[];
-                lastBuildDate?: string | undefined;
-                language?: string | undefined;
-            };
-        };
-    }, {
-        rss: {
-            channel: {
-                link: string;
-                title: string;
-                description: string;
-                pubDate: string;
-                lastBuildDate?: string | undefined;
-                language?: string | undefined;
-                item?: any;
-            };
-        };
-    }>;
-    collect(url: string): Promise<{
-        pmid: string;
-        title: string;
-        paragraphs: Record<string, string>;
-        tags: string[];
-    }[]>;
-    /**
-     * Parses the title of a paragraph which
-     * is in the format of "TITLE:"
-     */
-    private parseParagraphTitle;
-}
-type PubMedCollectedData = {
-    pmid: string;
-    title: string;
-    paragraphs: Record<string, string>;
-    tags: string[];
-}[];
-
-/**
- * A general-purpose RSS collector that can handle heterogeneous RSS feeds.
- * Intelligently extracts and cleans text content while preserving raw XML data.
+ * Tries to repair and parse LLM response as a JSON object. LLM must
+ * be configured to return a JSON object. This function only helps to
+ * get rid out of some additional formatting (e.g. ```json) and repair
+ * the JSON syntax (e.g missing comma, single quotes instead double).
  */
-declare class SimpleGeneralRSSCollector extends AbstractRSSCollector<SimpleGeneralRSSCollectedData> {
-    readonly identifier = "simple-general-rss";
-    feedSchema: z.ZodObject<{
-        rss: z.ZodOptional<z.ZodObject<{
-            channel: z.ZodObject<{
-                title: z.ZodOptional<z.ZodString>;
-                link: z.ZodOptional<z.ZodString>;
-                description: z.ZodOptional<z.ZodString>;
-                pubDate: z.ZodOptional<z.ZodString>;
-                lastBuildDate: z.ZodOptional<z.ZodString>;
-                language: z.ZodOptional<z.ZodString>;
-                item: z.ZodPipeline<z.ZodEffects<z.ZodAny, any[], any>, z.ZodArray<z.ZodObject<{
-                    title: z.ZodString;
-                    link: z.ZodOptional<z.ZodString>;
-                    description: z.ZodOptional<z.ZodString>;
-                    guid: z.ZodOptional<z.ZodUnion<[z.ZodString, z.ZodObject<{}, "passthrough", z.ZodTypeAny, z.objectOutputType<{}, z.ZodTypeAny, "passthrough">, z.objectInputType<{}, z.ZodTypeAny, "passthrough">>]>>;
-                    pubDate: z.ZodOptional<z.ZodString>;
-                    "content:encoded": z.ZodOptional<z.ZodString>;
-                    "dc:creator": z.ZodOptional<z.ZodUnion<[z.ZodString, z.ZodArray<z.ZodString, "many">]>>;
-                    "dc:date": z.ZodOptional<z.ZodString>;
-                    "dc:title": z.ZodOptional<z.ZodString>;
-                    "dc:subject": z.ZodOptional<z.ZodString>;
-                    "dc:publisher": z.ZodOptional<z.ZodString>;
-                    "dc:identifier": z.ZodOptional<z.ZodUnion<[z.ZodString, z.ZodArray<z.ZodString, "many">]>>;
-                    "dc:rights": z.ZodOptional<z.ZodString>;
-                    "dc:source": z.ZodOptional<z.ZodString>;
-                    "prism:publicationDate": z.ZodOptional<z.ZodString>;
-                    "prism:section": z.ZodOptional<z.ZodString>;
-                    "prism:volume": z.ZodOptional<z.ZodString>;
-                    "prism:number": z.ZodOptional<z.ZodString>;
-                    "prism:startingPage": z.ZodOptional<z.ZodString>;
-                    "prism:endingPage": z.ZodOptional<z.ZodString>;
-                    "arxiv:announce_type": z.ZodOptional<z.ZodString>;
-                    category: z.ZodOptional<z.ZodUnion<[z.ZodString, z.ZodArray<z.ZodString, "many">]>>;
-                }, "strip", z.ZodTypeAny, {
-                    title: string;
-                    link?: string | undefined;
-                    description?: string | undefined;
-                    pubDate?: string | undefined;
-                    guid?: string | z.objectOutputType<{}, z.ZodTypeAny, "passthrough"> | undefined;
-                    "content:encoded"?: string | undefined;
-                    "dc:creator"?: string | string[] | undefined;
-                    "dc:date"?: string | undefined;
-                    "dc:title"?: string | undefined;
-                    "dc:subject"?: string | undefined;
-                    "dc:publisher"?: string | undefined;
-                    "dc:identifier"?: string | string[] | undefined;
-                    "dc:rights"?: string | undefined;
-                    "dc:source"?: string | undefined;
-                    "prism:publicationDate"?: string | undefined;
-                    "prism:section"?: string | undefined;
-                    "prism:volume"?: string | undefined;
-                    "prism:number"?: string | undefined;
-                    "prism:startingPage"?: string | undefined;
-                    "prism:endingPage"?: string | undefined;
-                    "arxiv:announce_type"?: string | undefined;
-                    category?: string | string[] | undefined;
-                }, {
-                    title: string;
-                    link?: string | undefined;
-                    description?: string | undefined;
-                    pubDate?: string | undefined;
-                    guid?: string | z.objectInputType<{}, z.ZodTypeAny, "passthrough"> | undefined;
-                    "content:encoded"?: string | undefined;
-                    "dc:creator"?: string | string[] | undefined;
-                    "dc:date"?: string | undefined;
-                    "dc:title"?: string | undefined;
-                    "dc:subject"?: string | undefined;
-                    "dc:publisher"?: string | undefined;
-                    "dc:identifier"?: string | string[] | undefined;
-                    "dc:rights"?: string | undefined;
-                    "dc:source"?: string | undefined;
-                    "prism:publicationDate"?: string | undefined;
-                    "prism:section"?: string | undefined;
-                    "prism:volume"?: string | undefined;
-                    "prism:number"?: string | undefined;
-                    "prism:startingPage"?: string | undefined;
-                    "prism:endingPage"?: string | undefined;
-                    "arxiv:announce_type"?: string | undefined;
-                    category?: string | string[] | undefined;
-                }>, "many">>;
-            }, "strip", z.ZodTypeAny, {
-                item: {
-                    title: string;
-                    link?: string | undefined;
-                    description?: string | undefined;
-                    pubDate?: string | undefined;
-                    guid?: string | z.objectOutputType<{}, z.ZodTypeAny, "passthrough"> | undefined;
-                    "content:encoded"?: string | undefined;
-                    "dc:creator"?: string | string[] | undefined;
-                    "dc:date"?: string | undefined;
-                    "dc:title"?: string | undefined;
-                    "dc:subject"?: string | undefined;
-                    "dc:publisher"?: string | undefined;
-                    "dc:identifier"?: string | string[] | undefined;
-                    "dc:rights"?: string | undefined;
-                    "dc:source"?: string | undefined;
-                    "prism:publicationDate"?: string | undefined;
-                    "prism:section"?: string | undefined;
-                    "prism:volume"?: string | undefined;
-                    "prism:number"?: string | undefined;
-                    "prism:startingPage"?: string | undefined;
-                    "prism:endingPage"?: string | undefined;
-                    "arxiv:announce_type"?: string | undefined;
-                    category?: string | string[] | undefined;
-                }[];
-                link?: string | undefined;
-                title?: string | undefined;
-                description?: string | undefined;
-                pubDate?: string | undefined;
-                lastBuildDate?: string | undefined;
-                language?: string | undefined;
-            }, {
-                link?: string | undefined;
-                title?: string | undefined;
-                description?: string | undefined;
-                pubDate?: string | undefined;
-                lastBuildDate?: string | undefined;
-                language?: string | undefined;
-                item?: any;
-            }>;
-        }, "strip", z.ZodTypeAny, {
-            channel: {
-                item: {
-                    title: string;
-                    link?: string | undefined;
-                    description?: string | undefined;
-                    pubDate?: string | undefined;
-                    guid?: string | z.objectOutputType<{}, z.ZodTypeAny, "passthrough"> | undefined;
-                    "content:encoded"?: string | undefined;
-                    "dc:creator"?: string | string[] | undefined;
-                    "dc:date"?: string | undefined;
-                    "dc:title"?: string | undefined;
-                    "dc:subject"?: string | undefined;
-                    "dc:publisher"?: string | undefined;
-                    "dc:identifier"?: string | string[] | undefined;
-                    "dc:rights"?: string | undefined;
-                    "dc:source"?: string | undefined;
-                    "prism:publicationDate"?: string | undefined;
-                    "prism:section"?: string | undefined;
-                    "prism:volume"?: string | undefined;
-                    "prism:number"?: string | undefined;
-                    "prism:startingPage"?: string | undefined;
-                    "prism:endingPage"?: string | undefined;
-                    "arxiv:announce_type"?: string | undefined;
-                    category?: string | string[] | undefined;
-                }[];
-                link?: string | undefined;
-                title?: string | undefined;
-                description?: string | undefined;
-                pubDate?: string | undefined;
-                lastBuildDate?: string | undefined;
-                language?: string | undefined;
-            };
-        }, {
-            channel: {
-                link?: string | undefined;
-                title?: string | undefined;
-                description?: string | undefined;
-                pubDate?: string | undefined;
-                lastBuildDate?: string | undefined;
-                language?: string | undefined;
-                item?: any;
-            };
-        }>>;
-        rdf: z.ZodOptional<z.ZodObject<{
-            channel: z.ZodOptional<z.ZodObject<{
-                title: z.ZodOptional<z.ZodString>;
-                link: z.ZodOptional<z.ZodString>;
-                description: z.ZodOptional<z.ZodString>;
-                items: z.ZodOptional<z.ZodObject<{
-                    "rdf:Seq": z.ZodOptional<z.ZodObject<{
-                        "rdf:li": z.ZodOptional<z.ZodAny>;
-                    }, "strip", z.ZodTypeAny, {
-                        "rdf:li"?: any;
-                    }, {
-                        "rdf:li"?: any;
-                    }>>;
-                }, "strip", z.ZodTypeAny, {
-                    "rdf:Seq"?: {
-                        "rdf:li"?: any;
-                    } | undefined;
-                }, {
-                    "rdf:Seq"?: {
-                        "rdf:li"?: any;
-                    } | undefined;
-                }>>;
-                item: z.ZodOptional<z.ZodPipeline<z.ZodEffects<z.ZodAny, any[], any>, z.ZodArray<z.ZodObject<{
-                    title: z.ZodString;
-                    link: z.ZodOptional<z.ZodString>;
-                    description: z.ZodOptional<z.ZodString>;
-                    "dc:creator": z.ZodOptional<z.ZodUnion<[z.ZodString, z.ZodArray<z.ZodString, "many">]>>;
-                    "dc:date": z.ZodOptional<z.ZodString>;
-                    "dc:title": z.ZodOptional<z.ZodString>;
-                    "dc:subject": z.ZodOptional<z.ZodString>;
-                    "dc:publisher": z.ZodOptional<z.ZodString>;
-                    "dc:identifier": z.ZodOptional<z.ZodUnion<[z.ZodString, z.ZodArray<z.ZodString, "many">]>>;
-                    "dc:rights": z.ZodOptional<z.ZodString>;
-                    "dc:source": z.ZodOptional<z.ZodString>;
-                    "prism:publicationDate": z.ZodOptional<z.ZodString>;
-                    "prism:section": z.ZodOptional<z.ZodString>;
-                    "prism:volume": z.ZodOptional<z.ZodString>;
-                    "prism:number": z.ZodOptional<z.ZodString>;
-                    "prism:startingPage": z.ZodOptional<z.ZodString>;
-                    "prism:endingPage": z.ZodOptional<z.ZodString>;
-                    category: z.ZodOptional<z.ZodUnion<[z.ZodString, z.ZodArray<z.ZodString, "many">]>>;
-                }, "strip", z.ZodTypeAny, {
-                    title: string;
-                    link?: string | undefined;
-                    description?: string | undefined;
-                    "dc:creator"?: string | string[] | undefined;
-                    "dc:date"?: string | undefined;
-                    "dc:title"?: string | undefined;
-                    "dc:subject"?: string | undefined;
-                    "dc:publisher"?: string | undefined;
-                    "dc:identifier"?: string | string[] | undefined;
-                    "dc:rights"?: string | undefined;
-                    "dc:source"?: string | undefined;
-                    "prism:publicationDate"?: string | undefined;
-                    "prism:section"?: string | undefined;
-                    "prism:volume"?: string | undefined;
-                    "prism:number"?: string | undefined;
-                    "prism:startingPage"?: string | undefined;
-                    "prism:endingPage"?: string | undefined;
-                    category?: string | string[] | undefined;
-                }, {
-                    title: string;
-                    link?: string | undefined;
-                    description?: string | undefined;
-                    "dc:creator"?: string | string[] | undefined;
-                    "dc:date"?: string | undefined;
-                    "dc:title"?: string | undefined;
-                    "dc:subject"?: string | undefined;
-                    "dc:publisher"?: string | undefined;
-                    "dc:identifier"?: string | string[] | undefined;
-                    "dc:rights"?: string | undefined;
-                    "dc:source"?: string | undefined;
-                    "prism:publicationDate"?: string | undefined;
-                    "prism:section"?: string | undefined;
-                    "prism:volume"?: string | undefined;
-                    "prism:number"?: string | undefined;
-                    "prism:startingPage"?: string | undefined;
-                    "prism:endingPage"?: string | undefined;
-                    category?: string | string[] | undefined;
-                }>, "many">>>;
-            }, "strip", z.ZodTypeAny, {
-                link?: string | undefined;
-                title?: string | undefined;
-                description?: string | undefined;
-                item?: {
-                    title: string;
-                    link?: string | undefined;
-                    description?: string | undefined;
-                    "dc:creator"?: string | string[] | undefined;
-                    "dc:date"?: string | undefined;
-                    "dc:title"?: string | undefined;
-                    "dc:subject"?: string | undefined;
-                    "dc:publisher"?: string | undefined;
-                    "dc:identifier"?: string | string[] | undefined;
-                    "dc:rights"?: string | undefined;
-                    "dc:source"?: string | undefined;
-                    "prism:publicationDate"?: string | undefined;
-                    "prism:section"?: string | undefined;
-                    "prism:volume"?: string | undefined;
-                    "prism:number"?: string | undefined;
-                    "prism:startingPage"?: string | undefined;
-                    "prism:endingPage"?: string | undefined;
-                    category?: string | string[] | undefined;
-                }[] | undefined;
-                items?: {
-                    "rdf:Seq"?: {
-                        "rdf:li"?: any;
-                    } | undefined;
-                } | undefined;
-            }, {
-                link?: string | undefined;
-                title?: string | undefined;
-                description?: string | undefined;
-                item?: any;
-                items?: {
-                    "rdf:Seq"?: {
-                        "rdf:li"?: any;
-                    } | undefined;
-                } | undefined;
-            }>>;
-        }, "strip", z.ZodTypeAny, {
-            channel?: {
-                link?: string | undefined;
-                title?: string | undefined;
-                description?: string | undefined;
-                item?: {
-                    title: string;
-                    link?: string | undefined;
-                    description?: string | undefined;
-                    "dc:creator"?: string | string[] | undefined;
-                    "dc:date"?: string | undefined;
-                    "dc:title"?: string | undefined;
-                    "dc:subject"?: string | undefined;
-                    "dc:publisher"?: string | undefined;
-                    "dc:identifier"?: string | string[] | undefined;
-                    "dc:rights"?: string | undefined;
-                    "dc:source"?: string | undefined;
-                    "prism:publicationDate"?: string | undefined;
-                    "prism:section"?: string | undefined;
-                    "prism:volume"?: string | undefined;
-                    "prism:number"?: string | undefined;
-                    "prism:startingPage"?: string | undefined;
-                    "prism:endingPage"?: string | undefined;
-                    category?: string | string[] | undefined;
-                }[] | undefined;
-                items?: {
-                    "rdf:Seq"?: {
-                        "rdf:li"?: any;
-                    } | undefined;
-                } | undefined;
-            } | undefined;
-        }, {
-            channel?: {
-                link?: string | undefined;
-                title?: string | undefined;
-                description?: string | undefined;
-                item?: any;
-                items?: {
-                    "rdf:Seq"?: {
-                        "rdf:li"?: any;
-                    } | undefined;
-                } | undefined;
-            } | undefined;
-        }>>;
-        feed: z.ZodOptional<z.ZodObject<{
-            title: z.ZodOptional<z.ZodString>;
-            link: z.ZodOptional<z.ZodString>;
-            entry: z.ZodPipeline<z.ZodEffects<z.ZodAny, any[], any>, z.ZodArray<z.ZodObject<{
-                title: z.ZodString;
-                link: z.ZodOptional<z.ZodString>;
-                summary: z.ZodOptional<z.ZodString>;
-                content: z.ZodOptional<z.ZodString>;
-                author: z.ZodOptional<z.ZodString>;
-                published: z.ZodOptional<z.ZodString>;
-                id: z.ZodOptional<z.ZodString>;
-            }, "strip", z.ZodTypeAny, {
-                title: string;
-                content?: string | undefined;
-                id?: string | undefined;
-                link?: string | undefined;
-                summary?: string | undefined;
-                author?: string | undefined;
-                published?: string | undefined;
-            }, {
-                title: string;
-                content?: string | undefined;
-                id?: string | undefined;
-                link?: string | undefined;
-                summary?: string | undefined;
-                author?: string | undefined;
-                published?: string | undefined;
-            }>, "many">>;
-        }, "strip", z.ZodTypeAny, {
-            entry: {
-                title: string;
-                content?: string | undefined;
-                id?: string | undefined;
-                link?: string | undefined;
-                summary?: string | undefined;
-                author?: string | undefined;
-                published?: string | undefined;
-            }[];
-            link?: string | undefined;
-            title?: string | undefined;
-        }, {
-            link?: string | undefined;
-            title?: string | undefined;
-            entry?: any;
-        }>>;
-    }, "strip", z.ZodTypeAny, {
-        rss?: {
-            channel: {
-                item: {
-                    title: string;
-                    link?: string | undefined;
-                    description?: string | undefined;
-                    pubDate?: string | undefined;
-                    guid?: string | z.objectOutputType<{}, z.ZodTypeAny, "passthrough"> | undefined;
-                    "content:encoded"?: string | undefined;
-                    "dc:creator"?: string | string[] | undefined;
-                    "dc:date"?: string | undefined;
-                    "dc:title"?: string | undefined;
-                    "dc:subject"?: string | undefined;
-                    "dc:publisher"?: string | undefined;
-                    "dc:identifier"?: string | string[] | undefined;
-                    "dc:rights"?: string | undefined;
-                    "dc:source"?: string | undefined;
-                    "prism:publicationDate"?: string | undefined;
-                    "prism:section"?: string | undefined;
-                    "prism:volume"?: string | undefined;
-                    "prism:number"?: string | undefined;
-                    "prism:startingPage"?: string | undefined;
-                    "prism:endingPage"?: string | undefined;
-                    "arxiv:announce_type"?: string | undefined;
-                    category?: string | string[] | undefined;
-                }[];
-                link?: string | undefined;
-                title?: string | undefined;
-                description?: string | undefined;
-                pubDate?: string | undefined;
-                lastBuildDate?: string | undefined;
-                language?: string | undefined;
-            };
-        } | undefined;
-        rdf?: {
-            channel?: {
-                link?: string | undefined;
-                title?: string | undefined;
-                description?: string | undefined;
-                item?: {
-                    title: string;
-                    link?: string | undefined;
-                    description?: string | undefined;
-                    "dc:creator"?: string | string[] | undefined;
-                    "dc:date"?: string | undefined;
-                    "dc:title"?: string | undefined;
-                    "dc:subject"?: string | undefined;
-                    "dc:publisher"?: string | undefined;
-                    "dc:identifier"?: string | string[] | undefined;
-                    "dc:rights"?: string | undefined;
-                    "dc:source"?: string | undefined;
-                    "prism:publicationDate"?: string | undefined;
-                    "prism:section"?: string | undefined;
-                    "prism:volume"?: string | undefined;
-                    "prism:number"?: string | undefined;
-                    "prism:startingPage"?: string | undefined;
-                    "prism:endingPage"?: string | undefined;
-                    category?: string | string[] | undefined;
-                }[] | undefined;
-                items?: {
-                    "rdf:Seq"?: {
-                        "rdf:li"?: any;
-                    } | undefined;
-                } | undefined;
-            } | undefined;
-        } | undefined;
-        feed?: {
-            entry: {
-                title: string;
-                content?: string | undefined;
-                id?: string | undefined;
-                link?: string | undefined;
-                summary?: string | undefined;
-                author?: string | undefined;
-                published?: string | undefined;
-            }[];
-            link?: string | undefined;
-            title?: string | undefined;
-        } | undefined;
-    }, {
-        rss?: {
-            channel: {
-                link?: string | undefined;
-                title?: string | undefined;
-                description?: string | undefined;
-                pubDate?: string | undefined;
-                lastBuildDate?: string | undefined;
-                language?: string | undefined;
-                item?: any;
-            };
-        } | undefined;
-        rdf?: {
-            channel?: {
-                link?: string | undefined;
-                title?: string | undefined;
-                description?: string | undefined;
-                item?: any;
-                items?: {
-                    "rdf:Seq"?: {
-                        "rdf:li"?: any;
-                    } | undefined;
-                } | undefined;
-            } | undefined;
-        } | undefined;
-        feed?: {
-            link?: string | undefined;
-            title?: string | undefined;
-            entry?: any;
-        } | undefined;
-    }>;
-    collect(url: string): Promise<SimpleGeneralRSSCollectedData>;
-    /**
-     * Extracts GUID value from either a string or object
-     */
-    private extractGuidValue;
-    /**
-     * Dynamically searches for items in any XML structure
-     */
-    private findItemsDynamically;
-    /**
-     * Fallback method using regex to find items when standard parsing fails
-     */
-    private findItemsWithRegex;
-    /**
-   * Dynamically searches for channel title in any XML structure
-   */
-    private findChannelTitle;
-    /**
-     * Find channel title using regex
-     */
-    private findChannelTitleWithRegex;
-    /**
-     * Intelligently extracts the main text content from an RSS item.
-     * Finds the element with the most text and appends other text-rich elements.
-     * Returns cleaned, combined text content.
-     */
-    private extractMainText;
-    /**
-     * Checks if two text contents are substantially similar to avoid duplication
-     */
-    private isDuplicateContent;
-    /**
-     * Cleans text of HTML tags, special characters, and formatting.
-     * Preserves readable text while removing markup.
-     */
-    private cleanText;
-    /**
-     * Extracts structured metadata from RSS item fields.
-     */
-    private extractMetadata;
-    /**
-     * Extracts tags from the item and combines with feed-level tags.
-     */
-    private extractTags;
-}
-type SimpleGeneralRSSCollectedData = {
-    title: string;
-    link?: string;
-    mainText: string;
-    rawXml: any;
-    metadata: Record<string, any>;
-    tags: string[];
-    pubDate?: string;
-    guid?: string;
-}[];
+declare function parseResponseAsJSON<T>(response: string): T | undefined;
 
 /**
- * Simple string collector that takes a string as input and outputs it as-is.
- * Useful for basic string processing or when you need to pass through string data.
+ * Converts the given byte array to a string
  */
-declare class StringCollector extends AbstractCollector<string> {
-    readonly identifier = "string";
-    collect(source: unknown): Promise<string | undefined>;
-}
-
-declare class ForwardError extends Error {
-    startedAt: Date;
-    code?: string;
-    constructor(message: string, options?: ErrorOptions & {
-        startedAt: Date;
-        code?: string;
-    });
-}
-
-declare class InvalidDataError extends Error {
-    constructor(message?: string);
-}
-declare class ParserIsNotCompatibleError extends InvalidDataError {
-    constructor();
-}
-
-declare const PEERBENCH_ERROR_CODES: {
-    readonly PROVIDER_FORWARD_FAILED: "PROVIDER_FORWARD_FAILED";
-    readonly PROVIDER_UNAUTHORIZED: "PROVIDER_UNAUTHORIZED";
-    readonly PROVIDER_MAX_RETRIES_REACHED: "PROVIDER_MAX_RETRIES_REACHED";
-};
-type PeerBenchErrorCode = (typeof PEERBENCH_ERROR_CODES)[keyof typeof PEERBENCH_ERROR_CODES];
-
+declare function bufferToString(buffer: Uint8Array, encoding?: BufferEncoding): string;
 /**
- * Abstract prompt generator class
+ * Converts the given string to a byte array
  */
-declare abstract class AbstractGenerator {
-    abstract readonly identifier: string;
-    abstract inputSchema: z.ZodSchema<unknown>;
-    /**
-     * Generate prompt from the collected source data
-     * @param input - Raw input data that will be validated against inputSchema
-     * @param options - Optional configuration options
-     * @returns Promise resolving to an array of prompts
-     */
-    generate(input: unknown, options?: Parameters<this["generatePrompts"]>[1]): Promise<Prompt[]>;
-    /**
-     * Abstract method that implementors MUST override.
-     * This method receives already validated input data.
-     *
-     * NOTE: Callers must use `generate()` method
-     */
-    abstract generatePrompts(input: z.infer<(typeof this)["inputSchema"]>, options?: Record<string, any>): Promise<Prompt[]>;
-    /**
-     * Checks whether the generator can handle the given input
-     */
-    canHandle(input: any): MaybePromise<boolean>;
-    /**
-     * Initializes the generator (depends on the implementation)
-     */
-    initialize(...args: any[]): Promise<void>;
-    buildPrompt(params: {
-        /**
-         * Base question
-         */
-        question: string;
-        /**
-         * For multiple choice prompts, includes each option as letter-answer pairs
-         */
-        options?: Record<string, string>;
-        /**
-         * Correct answer that is expected. For multiple choice prompts,
-         * this is the letter of the correct answer.
-         */
-        correctAnswer: string;
-        /**
-         * The full Prompt text that will be sent.
-         */
-        fullPrompt: string;
-        /**
-         * Prompt type
-         */
-        type: PromptType;
-        /**
-         * Metadata
-         */
-        metadata?: Record<string, any>;
-        /**
-         * Expected Scorers that can be used to
-         * score the Responses for this Prompt
-         */
-        scorers?: string[];
-    }): Promise<Prompt>;
-}
+declare function stringToBuffer(str: string): Uint8Array;
 
-declare class TRPGenerator extends AbstractGenerator {
-    identifier: string;
-    inputSchema: z.ZodArray<z.ZodObject<{
-        pmid: z.ZodString;
-        title: z.ZodString;
-        paragraphs: z.ZodRecord<z.ZodString, z.ZodString>;
-        tags: z.ZodArray<z.ZodString, "many">;
-    }, "strip", z.ZodTypeAny, {
-        title: string;
-        pmid: string;
-        paragraphs: Record<string, string>;
-        tags: string[];
-    }, {
-        title: string;
-        pmid: string;
-        paragraphs: Record<string, string>;
-        tags: string[];
-    }>, "many">;
-    optionsSchema: z.ZodObject<{
-        openRouterApiKey: z.ZodString;
-        paragraphMergeStrategy: z.ZodDefault<z.ZodEffects<z.ZodString, NonNullable<"without-titles" | "with-titles-as-sentences" | "titles-within-sentences">, string>>;
-        model: z.ZodString;
-        placeholder: z.ZodDefault<z.ZodString>;
-        nerPrompt: z.ZodDefault<z.ZodOptional<z.ZodString>>;
-    }, "strip", z.ZodTypeAny, {
-        model: string;
-        openRouterApiKey: string;
-        paragraphMergeStrategy: NonNullable<"without-titles" | "with-titles-as-sentences" | "titles-within-sentences">;
-        placeholder: string;
-        nerPrompt: string;
-    }, {
-        model: string;
-        openRouterApiKey: string;
-        paragraphMergeStrategy?: string | undefined;
-        placeholder?: string | undefined;
-        nerPrompt?: string | undefined;
-    }>;
-    generatePrompts(input: TypeOf<this["inputSchema"]>, options?: z.input<(typeof this)["optionsSchema"]>): Promise<{
-        type: "multiple-choice" | "order-sentences" | "text-replacement" | "typo" | "open-ended" | "open-ended-with-docs";
-        did: string;
-        question: {
-            sha256: string;
-            data: string;
-            cid: string;
-        };
-        fullPrompt: {
-            sha256: string;
-            data: string;
-            cid: string;
-        };
-        answer?: string | undefined;
-        metadata?: Record<string, any> | undefined;
-        options?: Record<string, string> | undefined;
-        answerKey?: string | undefined;
-        scorers?: string[] | undefined;
-    }[]>;
-    private generatePromptFromArticle;
-    private doNER;
-}
+declare const idGeneratorUUIDv7: IdGenerator;
 
-/**
- * Abstract class for Providers.
- *
- * A Provider is the entity that is responsible for forwarding the given inputs
- * to the underlying infrastructure (e.g LLM, API, image generation model) and
- * collecting its response as a byte array.
- */
-declare abstract class AbstractProvider<TInput, TOutput> {
-    /**
-     * Unique identifier of the Provider.
-     */
-    abstract readonly identifier: string;
-    /**
-     * Forwards the given input to the Provider and returns the response.
-     *
-     * @example
-     * ```ts
-     * const provider = new ImageGenerationProvider();
-     *
-     * // Input and the options are provider specific.
-     * // In this example we are using a Provider that generates images from text inputs.
-     * const response = await provider.forward("A beautiful sunset over a calm ocean", {
-     *   width: 1024,
-     *   height: 1024,
-     * });
-     *
-     * // Write the generated image to a file
-     * await fs.writeFile("response.png", response.data);
-     * ```
-     */
-    abstract forward(input: TInput, options?: any): Promise<ProviderResponse<TOutput>>;
-}
-/**
- * Type of the response returned by the Provider.
- */
-type ProviderResponse<TData = Uint8Array> = {
-    /**
-     * The date and time when the response was started.
-     */
-    startedAt: Date;
-    /**
-     * The date and time when the response was completed.
-     */
-    completedAt: Date;
-    /**
-     * The response data.
-     */
-    data: TData;
-};
-
-/**
- * Base class for LLM based Providers. It uses OpenAI's API client to forward the
- * inputs to the underlying LLM and returns the response as a string. Also implements
- * rate limiting and timeout features for parallel execution.
- */
-declare abstract class BaseLLMProvider extends AbstractProvider<string, string> {
-    protected timestamps: number[];
-    timeout: number;
-    rateLimit: number;
-    rateLimitTimeWindow: number;
-    client: OpenAI;
-    maxRetries: number;
-    /**
-     * Initialize a new LLM provider
-     * @param options Options for the provider
-     */
-    constructor(options: BaseLLMProviderOptions);
-    enforceRateLimit(): Promise<void>;
-    /**
-     * Fetch all supported models from the provider
-     * @returns Array of model information
-     */
-    getSupportedModels(): Promise<ModelInfo[]>;
-    forward(input: string | ChatCompletionMessageParam[], options: BaseLLMProviderForwardOptions): Promise<ForwardResponse>;
-    /**
-     * Parses the given model ID that includes the model name, owner, and sub-provider (if any).
-     * @param id Model ID that has the format `<sub provider name if any>/<model owner>/<model name>`
-     * @deprecated
-     */
-    abstract parseModelInfo(modelOrId: OpenAI.Models.Model | string): MaybePromise<ModelInfo | undefined>;
-}
-type ForwardResponse = {
-    data: string;
-    startedAt: Date;
-    completedAt: Date;
-    inputTokensUsed?: number;
-    inputCost?: string;
-    outputTokensUsed?: number;
-    outputCost?: string;
-};
-type BaseLLMProviderOptions = {
-    /**
-     * API key for the provider
-     */
-    apiKey: string;
-    /**
-     * Base URL for the provider
-     */
-    baseURL?: string;
-    /**
-     * Maximum number of retries for the provider
-     */
-    maxRetries?: number;
-    /**
-     * Timeout for the provider
-     */
-    timeout?: number;
-    /**
-     * Rate limit for the provider
-     */
-    rateLimit?: number;
-    /**
-     * Rate limit time window for the provider
-     */
-    rateLimitTimeWindow?: number;
-};
-type BaseLLMProviderForwardOptions = {
-    model: string;
-    system?: string;
-    abortSignal?: AbortSignal;
-    temperature?: number;
-};
-/**
- * Parsed information about the model.
- */
-type ModelInfo = {
-    /**
-     * Original ID of the model that can be used in the requests
-     */
-    id: string;
-    /**
-     * Unified name of the model
-     */
-    name: LargeLanguageModelType;
-    /**
-     * Unified owner of the model
-     */
-    owner: LargeLanguageModelOwnerType;
-    /**
-     * Provider name of the model
-     * TODO: This field might be redundant
-     */
-    provider: string;
-    /**
-     * The entity that responsible for hosting the model
-     */
-    host?: string;
-    /**
-     * The tier of the model (e.g free, max)
-     */
-    tier?: string;
-    /**
-     * Additional metadata (warning: not might be available always)
-     */
-    metadata?: Record<string, unknown>;
-};
-/**
- * Known LLM owners
- */
-declare const LargeLanguageModelOwner: {
-    readonly Meta: "meta";
-    readonly OpenAI: "openai";
-    readonly Deepseek: "deepseek";
-    readonly Qwen: "qwen";
-    readonly Google: "google";
-    readonly XAI: "x-ai";
-    readonly Anthropic: "anthropic";
-    readonly Mistral: "mistral";
-};
-type LargeLanguageModelOwnerType = (typeof LargeLanguageModelOwner)[keyof typeof LargeLanguageModelOwner];
-/**
- * Known models of Meta
- */
-declare const MetaModels: {
-    readonly Llama_4_Maverick: "llama-4-maverick";
-    readonly Llama_4_Scout: "llama-4-scout";
-    readonly Llama_3_3_70b_Instruct: "llama-3.3-70b-instruct";
-    readonly Llama_3_1_8b_Instruct: "llama-3.1-8b-instruct";
-    readonly Llama_3_1_70b_Instruct: "llama-3.1-70b-instruct";
-};
-/**
- * Known models of Qwen
- */
-declare const QwenModels: {
-    readonly QwQ_32b: "qwq-32b";
-};
-/**
- * Known models of DeepSeek
- */
-declare const DeepSeekModels: {
-    readonly V3: "deepseek-v3";
-    readonly V3_0324: "deepseek-v3-0324";
-};
-/**
- * Known models of XAI
- */
-declare const XAIModels: {
-    readonly Grok3_Beta: "grok-3-beta";
-    readonly Grok4: "grok-4";
-};
-/**
- * Known models of Google
- */
-declare const GoogleModels: {
-    readonly Gemini_2_0_Flash: "gemini-2.0-flash";
-    readonly Gemini_2_5_Flash_Lite: "gemini-2.5-flash-lite";
-    readonly Gemini_2_5_Pro: "gemini-2.5-pro";
-};
-/**
- * Known models of Anthropic
- */
-declare const AnthropicModels: {
-    readonly Claude_3_7_Sonnet: "claude-3.7-sonnet";
-    readonly Claude_Sonnet_4_5: "claude-sonnet-4.5";
-};
-/**
- * Known models of OpenAI
- */
-declare const OpenAIModels: {
-    readonly ChatGPT_4o: "chatgpt-4o-latest";
-    readonly GPT_4o: "gpt-4o";
-    readonly GPT_4o_Mini: "gpt-4o-mini";
-    readonly GPT_5: "gpt-5";
-};
-declare const MistralModels: {
-    readonly Ministral_8B: "ministral-8b";
-};
-/**
- * Known models of all providers
- */
-declare const LargeLanguageModel: {
-    readonly meta: {
-        readonly Llama_4_Maverick: "llama-4-maverick";
-        readonly Llama_4_Scout: "llama-4-scout";
-        readonly Llama_3_3_70b_Instruct: "llama-3.3-70b-instruct";
-        readonly Llama_3_1_8b_Instruct: "llama-3.1-8b-instruct";
-        readonly Llama_3_1_70b_Instruct: "llama-3.1-70b-instruct";
-    };
-    readonly deepseek: {
-        readonly V3: "deepseek-v3";
-        readonly V3_0324: "deepseek-v3-0324";
-    };
-    readonly qwen: {
-        readonly QwQ_32b: "qwq-32b";
+type Runner<TTestCase extends z__default.ZodObject, TResponse extends z__default.ZodObject, TScore extends z__default.ZodObject, TProvider extends AbstractProvider, TScorer extends AbstractScorer, TRunConfig extends Record<string, unknown>> = (params: {
+    testCase: z__default.infer<TTestCase>;
+    provider: TProvider;
+    scorer?: TScorer;
+    runConfig: TRunConfig;
+    idGenerators?: {
+        response?: IdGenerator;
+        score?: IdGenerator;
     };
-    readonly google: {
-        readonly Gemini_2_0_Flash: "gemini-2.0-flash";
-        readonly Gemini_2_5_Flash_Lite: "gemini-2.5-flash-lite";
-        readonly Gemini_2_5_Pro: "gemini-2.5-pro";
-    };
-    readonly "x-ai": {
-        readonly Grok3_Beta: "grok-3-beta";
-        readonly Grok4: "grok-4";
-    };
-    readonly openai: {
-        readonly ChatGPT_4o: "chatgpt-4o-latest";
-        readonly GPT_4o: "gpt-4o";
-        readonly GPT_4o_Mini: "gpt-4o-mini";
-        readonly GPT_5: "gpt-5";
-    };
-    readonly anthropic: {
-        readonly Claude_3_7_Sonnet: "claude-3.7-sonnet";
-        readonly Claude_Sonnet_4_5: "claude-sonnet-4.5";
-    };
-    readonly mistral: {
-        readonly Ministral_8B: "ministral-8b";
-    };
-};
-type LargeLanguageModelType = (typeof MetaModels)[keyof typeof MetaModels] | (typeof DeepSeekModels)[keyof typeof DeepSeekModels] | (typeof QwenModels)[keyof typeof QwenModels] | (typeof GoogleModels)[keyof typeof GoogleModels] | (typeof XAIModels)[keyof typeof XAIModels] | (typeof AnthropicModels)[keyof typeof AnthropicModels] | (typeof OpenAIModels)[keyof typeof OpenAIModels] | (typeof MistralModels)[keyof typeof MistralModels];
-
-declare class OpenRouterProvider extends BaseLLMProvider {
-    private models;
-    private modelsCachePromise;
-    private modelsUpdatedAt;
-    readonly identifier = "openrouter.ai";
-    constructor(options: OpenRouterProviderOptions);
-    /**
-     * Updates the cache that holds information about OpenRouter models
-     * including pricing information. It will be valid for 24 hours as
-     * long as the instance of this Provider object is alive.
-     */
-    private updateModelsCache;
-    /**
-     * Returns the detailed information about
-     * the models that are available on OpenRouter.
-     */
-    getModelDetails(): Promise<ModelsResponse["data"] | undefined>;
-    getModelDetails(modelId: string): Promise<ModelsResponse["data"][number] | undefined>;
-    /**
-     * Returns the credits that the account has on openrouter.ai
-     */
-    getCredits(): Promise<any>;
-    /**
-     * Makes a request to a protected endpoint to validate the API key.
-     */
-    validateApiKey(): Promise<boolean>;
-    forward(input: string | ChatCompletionMessageParam$1[], options: BaseLLMProviderForwardOptions): Promise<ForwardResponse>;
-    parseModelInfo(modelOrId: OpenAI.Models.Model | string): ModelInfo | undefined;
-}
-type OpenRouterProviderOptions = Omit<BaseLLMProviderOptions, "baseURL">;
-type PutModality = "text" | "image" | "file" | "audio";
-type Modality = "text->text" | "text+image->text" | "text+image->text+image";
-type ModelsResponse = {
-    data: {
-        readonly id: string;
-        readonly canonical_slug: string;
-        readonly hugging_face_id: null | string;
-        readonly name: string;
-        readonly created: number;
-        readonly description: string;
-        readonly context_length: number;
-        readonly architecture: {
-            readonly modality: Modality;
-            readonly input_modalities: PutModality[];
-            readonly output_modalities: PutModality[];
-            readonly instruct_type: null | string;
-        };
-        readonly pricing: {
-            readonly prompt: string;
-            readonly completion: string;
-            readonly request?: string;
-            readonly image?: string;
-            readonly web_search?: string;
-            readonly internal_reasoning?: string;
-            readonly input_cache_read?: string;
-            readonly input_cache_write?: string;
-            readonly audio?: string;
-        };
-    }[];
-};
-
-declare class NearAIProvider extends BaseLLMProvider {
-    readonly identifier = "near.ai";
-    constructor(options: NearAIProviderOptions);
-    parseModelInfo(modelOrId: OpenAI.Models.Model | string): ModelInfo | undefined;
-}
-type NearAIProviderOptions = Omit<BaseLLMProviderOptions, "baseURL">;
-
-/**
- * Generic Multiple Choice Question Generator using an LLM model
- */
-declare class MCQGenerator extends AbstractGenerator {
-    readonly identifier = "mcq";
-    inputSchema: z.ZodArray<z.ZodAny, "many">;
-    optionsSchema: z.ZodEffects<z.ZodObject<{
-        /**
-         * The API key for the OpenRouter provider. Won't be used if `provider` is given.
-         */
-        openRouterApiKey: z.ZodOptional<z.ZodString>;
-        /**
-         * The Provider that is going to be used to generate the Prompts.
-         */
-        provider: z.ZodOptional<z.ZodType<BaseLLMProvider, z.ZodTypeDef, BaseLLMProvider>>;
-        /**
-         * The model that is going to be used to generate the prompts
-         */
-        model: z.ZodString;
-        systemPromptPrefix: z.ZodDefault<z.ZodOptional<z.ZodString>>;
-        /**
-         * Additional rules that will be appended to the system prompt.
-         */
-        systemPromptRules: z.ZodDefault<z.ZodArray<z.ZodString, "many">>;
-        systemPrompt: z.ZodDefault<z.ZodOptional<z.ZodString>>;
-        systemPromptSuffix: z.ZodDefault<z.ZodOptional<z.ZodString>>;
-        /**
-         * The main function that parses the input value to a string.
-         */
-        parseInput: z.ZodFunction<z.ZodTuple<[z.ZodAny], z.ZodUnknown>, z.ZodString>;
-        /**
-         * Whether to include the original data as a field inside the metadata.
-         */
-        includeOriginalInputAsMetadata: z.ZodDefault<z.ZodOptional<z.ZodBoolean>>;
-        /**
-         * Additional metadata that is going to be added to the each Prompt.
-         * If a function is given then that function takes an argument which
-         * is the input value for each item and generates the additional metadata.
-         */
-        additionalMetadata: z.ZodOptional<z.ZodUnion<[z.ZodFunction<z.ZodTuple<[z.ZodAny, z.ZodObject<{
-            systemPrompt: z.ZodString;
-        }, "strip", z.ZodTypeAny, {
-            systemPrompt: string;
-        }, {
-            systemPrompt: string;
-        }>], z.ZodUnknown>, z.ZodRecord<z.ZodString, z.ZodAny>>, z.ZodRecord<z.ZodString, z.ZodAny>]>>;
-    }, "strip", z.ZodTypeAny, {
-        model: string;
-        systemPrompt: string;
-        systemPromptPrefix: string;
-        systemPromptRules: string[];
-        systemPromptSuffix: string;
-        parseInput: (args_0: any, ...args: unknown[]) => string;
-        includeOriginalInputAsMetadata: boolean;
-        provider?: BaseLLMProvider | undefined;
-        openRouterApiKey?: string | undefined;
-        additionalMetadata?: Record<string, any> | ((args_0: any, args_1: {
-            systemPrompt: string;
-        }, ...args: unknown[]) => Record<string, any>) | undefined;
-    }, {
-        model: string;
-        parseInput: (args_0: any, ...args: unknown[]) => string;
-        provider?: BaseLLMProvider | undefined;
-        openRouterApiKey?: string | undefined;
-        systemPrompt?: string | undefined;
-        systemPromptPrefix?: string | undefined;
-        systemPromptRules?: string[] | undefined;
-        systemPromptSuffix?: string | undefined;
-        includeOriginalInputAsMetadata?: boolean | undefined;
-        additionalMetadata?: Record<string, any> | ((args_0: any, args_1: {
-            systemPrompt: string;
-        }, ...args: unknown[]) => Record<string, any>) | undefined;
-    }>, {
-        model: string;
-        systemPrompt: string;
-        systemPromptPrefix: string;
-        systemPromptRules: string[];
-        systemPromptSuffix: string;
-        parseInput: (args_0: any, ...args: unknown[]) => string;
-        includeOriginalInputAsMetadata: boolean;
-        provider?: BaseLLMProvider | undefined;
-        openRouterApiKey?: string | undefined;
-        additionalMetadata?: Record<string, any> | ((args_0: any, args_1: {
-            systemPrompt: string;
-        }, ...args: unknown[]) => Record<string, any>) | undefined;
-    }, {
-        model: string;
-        parseInput: (args_0: any, ...args: unknown[]) => string;
-        provider?: BaseLLMProvider | undefined;
-        openRouterApiKey?: string | undefined;
-        systemPrompt?: string | undefined;
-        systemPromptPrefix?: string | undefined;
-        systemPromptRules?: string[] | undefined;
-        systemPromptSuffix?: string | undefined;
-        includeOriginalInputAsMetadata?: boolean | undefined;
-        additionalMetadata?: Record<string, any> | ((args_0: any, args_1: {
-            systemPrompt: string;
-        }, ...args: unknown[]) => Record<string, any>) | undefined;
-    }>;
-    generatePrompts(input: z.input<this["inputSchema"]>, options?: z.input<this["optionsSchema"]>): Promise<{
-        type: "multiple-choice" | "order-sentences" | "text-replacement" | "typo" | "open-ended" | "open-ended-with-docs";
-        did: string;
-        question: {
-            sha256: string;
-            data: string;
-            cid: string;
-        };
-        fullPrompt: {
-            sha256: string;
-            data: string;
-            cid: string;
-        };
-        answer?: string | undefined;
-        metadata?: Record<string, any> | undefined;
-        options?: Record<string, string> | undefined;
-        answerKey?: string | undefined;
-        scorers?: string[] | undefined;
-    }[]>;
-    private generateFromInput;
-}
-
-declare class OpenEndedGenerator extends AbstractGenerator {
-    readonly identifier = "open-ended";
-    inputSchema: z.ZodArray<z.ZodAny, "many">;
-    optionsSchema: z.ZodEffects<z.ZodObject<{
-        /**
-         * The API key for the OpenRouter provider. Won't be used if `provider` is given.
-         */
-        openRouterApiKey: z.ZodOptional<z.ZodString>;
-        /**
-         * The Provider that is going to be used to generate the Prompts.
-         */
-        provider: z.ZodOptional<z.ZodType<BaseLLMProvider, z.ZodTypeDef, BaseLLMProvider>>;
-        /**
-         * The model that is going to be used to generate the prompts
-         */
-        model: z.ZodString;
-        /**
-         * Adds an additional clue to the end of the question such as "Answer with a positive number"
-         * which is relevant with the answer type expected.
-         */
-        addClueToQuestion: z.ZodDefault<z.ZodOptional<z.ZodBoolean>>;
-        /**
-         * Additional rules that will be appended to the system prompt.
-         */
-        systemPromptRules: z.ZodDefault<z.ZodArray<z.ZodString, "many">>;
-        systemPromptPrefix: z.ZodDefault<z.ZodOptional<z.ZodString>>;
-        systemPrompt: z.ZodDefault<z.ZodOptional<z.ZodString>>;
-        systemPromptSuffix: z.ZodDefault<z.ZodOptional<z.ZodString>>;
-        /**
-         * The main function that parses the input value to a string.
-         */
-        parseInput: z.ZodFunction<z.ZodTuple<[z.ZodAny], z.ZodUnknown>, z.ZodString>;
-        /**
-         * Whether to include the original as a field inside the metadata.
-         */
-        includeOriginalInputAsMetadata: z.ZodDefault<z.ZodOptional<z.ZodBoolean>>;
-        /**
-         * Additional metadata that is going to be added to the each Prompt.
-         * If a function is given then that function takes an argument which
-         * is the input value for each item and generates the additional metadata.
-         */
-        additionalMetadata: z.ZodOptional<z.ZodUnion<[z.ZodFunction<z.ZodTuple<[z.ZodAny, z.ZodObject<{
-            systemPrompt: z.ZodString;
-        }, "strip", z.ZodTypeAny, {
-            systemPrompt: string;
-        }, {
-            systemPrompt: string;
-        }>], z.ZodUnknown>, z.ZodRecord<z.ZodString, z.ZodAny>>, z.ZodRecord<z.ZodString, z.ZodAny>]>>;
-    }, "strip", z.ZodTypeAny, {
-        model: string;
-        systemPrompt: string;
-        systemPromptPrefix: string;
-        systemPromptRules: string[];
-        systemPromptSuffix: string;
-        parseInput: (args_0: any, ...args: unknown[]) => string;
-        includeOriginalInputAsMetadata: boolean;
-        addClueToQuestion: boolean;
-        provider?: BaseLLMProvider | undefined;
-        openRouterApiKey?: string | undefined;
-        additionalMetadata?: Record<string, any> | ((args_0: any, args_1: {
-            systemPrompt: string;
-        }, ...args: unknown[]) => Record<string, any>) | undefined;
-    }, {
-        model: string;
-        parseInput: (args_0: any, ...args: unknown[]) => string;
-        provider?: BaseLLMProvider | undefined;
-        openRouterApiKey?: string | undefined;
-        systemPrompt?: string | undefined;
-        systemPromptPrefix?: string | undefined;
-        systemPromptRules?: string[] | undefined;
-        systemPromptSuffix?: string | undefined;
-        includeOriginalInputAsMetadata?: boolean | undefined;
-        additionalMetadata?: Record<string, any> | ((args_0: any, args_1: {
-            systemPrompt: string;
-        }, ...args: unknown[]) => Record<string, any>) | undefined;
-        addClueToQuestion?: boolean | undefined;
-    }>, {
-        model: string;
-        systemPrompt: string;
-        systemPromptPrefix: string;
-        systemPromptRules: string[];
-        systemPromptSuffix: string;
-        parseInput: (args_0: any, ...args: unknown[]) => string;
-        includeOriginalInputAsMetadata: boolean;
-        addClueToQuestion: boolean;
-        provider?: BaseLLMProvider | undefined;
-        openRouterApiKey?: string | undefined;
-        additionalMetadata?: Record<string, any> | ((args_0: any, args_1: {
-            systemPrompt: string;
-        }, ...args: unknown[]) => Record<string, any>) | undefined;
-    }, {
-        model: string;
-        parseInput: (args_0: any, ...args: unknown[]) => string;
-        provider?: BaseLLMProvider | undefined;
-        openRouterApiKey?: string | undefined;
-        systemPrompt?: string | undefined;
-        systemPromptPrefix?: string | undefined;
-        systemPromptRules?: string[] | undefined;
-        systemPromptSuffix?: string | undefined;
-        includeOriginalInputAsMetadata?: boolean | undefined;
-        additionalMetadata?: Record<string, any> | ((args_0: any, args_1: {
-            systemPrompt: string;
-        }, ...args: unknown[]) => Record<string, any>) | undefined;
-        addClueToQuestion?: boolean | undefined;
-    }>;
-    generatePrompts(input: TypeOf<this["inputSchema"]>, options?: z.input<(typeof this)["optionsSchema"]>): Promise<{
-        type: "multiple-choice" | "order-sentences" | "text-replacement" | "typo" | "open-ended" | "open-ended-with-docs";
-        did: string;
-        question: {
-            sha256: string;
-            data: string;
-            cid: string;
-        };
-        fullPrompt: {
-            sha256: string;
-            data: string;
-            cid: string;
-        };
-        answer?: string | undefined;
-        metadata?: Record<string, any> | undefined;
-        options?: Record<string, string> | undefined;
-        answerKey?: string | undefined;
-        scorers?: string[] | undefined;
-    }[]>;
-    private generateFromInput;
-}
-
-/**
- * Merges the paragraphs of a Pubmed article into a single string.
- */
-declare function paragraphMerge(paragraphs: Record<string, string>, strategy: ParagraphMergeStrategyType): string;
-declare const ParagraphMergeStrategy: {
-    readonly WithoutTitles: "without-titles";
-    readonly TitlesAsSentences: "with-titles-as-sentences";
-    readonly TitlesWithinSentences: "titles-within-sentences";
-};
-type ParagraphMergeStrategyType = (typeof ParagraphMergeStrategy)[keyof typeof ParagraphMergeStrategy];
-
-/**
- * Abstract Registry class
- *
- * A Registry implementation is responsible from sending, receiving and doing
- * other operations with an external service such as a remote server.
- */
-declare abstract class AbstractRegistry {
-    /**
-     * The unique identifier of the Registry.
-     */
-    abstract readonly identifier: string;
-    /**
-     * Uploads the given prompts to the Registry.
-     *
-     * @returns the number of prompts uploaded
-     */
-    abstract uploadPrompts(prompts: Prompt[], options?: Record<string, any>): MaybePromise<number>;
-    /**
-     * Uploads the given responses to the Registry.
-     *
-     * @returns the number of responses uploaded
-     */
-    abstract uploadResponses(responses: PromptResponse[], options?: Record<string, any>): MaybePromise<number>;
-    /**
-     * Uploads the given scores to the Registry.
-     *
-     * @returns the number of scores uploaded
-     */
-    abstract uploadScores(scores: PromptScore[], options?: Record<string, any> & {
-        /**
-         * Whether to include the response and prompt data in the scores
-         */
-        includeData?: boolean;
-    }): MaybePromise<number>;
-}
-
-/**
- * Registry implementation for peerBench server
- */
-declare class PeerBenchRegistry extends AbstractRegistry {
-    static readonly identifier = "peerbench";
-    readonly identifier = "peerbench";
-    private token?;
-    private supabaseClient?;
-    private apiURL;
-    private session;
-    private readonly email?;
-    private readonly password?;
-    private readonly authMethod?;
-    private refreshTokenInterval?;
-    private isRefreshingToken;
-    private isClosed;
-    private isInitialized;
-    private tokenRefresher;
-    constructor(options: PeerBenchRegistryOptions);
-    uploadPrompts(prompts: Prompt[], options: UploadPromptsOptions): Promise<number>;
-    uploadResponses(responses: PromptResponse[], options?: UploadResponsesOptions): Promise<number>;
-    uploadScores(scores: PromptScore[], options?: UploadScoresOptions): Promise<number>;
-    /**
-     * Clears the interval execution for refreshing the token.
-     */
-    clearRefreshInterval(): Promise<void>;
-    private init;
-    /**
-     * Refreshes the token if it is about to expire.
-     */
-    private refreshToken;
-    private login;
-}
-type PeerBenchRegistryOptionsBase = {
-    peerbenchApiURL: string;
-    /**
-     * If you are using this instance in a browser environment, you can
-     * use `cookie` approach to authenticate requests. In this case the
-     * user must be logged in via Supabase auth.
-     * @default "token"
-     */
-    authMethod?: "token" | "cookie";
-    /**
-     * Supabase tokens must be refreshed periodically.
-     * But if the class usage is short-lived, it's not necessary to start
-     * a background interval for refreshing the token.
-     * @default false
-     */
-    tokenRefresher?: boolean;
-    peerbenchSupabaseURL?: string;
-    peerbenchSupabaseAnonKey?: string;
-    email?: string;
-    password?: string;
-};
-type PeerBenchRegistryOptions = (PeerBenchRegistryOptionsBase & {
-    authMethod?: "token";
-    tokenRefresher?: boolean;
-    peerbenchSupabaseURL: string;
-    peerbenchSupabaseAnonKey: string;
-    email: string;
-    password: string;
-}) | (PeerBenchRegistryOptionsBase & {
-    authMethod: "cookie";
-    tokenRefresher?: never;
-    peerbenchSupabaseURL?: never;
-    peerbenchSupabaseAnonKey?: never;
-    email?: never;
-    password?: never;
-});
-/**
- * Options for uploading prompts
- */
-type UploadPromptsOptions = {
-    promptSetId: number;
-    account?: Account;
-};
-/**
- * Options for uploading responses
- */
-type UploadResponsesOptions = {
-    account?: Account;
-};
-/**
- * Options for uploading scores
- */
-type UploadScoresOptions = {
-    account?: Account;
-};
-
-/**
- * Abstract base class for scorers
- */
-declare abstract class AbstractScorer {
-    /**
-     * Unique identifier for the scorer
-     */
-    abstract readonly identifier: string;
-    /**
-     * Score a single response
-     * @param response The response to score
-     * @param options Additional options for scoring
-     * @returns PromptScore object including the given Response, Prompt and the calculated Score
-     */
-    abstract scoreOne(response: PromptResponse, options?: Record<string, any>): MaybePromise<PromptScore | undefined>;
-    /**
-     * Checks whether the scorer is eligible to score the given response
-     */
-    abstract canScore(response: PromptResponse): MaybePromise<boolean>;
-}
-
-/**
- * Scorer implementation for multiple choice questions. Parses the answer letter
- * from the response text by looking for patterns like `answer is <letter>` or
- * `<letter>:` and compares it with the `answerKey` of the prompt.
- */
-declare class MultipleChoiceScorer extends AbstractScorer {
-    readonly identifier = "multiple-choice";
-    private readonly EXPLANATION_TEXT;
-    /**
-     * Score a multiple choice response
-     */
-    scoreOne(response: PromptResponse): Promise<{
-        sha256: string;
-        startedAt: number;
-        modelName: string;
-        did: string;
-        cid: string;
-        provider: string;
-        modelId: string;
-        modelOwner: string;
-        modelHost: string;
-        finishedAt: number;
-        runId: string;
-        scoreDID: string;
-        score: number;
-        method: "human" | "ai" | "algo";
-        metadata?: Record<string, any> | undefined;
-        inputTokensUsed?: number | undefined;
-        outputTokensUsed?: number | undefined;
-        data?: string | undefined;
-        inputCost?: string | undefined;
-        outputCost?: string | undefined;
-        prompt?: {
-            did: string;
-            question: {
-                sha256: string;
-                cid: string;
-                data?: string | undefined;
-            };
-            fullPrompt: {
-                sha256: string;
-                cid: string;
-                data?: string | undefined;
-            };
-            answer?: string | undefined;
-            metadata?: Record<string, any> | undefined;
-            options?: Record<string, string> | undefined;
-            type?: "multiple-choice" | "order-sentences" | "text-replacement" | "typo" | "open-ended" | "open-ended-with-docs" | undefined;
-            answerKey?: string | undefined;
-            scorers?: string[] | undefined;
-        } | undefined;
-        scoreMetadata?: Record<string, any> | undefined;
-        explanation?: string | undefined;
-        scorerAI?: {
-            modelName: string;
-            provider: string;
-            modelId: string;
-            modelOwner: string;
-            modelHost: string;
-            inputTokensUsed?: number | undefined;
-            outputTokensUsed?: number | undefined;
-            inputCost?: string | undefined;
-            outputCost?: string | undefined;
-        } | undefined;
-    } | undefined>;
-    canScore(response: PromptResponse): Promise<boolean>;
-    /**
-     * Extracts answer from Response text using regex patterns
-     */
-    private lookForAnswer;
-    private escapeRegex;
-}
-
-/**
- * It can be used both for multiple choice and free form questions.
- * If the Prompt includes options (aka it is multiple choice question) then
- * compares the `answerKey` field with the response and checks if they are the same.
- * Otherwise uses `answer` field for the same thing.
- */
-declare class ExactMatchScorer extends AbstractScorer {
-    readonly identifier = "exact-match";
-    scoreOne(response: PromptResponse): Promise<{
-        sha256: string;
-        startedAt: number;
-        modelName: string;
-        did: string;
-        cid: string;
-        provider: string;
-        modelId: string;
-        modelOwner: string;
-        modelHost: string;
-        finishedAt: number;
-        runId: string;
-        scoreDID: string;
-        score: number;
-        method: "human" | "ai" | "algo";
-        metadata?: Record<string, any> | undefined;
-        inputTokensUsed?: number | undefined;
-        outputTokensUsed?: number | undefined;
-        data?: string | undefined;
-        inputCost?: string | undefined;
-        outputCost?: string | undefined;
-        prompt?: {
-            did: string;
-            question: {
-                sha256: string;
-                cid: string;
-                data?: string | undefined;
-            };
-            fullPrompt: {
-                sha256: string;
-                cid: string;
-                data?: string | undefined;
-            };
-            answer?: string | undefined;
-            metadata?: Record<string, any> | undefined;
-            options?: Record<string, string> | undefined;
-            type?: "multiple-choice" | "order-sentences" | "text-replacement" | "typo" | "open-ended" | "open-ended-with-docs" | undefined;
-            answerKey?: string | undefined;
-            scorers?: string[] | undefined;
-        } | undefined;
-        scoreMetadata?: Record<string, any> | undefined;
-        explanation?: string | undefined;
-        scorerAI?: {
-            modelName: string;
-            provider: string;
-            modelId: string;
-            modelOwner: string;
-            modelHost: string;
-            inputTokensUsed?: number | undefined;
-            outputTokensUsed?: number | undefined;
-            inputCost?: string | undefined;
-            outputCost?: string | undefined;
-        } | undefined;
-    } | undefined>;
-    canScore(response: PromptResponse): Promise<boolean>;
-}
-
-/**
- * A Scorer which uses an LLM model to judge the response against the correct answer.
- */
-declare class RefAnswerEqualityLLMJudgeScorer extends AbstractScorer {
-    readonly identifier = "ref-answer-equality-llm-judge-scorer";
-    optionsSchema: z.ZodEffects<z.ZodObject<{
-        openRouterApiKey: z.ZodOptional<z.ZodString>;
-        provider: z.ZodOptional<z.ZodType<BaseLLMProvider, z.ZodTypeDef, BaseLLMProvider>>;
-        model: z.ZodString;
-        prompt: z.ZodDefault<z.ZodString>;
-        promptSuffix: z.ZodDefault<z.ZodString>;
-        promptPrefix: z.ZodDefault<z.ZodString>;
-    }, "strip", z.ZodTypeAny, {
-        model: string;
-        prompt: string;
-        promptSuffix: string;
-        promptPrefix: string;
-        provider?: BaseLLMProvider | undefined;
-        openRouterApiKey?: string | undefined;
-    }, {
-        model: string;
-        provider?: BaseLLMProvider | undefined;
-        prompt?: string | undefined;
-        openRouterApiKey?: string | undefined;
-        promptSuffix?: string | undefined;
-        promptPrefix?: string | undefined;
-    }>, {
-        model: string;
-        prompt: string;
-        promptSuffix: string;
-        promptPrefix: string;
-        provider?: BaseLLMProvider | undefined;
-        openRouterApiKey?: string | undefined;
-    }, {
-        model: string;
-        provider?: BaseLLMProvider | undefined;
-        prompt?: string | undefined;
-        openRouterApiKey?: string | undefined;
-        promptSuffix?: string | undefined;
-        promptPrefix?: string | undefined;
-    }>;
-    scoreOne(response: PromptResponse, options: z.input<typeof this.optionsSchema>): Promise<{
-        sha256: string;
-        startedAt: number;
-        modelName: string;
-        did: string;
-        cid: string;
-        provider: string;
-        modelId: string;
-        modelOwner: string;
-        modelHost: string;
-        finishedAt: number;
-        runId: string;
-        scoreDID: string;
-        score: number;
-        method: "human" | "ai" | "algo";
-        metadata?: Record<string, any> | undefined;
-        inputTokensUsed?: number | undefined;
-        outputTokensUsed?: number | undefined;
-        data?: string | undefined;
-        inputCost?: string | undefined;
-        outputCost?: string | undefined;
-        prompt?: {
-            did: string;
-            question: {
-                sha256: string;
-                cid: string;
-                data?: string | undefined;
-            };
-            fullPrompt: {
-                sha256: string;
-                cid: string;
-                data?: string | undefined;
-            };
-            answer?: string | undefined;
-            metadata?: Record<string, any> | undefined;
-            options?: Record<string, string> | undefined;
-            type?: "multiple-choice" | "order-sentences" | "text-replacement" | "typo" | "open-ended" | "open-ended-with-docs" | undefined;
-            answerKey?: string | undefined;
-            scorers?: string[] | undefined;
-        } | undefined;
-        scoreMetadata?: Record<string, any> | undefined;
-        explanation?: string | undefined;
-        scorerAI?: {
-            modelName: string;
-            provider: string;
-            modelId: string;
-            modelOwner: string;
-            modelHost: string;
-            inputTokensUsed?: number | undefined;
-            outputTokensUsed?: number | undefined;
-            inputCost?: string | undefined;
-            outputCost?: string | undefined;
-        } | undefined;
-    } | undefined>;
-    canScore(response: PromptResponse): Promise<boolean>;
-}
-
-/**
- * This Scorer splits the given Response and the answer into sentences and checks the similarity between them.
- * For each sentence that is placed correctly in the Response (same position as in the answer) it counts one score.
- * The final score is the ratio of the number of sentences that are placed correctly in the Response to the total number of sentences in the answer.
- * Simply `correctly_placed_sentences / total_sentence_count`. It uses `sbd` library to split the text into sentences.
- */
-declare class SimilarityScorer extends AbstractScorer {
-    readonly identifier = "similarity";
-    scoreOne(response: PromptResponse, options?: {
-        ignoreCase?: boolean;
-    }): Promise<{
-        sha256: string;
-        startedAt: number;
-        modelName: string;
-        did: string;
-        cid: string;
-        provider: string;
-        modelId: string;
-        modelOwner: string;
-        modelHost: string;
-        finishedAt: number;
-        runId: string;
-        scoreDID: string;
-        score: number;
-        method: "human" | "ai" | "algo";
-        metadata?: Record<string, any> | undefined;
-        inputTokensUsed?: number | undefined;
-        outputTokensUsed?: number | undefined;
-        data?: string | undefined;
-        inputCost?: string | undefined;
-        outputCost?: string | undefined;
-        prompt?: {
-            did: string;
-            question: {
-                sha256: string;
-                cid: string;
-                data?: string | undefined;
-            };
-            fullPrompt: {
-                sha256: string;
-                cid: string;
-                data?: string | undefined;
-            };
-            answer?: string | undefined;
-            metadata?: Record<string, any> | undefined;
-            options?: Record<string, string> | undefined;
-            type?: "multiple-choice" | "order-sentences" | "text-replacement" | "typo" | "open-ended" | "open-ended-with-docs" | undefined;
-            answerKey?: string | undefined;
-            scorers?: string[] | undefined;
-        } | undefined;
-        scoreMetadata?: Record<string, any> | undefined;
-        explanation?: string | undefined;
-        scorerAI?: {
-            modelName: string;
-            provider: string;
-            modelId: string;
-            modelOwner: string;
-            modelHost: string;
-            inputTokensUsed?: number | undefined;
-            outputTokensUsed?: number | undefined;
-            inputCost?: string | undefined;
-            outputCost?: string | undefined;
-        } | undefined;
-    } | undefined>;
-    canScore(response: PromptResponse): Promise<boolean>;
-}
-
-/**
- * LLM Judge Scorer that supports both pointwise and pairwise evaluation modes
- */
-declare class LLMJudgeScorer extends AbstractScorer {
-    readonly identifier = "llm-judge";
-    optionsSchema: z.ZodEffects<z.ZodObject<{
-        openRouterApiKey: z.ZodOptional<z.ZodString>;
-        provider: z.ZodOptional<z.ZodType<BaseLLMProvider, z.ZodTypeDef, BaseLLMProvider>>;
-        model: z.ZodDefault<z.ZodString>;
-        mode: z.ZodDefault<z.ZodEnum<["pointwise", "pairwise"]>>;
-        criteria: z.ZodArray<z.ZodObject<{
-            id: z.ZodString;
-            description: z.ZodString;
-            weight: z.ZodDefault<z.ZodNumber>;
-            scale: z.ZodDefault<z.ZodObject<{
-                min: z.ZodDefault<z.ZodNumber>;
-                max: z.ZodDefault<z.ZodNumber>;
-            }, "strip", z.ZodTypeAny, {
-                min: number;
-                max: number;
-            }, {
-                min?: number | undefined;
-                max?: number | undefined;
-            }>>;
-        }, "strip", z.ZodTypeAny, {
-            id: string;
-            description: string;
-            weight: number;
-            scale: {
-                min: number;
-                max: number;
-            };
-        }, {
-            id: string;
-            description: string;
-            weight?: number | undefined;
-            scale?: {
-                min?: number | undefined;
-                max?: number | undefined;
-            } | undefined;
-        }>, "many">;
-        meta: z.ZodOptional<z.ZodRecord<z.ZodAny, z.ZodAny>>;
-        temperature: z.ZodDefault<z.ZodNumber>;
-        promptPrefix: z.ZodDefault<z.ZodString>;
-        promptSuffix: z.ZodDefault<z.ZodString>;
-        responseB: z.ZodOptional<z.ZodObject<{
-            did: z.ZodPipeline<z.ZodEffects<z.ZodString, string, string>, z.ZodType<string, z.ZodTypeDef, string>>;
-            provider: z.ZodString;
-            modelId: z.ZodString;
-            modelName: z.ZodString;
-            modelOwner: z.ZodString;
-            modelHost: z.ZodString;
-            prompt: z.ZodEffects<z.ZodObject<{
-                did: z.ZodPipeline<z.ZodEffects<z.ZodString, string, string>, z.ZodType<string, z.ZodTypeDef, string>>;
-                question: z.ZodObject<{
-                    data: z.ZodString;
-                    cid: z.ZodString;
-                    sha256: z.ZodString;
-                }, "strip", z.ZodTypeAny, {
-                    sha256: string;
-                    data: string;
-                    cid: string;
-                }, {
-                    sha256: string;
-                    data: string;
-                    cid: string;
-                }>;
-                options: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
-                fullPrompt: z.ZodObject<{
-                    data: z.ZodString;
-                    cid: z.ZodString;
-                    sha256: z.ZodString;
-                }, "strip", z.ZodTypeAny, {
-                    sha256: string;
-                    data: string;
-                    cid: string;
-                }, {
-                    sha256: string;
-                    data: string;
-                    cid: string;
-                }>;
-                type: z.ZodNativeEnum<{
-                    readonly MultipleChoice: "multiple-choice";
-                    readonly OrderSentences: "order-sentences";
-                    readonly TextReplacement: "text-replacement";
-                    readonly Typo: "typo";
-                    readonly OpenEnded: "open-ended";
-                    readonly OpenEndedWithDocs: "open-ended-with-docs";
-                }>;
-                answer: z.ZodOptional<z.ZodString>;
-                answerKey: z.ZodOptional<z.ZodString>;
-                metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodAny>>;
-                scorers: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
-            }, "strip", z.ZodTypeAny, {
-                type: "multiple-choice" | "order-sentences" | "text-replacement" | "typo" | "open-ended" | "open-ended-with-docs";
-                did: string;
-                question: {
-                    sha256: string;
-                    data: string;
-                    cid: string;
-                };
-                fullPrompt: {
-                    sha256: string;
-                    data: string;
-                    cid: string;
-                };
-                answer?: string | undefined;
-                metadata?: Record<string, any> | undefined;
-                options?: Record<string, string> | undefined;
-                answerKey?: string | undefined;
-                scorers?: string[] | undefined;
-            }, {
-                type: "multiple-choice" | "order-sentences" | "text-replacement" | "typo" | "open-ended" | "open-ended-with-docs";
-                did: string;
-                question: {
-                    sha256: string;
-                    data: string;
-                    cid: string;
-                };
-                fullPrompt: {
-                    sha256: string;
-                    data: string;
-                    cid: string;
-                };
-                answer?: string | undefined;
-                metadata?: Record<string, any> | undefined;
-                options?: Record<string, string> | undefined;
-                answerKey?: string | undefined;
-                scorers?: string[] | undefined;
-            }>, {
-                type: "multiple-choice" | "order-sentences" | "text-replacement" | "typo" | "open-ended" | "open-ended-with-docs";
-                did: string;
-                question: {
-                    sha256: string;
-                    data: string;
-                    cid: string;
-                };
-                fullPrompt: {
-                    sha256: string;
-                    data: string;
-                    cid: string;
-                };
-                answer?: string | undefined;
-                metadata?: Record<string, any> | undefined;
-                options?: Record<string, string> | undefined;
-                answerKey?: string | undefined;
-                scorers?: string[] | undefined;
-            }, {
-                type: "multiple-choice" | "order-sentences" | "text-replacement" | "typo" | "open-ended" | "open-ended-with-docs";
-                did: string;
-                question: {
-                    sha256: string;
-                    data: string;
-                    cid: string;
-                };
-                fullPrompt: {
-                    sha256: string;
-                    data: string;
-                    cid: string;
-                };
-                answer?: string | undefined;
-                metadata?: Record<string, any> | undefined;
-                options?: Record<string, string> | undefined;
-                answerKey?: string | undefined;
-                scorers?: string[] | undefined;
-            }>;
-            cid: z.ZodString;
-            sha256: z.ZodString;
-            data: z.ZodString;
-            startedAt: z.ZodNumber;
-            finishedAt: z.ZodNumber;
-            runId: z.ZodString;
-            inputTokensUsed: z.ZodOptional<z.ZodNumber>;
-            outputTokensUsed: z.ZodOptional<z.ZodNumber>;
-            inputCost: z.ZodOptional<z.ZodString>;
-            outputCost: z.ZodOptional<z.ZodString>;
-            metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodAny>>;
-        }, "strip", z.ZodTypeAny, {
-            sha256: string;
-            startedAt: number;
-            data: string;
-            modelName: string;
-            did: string;
-            cid: string;
-            provider: string;
-            modelId: string;
-            modelOwner: string;
-            modelHost: string;
-            prompt: {
-                type: "multiple-choice" | "order-sentences" | "text-replacement" | "typo" | "open-ended" | "open-ended-with-docs";
-                did: string;
-                question: {
-                    sha256: string;
-                    data: string;
-                    cid: string;
-                };
-                fullPrompt: {
-                    sha256: string;
-                    data: string;
-                    cid: string;
-                };
-                answer?: string | undefined;
-                metadata?: Record<string, any> | undefined;
-                options?: Record<string, string> | undefined;
-                answerKey?: string | undefined;
-                scorers?: string[] | undefined;
-            };
-            finishedAt: number;
-            runId: string;
-            metadata?: Record<string, any> | undefined;
-            inputTokensUsed?: number | undefined;
-            outputTokensUsed?: number | undefined;
-            inputCost?: string | undefined;
-            outputCost?: string | undefined;
-        }, {
-            sha256: string;
-            startedAt: number;
-            data: string;
-            modelName: string;
-            did: string;
-            cid: string;
-            provider: string;
-            modelId: string;
-            modelOwner: string;
-            modelHost: string;
-            prompt: {
-                type: "multiple-choice" | "order-sentences" | "text-replacement" | "typo" | "open-ended" | "open-ended-with-docs";
-                did: string;
-                question: {
-                    sha256: string;
-                    data: string;
-                    cid: string;
-                };
-                fullPrompt: {
-                    sha256: string;
-                    data: string;
-                    cid: string;
-                };
-                answer?: string | undefined;
-                metadata?: Record<string, any> | undefined;
-                options?: Record<string, string> | undefined;
-                answerKey?: string | undefined;
-                scorers?: string[] | undefined;
-            };
-            finishedAt: number;
-            runId: string;
-            metadata?: Record<string, any> | undefined;
-            inputTokensUsed?: number | undefined;
-            outputTokensUsed?: number | undefined;
-            inputCost?: string | undefined;
-            outputCost?: string | undefined;
-        }>>;
-    }, "strip", z.ZodTypeAny, {
-        model: string;
-        promptSuffix: string;
-        promptPrefix: string;
-        mode: "pointwise" | "pairwise";
-        criteria: {
-            id: string;
-            description: string;
-            weight: number;
-            scale: {
-                min: number;
-                max: number;
-            };
-        }[];
-        temperature: number;
-        meta?: Record<any, any> | undefined;
-        provider?: BaseLLMProvider | undefined;
-        openRouterApiKey?: string | undefined;
-        responseB?: {
-            sha256: string;
-            startedAt: number;
-            data: string;
-            modelName: string;
-            did: string;
-            cid: string;
-            provider: string;
-            modelId: string;
-            modelOwner: string;
-            modelHost: string;
-            prompt: {
-                type: "multiple-choice" | "order-sentences" | "text-replacement" | "typo" | "open-ended" | "open-ended-with-docs";
-                did: string;
-                question: {
-                    sha256: string;
-                    data: string;
-                    cid: string;
-                };
-                fullPrompt: {
-                    sha256: string;
-                    data: string;
-                    cid: string;
-                };
-                answer?: string | undefined;
-                metadata?: Record<string, any> | undefined;
-                options?: Record<string, string> | undefined;
-                answerKey?: string | undefined;
-                scorers?: string[] | undefined;
-            };
-            finishedAt: number;
-            runId: string;
-            metadata?: Record<string, any> | undefined;
-            inputTokensUsed?: number | undefined;
-            outputTokensUsed?: number | undefined;
-            inputCost?: string | undefined;
-            outputCost?: string | undefined;
-        } | undefined;
-    }, {
-        criteria: {
-            id: string;
-            description: string;
-            weight?: number | undefined;
-            scale?: {
-                min?: number | undefined;
-                max?: number | undefined;
-            } | undefined;
-        }[];
-        meta?: Record<any, any> | undefined;
-        model?: string | undefined;
-        provider?: BaseLLMProvider | undefined;
-        openRouterApiKey?: string | undefined;
-        promptSuffix?: string | undefined;
-        promptPrefix?: string | undefined;
-        mode?: "pointwise" | "pairwise" | undefined;
-        temperature?: number | undefined;
-        responseB?: {
-            sha256: string;
-            startedAt: number;
-            data: string;
-            modelName: string;
-            did: string;
-            cid: string;
-            provider: string;
-            modelId: string;
-            modelOwner: string;
-            modelHost: string;
-            prompt: {
-                type: "multiple-choice" | "order-sentences" | "text-replacement" | "typo" | "open-ended" | "open-ended-with-docs";
-                did: string;
-                question: {
-                    sha256: string;
-                    data: string;
-                    cid: string;
-                };
-                fullPrompt: {
-                    sha256: string;
-                    data: string;
-                    cid: string;
-                };
-                answer?: string | undefined;
-                metadata?: Record<string, any> | undefined;
-                options?: Record<string, string> | undefined;
-                answerKey?: string | undefined;
-                scorers?: string[] | undefined;
-            };
-            finishedAt: number;
-            runId: string;
-            metadata?: Record<string, any> | undefined;
-            inputTokensUsed?: number | undefined;
-            outputTokensUsed?: number | undefined;
-            inputCost?: string | undefined;
-            outputCost?: string | undefined;
-        } | undefined;
-    }>, {
-        model: string;
-        promptSuffix: string;
-        promptPrefix: string;
-        mode: "pointwise" | "pairwise";
-        criteria: {
-            id: string;
-            description: string;
-            weight: number;
-            scale: {
-                min: number;
-                max: number;
-            };
-        }[];
-        temperature: number;
-        meta?: Record<any, any> | undefined;
-        provider?: BaseLLMProvider | undefined;
-        openRouterApiKey?: string | undefined;
-        responseB?: {
-            sha256: string;
-            startedAt: number;
-            data: string;
-            modelName: string;
-            did: string;
-            cid: string;
-            provider: string;
-            modelId: string;
-            modelOwner: string;
-            modelHost: string;
-            prompt: {
-                type: "multiple-choice" | "order-sentences" | "text-replacement" | "typo" | "open-ended" | "open-ended-with-docs";
-                did: string;
-                question: {
-                    sha256: string;
-                    data: string;
-                    cid: string;
-                };
-                fullPrompt: {
-                    sha256: string;
-                    data: string;
-                    cid: string;
-                };
-                answer?: string | undefined;
-                metadata?: Record<string, any> | undefined;
-                options?: Record<string, string> | undefined;
-                answerKey?: string | undefined;
-                scorers?: string[] | undefined;
-            };
-            finishedAt: number;
-            runId: string;
-            metadata?: Record<string, any> | undefined;
-            inputTokensUsed?: number | undefined;
-            outputTokensUsed?: number | undefined;
-            inputCost?: string | undefined;
-            outputCost?: string | undefined;
-        } | undefined;
-    }, {
-        criteria: {
-            id: string;
-            description: string;
-            weight?: number | undefined;
-            scale?: {
-                min?: number | undefined;
-                max?: number | undefined;
-            } | undefined;
-        }[];
-        meta?: Record<any, any> | undefined;
-        model?: string | undefined;
-        provider?: BaseLLMProvider | undefined;
-        openRouterApiKey?: string | undefined;
-        promptSuffix?: string | undefined;
-        promptPrefix?: string | undefined;
-        mode?: "pointwise" | "pairwise" | undefined;
-        temperature?: number | undefined;
-        responseB?: {
-            sha256: string;
-            startedAt: number;
-            data: string;
-            modelName: string;
-            did: string;
-            cid: string;
-            provider: string;
-            modelId: string;
-            modelOwner: string;
-            modelHost: string;
-            prompt: {
-                type: "multiple-choice" | "order-sentences" | "text-replacement" | "typo" | "open-ended" | "open-ended-with-docs";
-                did: string;
-                question: {
-                    sha256: string;
-                    data: string;
-                    cid: string;
-                };
-                fullPrompt: {
-                    sha256: string;
-                    data: string;
-                    cid: string;
-                };
-                answer?: string | undefined;
-                metadata?: Record<string, any> | undefined;
-                options?: Record<string, string> | undefined;
-                answerKey?: string | undefined;
-                scorers?: string[] | undefined;
-            };
-            finishedAt: number;
-            runId: string;
-            metadata?: Record<string, any> | undefined;
-            inputTokensUsed?: number | undefined;
-            outputTokensUsed?: number | undefined;
-            inputCost?: string | undefined;
-            outputCost?: string | undefined;
-        } | undefined;
-    }>;
-    scoreOne(response: PromptResponse, options: z.input<typeof this.optionsSchema>): Promise<{
-        sha256: string;
-        startedAt: number;
-        modelName: string;
-        did: string;
-        cid: string;
-        provider: string;
-        modelId: string;
-        modelOwner: string;
-        modelHost: string;
-        finishedAt: number;
-        runId: string;
-        scoreDID: string;
-        score: number;
-        method: "human" | "ai" | "algo";
-        metadata?: Record<string, any> | undefined;
-        inputTokensUsed?: number | undefined;
-        outputTokensUsed?: number | undefined;
-        data?: string | undefined;
-        inputCost?: string | undefined;
-        outputCost?: string | undefined;
-        prompt?: {
-            did: string;
-            question: {
-                sha256: string;
-                cid: string;
-                data?: string | undefined;
-            };
-            fullPrompt: {
-                sha256: string;
-                cid: string;
-                data?: string | undefined;
-            };
-            answer?: string | undefined;
-            metadata?: Record<string, any> | undefined;
-            options?: Record<string, string> | undefined;
-            type?: "multiple-choice" | "order-sentences" | "text-replacement" | "typo" | "open-ended" | "open-ended-with-docs" | undefined;
-            answerKey?: string | undefined;
-            scorers?: string[] | undefined;
-        } | undefined;
-        scoreMetadata?: Record<string, any> | undefined;
-        explanation?: string | undefined;
-        scorerAI?: {
-            modelName: string;
-            provider: string;
-            modelId: string;
-            modelOwner: string;
-            modelHost: string;
-            inputTokensUsed?: number | undefined;
-            outputTokensUsed?: number | undefined;
-            inputCost?: string | undefined;
-            outputCost?: string | undefined;
-        } | undefined;
-    } | undefined>;
-    private systemPrompt;
-    private scorePointwise;
-    private scorePairwise;
-    /**
-     * Build the pairwise evaluation prompt
-     */
-    private buildPairwisePrompt;
-    /**
-     * Normalize criterion weights to sum to 1
-     */
-    private normalizeWeights;
-    /**
-     * Render criteria as a formatted string
-     */
-    private renderCriteria;
-    /**
-     * Compute overall score from per-criterion scores
-     */
-    private computeOverallScore;
-    /**
-     * Extract criteria IDs from prompt text (fallback)
-     */
-    private extractCriteria;
-    canScore(response: PromptResponse, options?: Record<string, any>): Promise<boolean>;
-}
-
-declare abstract class AbstractParser {
-    /**
-     * String identifier of the parser. Must be
-     * set to distunguish the parser from the others.
-     */
-    static readonly identifier: string;
-    /**
-     * Parses the given file
-     */
-    abstract parseFile(path: string): MaybePromise<ParseResult>;
-    /**
-     * Parses the given buffer or string
-     */
-    abstract parseContent(content: string | Uint8Array): MaybePromise<ParseResult>;
-    /**
-     * Returns the identifier of the parser.
-     */
-    getIdentifier(): string | undefined;
-}
-type ParseResult = {
-    prompts: Prompt[];
-    responses: PromptResponse[];
-    scores: PromptScore[];
-};
+}) => Promise<{
+    response: z__default.infer<TResponse>;
+    score?: z__default.infer<TScore>;
+}>;
+type InferRunConfig<TRunConfigSchema extends z__default.ZodRawShape> = z__default.infer<z__default.ZodObject<TRunConfigSchema>>;
 
-/**
- * Generic parser for the contents that formatted as JSON or JSONL array of objects.
- *
- * It uses the given builder functions to build the Prompt, Response,
- * and Score objects for the each object from the data that is parsed from the file/content.
- *
- * It also prevents having same Prompts with different IDs using the
- * `fullPrompt` CID calculation of the produced Prompt objects.
- */
-declare class GenericArrayParser extends AbstractParser {
-    static readonly identifier: string;
-    private promptBuilder;
-    private responseBuilder?;
-    private scoreBuilder?;
-    constructor(options: {
-        promptBuilder: GenericArrayParserPromptBuilder;
-        responseBuilder?: GenericArrayParserResponseBuilder;
-        scoreBuilder?: GenericArrayParserScoreBuilder;
+declare class PeerbenchError extends Error {
+    code: number;
+    constructor(message?: string, options?: ErrorOptions & {
+        code: number;
     });
-    parseFile(path: string): Promise<ParseResult>;
-    parseContent(content: string | Uint8Array): Promise<ParseResult>;
-}
-type GenericArrayParserPromptBuilder = (data: any, context: {
-    result: ParseResult;
-}) => MaybePromise<Prompt | undefined>;
-type GenericArrayParserResponseBuilder = (data: any, context: {
-    result: ParseResult;
-    /**
-     * The generated Prompt object for the current item if there is one.
-     */
-    prompt?: Prompt;
-}) => MaybePromise<PromptResponse | undefined>;
-type GenericArrayParserScoreBuilder = (data: any, context: {
-    result: ParseResult;
-    /**
-     * The generated Prompt object for the current item if there is one.
-     */
-    prompt?: Prompt;
-    /**
-     * The generated Response object for the current item if there is one.
-     */
-    response?: PromptResponse;
-}) => MaybePromise<PromptScore | undefined>;
-
-declare class MedQAParser extends AbstractParser {
-    static readonly identifier = "medqa";
-    parseFile(path: string): Promise<ParseResult>;
-    parseContent(content: string | Uint8Array): Promise<ParseResult>;
-}
-
-declare class MMLUProParser extends AbstractParser {
-    static readonly identifier = "mmlu-pro";
-    parseFile(path: string): Promise<ParseResult>;
-    parseContent(content: string | Uint8Array): Promise<ParseResult>;
 }
 
-/**
- * Parser for the standard PeerBench format.
- *
- * The given data must be formatted as JSON or JSONL array of objects.
- *
- * Each object can be one of the following:
- *  - Prompt,
- *  - Response,
- *  - Score
- *
- * Prompts that are included within the Response or Score objects are counted as Prompts in the result (deduplicated).
- */
-declare class PBParser extends GenericArrayParser {
-    static readonly identifier: string;
-    constructor();
-}
+declare function captureStackTrace(error: Error, constructor: Function): void;
 
-/**
- * Parser class for automatically detect and parse the data
- * with the appropriate parser.
- */
-declare class DataParser {
-    static parsers: Map<string, AbstractParser>;
-    /**
-     * Tries to parse data from a file path with one of the available parsers.
-     * @example
-     * ```typescript
-     * const { result, parser } = await DataParser.parseFile("path/to/file.json");
-     * ```
-     */
-    static parseFile(path: string): Promise<{
-        result: ParseResult;
-        parser: AbstractParser;
-    }>;
-    /**
-     * Tries to parse data from a string or ArrayBuffer content with one of the available parsers.
-     * @example
-     * ```typescript
-     * // With string content
-     * const { result, parser } = await DataParser.parseContent('{"question": "What is...", ...}');
-     *
-     * // With file upload in browser
-     * const file = event.target.files[0];
-     * const content = await file.arrayBuffer();
-     * const { result, parser } = await DataParser.parseContent(content);
-     * ```
-     */
-    static parseContent(content: string | Uint8Array): Promise<{
-        result: ParseResult;
-        parser: AbstractParser;
-    }>;
-}
-
-/**
- * Parses JSONL formatted string into an array
- * @returns An array of parsed JSON lines
- */
-declare function parseJSONL<T>(str: string, options?: {
-    errorOnInvalid?: boolean;
-}): T[];
-/**
- * Tries to parse the given string as JSON.
- * Returns `undefined` if it is not a valid JSON entity.
- */
-declare function tryParseJson<T = any>(content: string): T | undefined;
-/**
- * Stringifies the given JSON value using `safe-stable-stringify` in a stable manner.
- * This stable method generates the same string output for the same input value (including objects).
- */
-declare function stableStringify(value: any): string | undefined;
-
-/**
- * Calculates the CID of the given data
- */
-declare function calculateCID(data: unknown): Promise<CID<unknown, 512, 18, 1>>;
-
-declare function sleep(ms: number): Promise<void>;
-
-/**
- * Reads the file and returns the content as a string.
- * @param path - The path to the file.
- * @returns The content of the file as a string.
- */
-declare function readFile(path: string): Promise<Uint8Array>;
-
-/**
- * Prepares the full Prompt that is going to be sent to the model
- */
-declare function preparePrompt(question: string, options?: PromptOptions): string;
-
-/**
- * Calculates the SHA-256 hash of the given text or byte array.
- * @param input - The text or byte array to calculate the hash of.
- * @returns The SHA-256 hash of the given input.
- */
-declare function calculateSHA256(input: string | Uint8Array): Promise<string>;
-
-/**
- * Removes the `did:<entity type>:` prefix from the given string and returns the rest.
- */
-declare function removeDIDPrefix(did: string): string;
+declare const PEERBENCH_NAMESPACE: "peerbench.ai";
+declare const CATEGORIES: {
+    LLM: string;
+};
 
-/**
- * Formats the given milliseconds into a human-readable time string.
- * @param ms The time in milliseconds
- */
-declare function formatMs(ms: number, options?: {
-    /**
-     * When true, includes all time units (days, hours, minutes, seconds, ms) in the output
-     */
-    full?: boolean;
+declare function defineRunner<const TProviders extends ProviderCtor[], const TScorers extends ScorerCtor[], const TSchemaSets extends SchemaSetDefinition[], const TRunConfigSchema extends z__default.ZodRawShape = {}>(config: {
+    schemaSets: TSchemaSets;
+    providers: TProviders;
+    scorers: TScorers;
+    runConfigSchema?: TRunConfigSchema;
     /**
-     * Indicates which time units to include in the output. Defaults to ["millisecond", "second", "minute", "hour", "day"]. Only used if `full` option is true.
+     * @default true
      */
-    include?: ("millisecond" | "hour" | "second" | "minute" | "day")[];
-}): string;
-/**
- * Converts a byte array to a string using the TextDecoder
- * @param buffer - The byte array to convert
- * @returns The string representation of the byte array
- */
-declare function bufferToString(buffer: Uint8Array): string;
-/**
- * Converts a string to a byte array using the TextEncoder
- * @param str - The string to convert
- * @returns The byte array representation of the string
- */
-declare function stringToBuffer(str: string): Uint8Array;
-/**
- * Python like string formatter. Replaces the `{key}` with the value from the `values` object.
- */
-declare function formatString(str: string, values: Record<string, string>): string;
-
-/**
- * Tries to repair and parse LLM response as a JSON object. LLM must
- * be configured to return a JSON object. This function only helps to
- * get rid out of some additional formatting (e.g. ```json).
- */
-declare function parseResponseAsJSON<T>(response: string): T | undefined;
-/**
- * Extracts the first JSON formatted part from the response
- */
-declare function extractJSONFromResponse(response: string): string | undefined;
-
-/**
- * Extracts the first error message (if there is any)
- * from a Zod safe parse result and format it.
- * @param safeParseReturn
- * @param path Path of the parsing object. It will be used to indicate the invalid field if the info is not available in the validation error.
- */
-declare function parseValidationError<T, K>(safeParseReturn: z.SafeParseReturnType<T, K>, path?: string): string | undefined;
-/**
- * Checks the error state of the given Zod safe parse result
- * and throws an error if there is any.
- * @param safeParseReturn
- * @param path Path of the parsing object. It will be used to indicate the invalid field if the info is not available in the validation error.
- */
-declare function checkValidationError<T, K>(safeParseReturn: z.SafeParseReturnType<T, K>, path?: string): K;
-
-/**
- * Builds a valid Prompt object from the given parameters.
- */
-declare function buildPrompt(params: {
-    did?: string;
-    question: string;
-    fullPrompt?: string;
-    options?: Record<string, string>;
-    answer?: string;
-    answerKey?: string;
-    type: PromptType;
-    metadata?: Record<string, any>;
-    scorers?: string[];
-}): Promise<{
-    type: "multiple-choice" | "order-sentences" | "text-replacement" | "typo" | "open-ended" | "open-ended-with-docs";
-    did: string;
-    question: {
-        sha256: string;
-        data: string;
-        cid: string;
+    parseRunConfig?: boolean;
+    defaults?: {
+        scorer?: InstanceType<TScorers[number]>;
+        responseIdGenerator?: IdGenerator;
+        scoreIdGenerator?: IdGenerator;
     };
-    fullPrompt: {
-        sha256: string;
-        data: string;
-        cid: string;
-    };
-    answer?: string | undefined;
-    metadata?: Record<string, any> | undefined;
-    options?: Record<string, string> | undefined;
-    answerKey?: string | undefined;
-    scorers?: string[] | undefined;
+}, fn: Runner<TSchemaSets[number]["testCase"], TSchemaSets[number]["response"], TSchemaSets[number]["score"], InstanceType<TProviders[number]>, InstanceType<TScorers[number]>, InferRunConfig<TRunConfigSchema>>): (params: Parameters<typeof fn>[0]) => Promise<{
+    response: z__default.core.output<TSchemaSets[number]["response"]>;
+    score?: z__default.core.output<TSchemaSets[number]["score"]> | undefined;
 }>;
-/**
- * Builds a valid PromptResponse object from the given parameters.
- */
-declare function buildResponse(params: {
-    prompt: Prompt;
-    forwardResponse: ForwardResponse;
-    provider: string;
-    modelId: string;
-    did?: string;
-    modelName?: string;
-    modelOwner?: string;
-    modelHost?: string;
-    runId?: string;
-    metadata?: Record<string, any>;
-}): Promise<{
-    sha256: string;
-    startedAt: number;
-    data: string;
-    modelName: string;
-    did: string;
-    cid: string;
-    provider: string;
-    modelId: string;
-    modelOwner: string;
-    modelHost: string;
-    prompt: {
-        type: "multiple-choice" | "order-sentences" | "text-replacement" | "typo" | "open-ended" | "open-ended-with-docs";
-        did: string;
-        question: {
-            sha256: string;
-            data: string;
-            cid: string;
-        };
-        fullPrompt: {
-            sha256: string;
-            data: string;
-            cid: string;
-        };
-        answer?: string | undefined;
-        metadata?: Record<string, any> | undefined;
-        options?: Record<string, string> | undefined;
-        answerKey?: string | undefined;
-        scorers?: string[] | undefined;
-    };
-    finishedAt: number;
-    runId: string;
-    metadata?: Record<string, any> | undefined;
-    inputTokensUsed?: number | undefined;
-    outputTokensUsed?: number | undefined;
-    inputCost?: string | undefined;
-    outputCost?: string | undefined;
-}>;
-/**
- * Builds a valid PromptScore object from the given parameters.
- */
-declare function buildScore(params: Omit<z.input<typeof PromptScoreSchema>, "scoreDID"> & {
-    scoreDID?: string;
-}): Promise<{
-    sha256: string;
-    startedAt: number;
-    modelName: string;
-    did: string;
-    cid: string;
-    provider: string;
-    modelId: string;
-    modelOwner: string;
-    modelHost: string;
-    finishedAt: number;
-    runId: string;
-    scoreDID: string;
-    score: number;
-    method: "human" | "ai" | "algo";
-    metadata?: Record<string, any> | undefined;
-    inputTokensUsed?: number | undefined;
-    outputTokensUsed?: number | undefined;
-    data?: string | undefined;
-    inputCost?: string | undefined;
-    outputCost?: string | undefined;
-    prompt?: {
-        did: string;
-        question: {
-            sha256: string;
-            cid: string;
-            data?: string | undefined;
-        };
-        fullPrompt: {
-            sha256: string;
-            cid: string;
-            data?: string | undefined;
-        };
-        answer?: string | undefined;
-        metadata?: Record<string, any> | undefined;
-        options?: Record<string, string> | undefined;
-        type?: "multiple-choice" | "order-sentences" | "text-replacement" | "typo" | "open-ended" | "open-ended-with-docs" | undefined;
-        answerKey?: string | undefined;
-        scorers?: string[] | undefined;
-    } | undefined;
-    scoreMetadata?: Record<string, any> | undefined;
-    explanation?: string | undefined;
-    scorerAI?: {
-        modelName: string;
-        provider: string;
-        modelId: string;
-        modelOwner: string;
-        modelHost: string;
-        inputTokensUsed?: number | undefined;
-        outputTokensUsed?: number | undefined;
-        inputCost?: string | undefined;
-        outputCost?: string | undefined;
-    } | undefined;
-}>;
-
-/**
- * Parses the given string as an enum value based on either
- * the keys or values of the given enum.
- * @param lookFor - Whether to look for the key or value
- */
-declare function EnumSchema<T>(_enum: Record<string, T>, lookFor?: "key" | "value"): z.ZodEffects<z.ZodString, NonNullable<T>, string>;
-
-declare const MULTIPLE_CHOICE_SYSTEM_PROMPT = "Your explanation can't be longer than 400 tokens. The last sentence must be formatted as one of the following:\n- The answer is <answer letter>\n- The answer is **<answer letter>**\n- <answer letter>: ...\n- <answer letter>) ...\nReplace <answer letter> with the letter of your chosen answer.\n\nUse the following string as your last sentence if you are not capable of answering the question:\n<!NO ANSWER!>";
-declare const SENTENCE_REORDER_SYSTEM_PROMPT = "Your task is ordering the given sentences (each line is a sentence) in a correct order. Your output must be formatted as the input but with the sentences in the correct order. Markdown formatting is forbidden.";
-declare const TEXT_REPLACEMENT_SYSTEM_PROMPT = "Your task is placing all the entities that are provided in the ENTITIES section to the input text in a correct order. Your output only and only includes the modified text, nothing else. It is forbidden to modify anything else from the input text. Markdown formatting is forbidden too.";
-declare const TYPO_SYSTEM_PROMPT = "Your task is to find all the typos in the given text. Your output must include the corrected text, nothing else.";
-declare const OPEN_ENDED_SYSTEM_PROMPT = "You are a knowledgeable expert. Please provide a clear, accurate, short and well-reasoned answer to the following question. Be concise but comprehensive in your response. Your answer must be short and clear with less than 20 words";
+type SchemaSetDefinition<TTestCase extends z__default.ZodObject = z__default.ZodObject, TResponse extends z__default.ZodObject = z__default.ZodObject, TScore extends z__default.ZodObject = z__default.ZodObject> = {
+    testCase: TTestCase;
+    response: TResponse;
+    score: TScore;
+};
 
-export { AbstractCollector, AbstractGenerator, AbstractParser, AbstractProvider, AbstractRSSCollector, AbstractRegistry, AbstractScorer, AnthropicModels, BaseLLMProvider, type BaseLLMProviderForwardOptions, type BaseLLMProviderOptions, DataParser, DeepSeekModels, EnumSchema, ExactMatchScorer, ForwardError, type ForwardResponse, GenericArrayParser, GoogleModels, InvalidDataError, LLMJudgeScorer, LargeLanguageModel, LargeLanguageModelOwner, type LargeLanguageModelOwnerType, type LargeLanguageModelType, MCQGenerator, MMLUProParser, MULTIPLE_CHOICE_SYSTEM_PROMPT, type MaybePromise, MedQAParser, MetaModels, MistralModels, type ModelInfo, MultipleChoiceScorer, NearAIProvider, type NearAIProviderOptions, OPEN_ENDED_SYSTEM_PROMPT, OpenAIModels, OpenEndedGenerator, OpenRouterProvider, type OpenRouterProviderOptions, PBParser, PEERBENCH_ERROR_CODES, ParagraphMergeStrategy, type ParagraphMergeStrategyType, ParserIsNotCompatibleError, type PeerBenchErrorCode, PeerBenchRegistry, type Prompt, type PromptOptions, type PromptResponse, PromptResponseSchema, PromptSchema, type PromptScore, PromptScoreSchema, type PromptType, PromptTypes, type ProviderResponse, type PubMedCollectedData, PubMedCollector, QwenModels, RefAnswerEqualityLLMJudgeScorer, SENTENCE_REORDER_SYSTEM_PROMPT, type ScoringMethod, ScoringMethods, SimilarityScorer, type SimpleGeneralRSSCollectedData, SimpleGeneralRSSCollector, StringCollector, TEXT_REPLACEMENT_SYSTEM_PROMPT, TRPGenerator, TYPO_SYSTEM_PROMPT, type Task, TaskSchema, XAIModels, bufferToString, buildPrompt, buildResponse, buildScore, calculateCID, calculateSHA256, checkValidationError, extractJSONFromResponse, formatMs, formatString, paragraphMerge, parseJSONL, parseResponseAsJSON, parseValidationError, preparePrompt, readFile, removeDIDPrefix, sleep, stableStringify, stringToBuffer, tryParseJson };
+export { CATEGORIES, IdGenerator, type InferRunConfig, PEERBENCH_NAMESPACE, PeerbenchError, ProviderCtor, type Runner, ScorerCtor, bufferToString, captureStackTrace, defineRunner, idGeneratorUUIDv7, parseResponseAsJSON, sleep, stringToBuffer };