promptfoo 0.91.2 → 0.92.0

package/dist/test/factories/evalFactory.d.ts

@@ -0,0 +1,768 @@
+import Eval from '../../src/models/eval';
+export default class EvalFactory {
+    static create(): Promise<Eval>;
+    static createOldResult(): Promise<{
+        id: string;
+        config: Partial<{
+            prompts: string | Record<string, string> | (string | {
+                raw: string;
+                label: string;
+                function?: ((args_0: {
+                    vars: Record<string, any>;
+                    provider?: import("../../src").ApiProvider | undefined;
+                }, ...args: unknown[]) => Promise<any>) | undefined;
+                id?: string | undefined;
+                config?: any;
+                display?: string | undefined;
+            } | {
+                id: string;
+                raw?: string | undefined;
+                label?: string | undefined;
+            })[];
+            providers: string | (((args_0: string, args_1: {
+                vars: Record<string, string | {}>;
+                prompt: {
+                    raw: string;
+                    label: string;
+                    function?: ((args_0: {
+                        vars: Record<string, any>;
+                        provider?: import("../../src").ApiProvider | undefined;
+                    }, ...args: unknown[]) => Promise<any>) | undefined;
+                    id?: string | undefined;
+                    config?: any;
+                    display?: string | undefined;
+                };
+                fetchWithCache?: any;
+                filters?: Record<string, (args_0: any, ...args: any[]) => string> | undefined;
+                getCache?: any;
+                logger?: any;
+                originalProvider?: any;
+            } | undefined, args_2: {
+                includeLogProbs?: boolean | undefined;
+            } | undefined, ...args: unknown[]) => Promise<import("../../src").ProviderResponse>) & {
+                label?: string | undefined;
+            }) | (string | {
+                id?: string | undefined;
+                config?: any;
+                label?: string | undefined;
+                prompts?: string[] | undefined;
+                transform?: string | undefined;
+                delay?: number | undefined;
+                env?: {
+                    AI21_API_BASE_URL?: string | undefined;
+                    AI21_API_KEY?: string | undefined;
+                    ANTHROPIC_API_KEY?: string | undefined;
+                    AWS_BEDROCK_REGION?: string | undefined;
+                    AZURE_OPENAI_API_BASE_URL?: string | undefined;
+                    AZURE_OPENAI_API_HOST?: string | undefined;
+                    AZURE_OPENAI_API_KEY?: string | undefined;
+                    AZURE_OPENAI_BASE_URL?: string | undefined;
+                    BAM_API_HOST?: string | undefined;
+                    BAM_API_KEY?: string | undefined;
+                    CLOUDFLARE_ACCOUNT_ID?: string | undefined;
+                    CLOUDFLARE_API_KEY?: string | undefined;
+                    COHERE_API_KEY?: string | undefined;
+                    GOOGLE_API_HOST?: string | undefined;
+                    GOOGLE_API_KEY?: string | undefined;
+                    GROQ_API_KEY?: string | undefined;
+                    LOCALAI_BASE_URL?: string | undefined;
+                    MISTRAL_API_BASE_URL?: string | undefined;
+                    MISTRAL_API_HOST?: string | undefined;
+                    MISTRAL_API_KEY?: string | undefined;
+                    OPENAI_API_BASE_URL?: string | undefined;
+                    OPENAI_API_HOST?: string | undefined;
+                    OPENAI_API_KEY?: string | undefined;
+                    OPENAI_BASE_URL?: string | undefined;
+                    OPENAI_ORGANIZATION?: string | undefined;
+                    PALM_API_HOST?: string | undefined;
+                    PALM_API_KEY?: string | undefined;
+                    REPLICATE_API_KEY?: string | undefined;
+                    REPLICATE_API_TOKEN?: string | undefined;
+                    VERTEX_API_HOST?: string | undefined;
+                    VERTEX_API_KEY?: string | undefined;
+                    VERTEX_PROJECT_ID?: string | undefined;
+                    VERTEX_PUBLISHER?: string | undefined;
+                    VERTEX_REGION?: string | undefined;
+                } | undefined;
+            } | (((args_0: string, args_1: {
+                vars: Record<string, string | {}>;
+                prompt: {
+                    raw: string;
+                    label: string;
+                    function?: ((args_0: {
+                        vars: Record<string, any>;
+                        provider?: import("../../src").ApiProvider | undefined;
+                    }, ...args: unknown[]) => Promise<any>) | undefined;
+                    id?: string | undefined;
+                    config?: any;
+                    display?: string | undefined;
+                };
+                fetchWithCache?: any;
+                filters?: Record<string, (args_0: any, ...args: any[]) => string> | undefined;
+                getCache?: any;
+                logger?: any;
+                originalProvider?: any;
+            } | undefined, args_2: {
+                includeLogProbs?: boolean | undefined;
+            } | undefined, ...args: unknown[]) => Promise<import("../../src").ProviderResponse>) & {
+                label?: string | undefined;
+            }) | Record<string, {
+                id?: string | undefined;
+                config?: any;
+                label?: string | undefined;
+                prompts?: string[] | undefined;
+                transform?: string | undefined;
+                delay?: number | undefined;
+                env?: {
+                    AI21_API_BASE_URL?: string | undefined;
+                    AI21_API_KEY?: string | undefined;
+                    ANTHROPIC_API_KEY?: string | undefined;
+                    AWS_BEDROCK_REGION?: string | undefined;
+                    AZURE_OPENAI_API_BASE_URL?: string | undefined;
+                    AZURE_OPENAI_API_HOST?: string | undefined;
+                    AZURE_OPENAI_API_KEY?: string | undefined;
+                    AZURE_OPENAI_BASE_URL?: string | undefined;
+                    BAM_API_HOST?: string | undefined;
+                    BAM_API_KEY?: string | undefined;
+                    CLOUDFLARE_ACCOUNT_ID?: string | undefined;
+                    CLOUDFLARE_API_KEY?: string | undefined;
+                    COHERE_API_KEY?: string | undefined;
+                    GOOGLE_API_HOST?: string | undefined;
+                    GOOGLE_API_KEY?: string | undefined;
+                    GROQ_API_KEY?: string | undefined;
+                    LOCALAI_BASE_URL?: string | undefined;
+                    MISTRAL_API_BASE_URL?: string | undefined;
+                    MISTRAL_API_HOST?: string | undefined;
+                    MISTRAL_API_KEY?: string | undefined;
+                    OPENAI_API_BASE_URL?: string | undefined;
+                    OPENAI_API_HOST?: string | undefined;
+                    OPENAI_API_KEY?: string | undefined;
+                    OPENAI_BASE_URL?: string | undefined;
+                    OPENAI_ORGANIZATION?: string | undefined;
+                    PALM_API_HOST?: string | undefined;
+                    PALM_API_KEY?: string | undefined;
+                    REPLICATE_API_KEY?: string | undefined;
+                    REPLICATE_API_TOKEN?: string | undefined;
+                    VERTEX_API_HOST?: string | undefined;
+                    VERTEX_API_KEY?: string | undefined;
+                    VERTEX_PROJECT_ID?: string | undefined;
+                    VERTEX_PUBLISHER?: string | undefined;
+                    VERTEX_REGION?: string | undefined;
+                } | undefined;
+            }>)[];
+            env?: {
+                AI21_API_BASE_URL?: string | undefined;
+                AI21_API_KEY?: string | undefined;
+                ANTHROPIC_API_KEY?: string | undefined;
+                AWS_BEDROCK_REGION?: string | undefined;
+                AZURE_OPENAI_API_BASE_URL?: string | undefined;
+                AZURE_OPENAI_API_HOST?: string | undefined;
+                AZURE_OPENAI_API_KEY?: string | undefined;
+                AZURE_OPENAI_BASE_URL?: string | undefined;
+                BAM_API_HOST?: string | undefined;
+                BAM_API_KEY?: string | undefined;
+                CLOUDFLARE_ACCOUNT_ID?: string | undefined;
+                CLOUDFLARE_API_KEY?: string | undefined;
+                COHERE_API_KEY?: string | undefined;
+                GOOGLE_API_HOST?: string | undefined;
+                GOOGLE_API_KEY?: string | undefined;
+                GROQ_API_KEY?: string | undefined;
+                LOCALAI_BASE_URL?: string | undefined;
+                MISTRAL_API_BASE_URL?: string | undefined;
+                MISTRAL_API_HOST?: string | undefined;
+                MISTRAL_API_KEY?: string | undefined;
+                OPENAI_API_BASE_URL?: string | undefined;
+                OPENAI_API_HOST?: string | undefined;
+                OPENAI_API_KEY?: string | undefined;
+                OPENAI_BASE_URL?: string | undefined;
+                OPENAI_ORGANIZATION?: string | undefined;
+                PALM_API_HOST?: string | undefined;
+                PALM_API_KEY?: string | undefined;
+                REPLICATE_API_KEY?: string | undefined;
+                REPLICATE_API_TOKEN?: string | undefined;
+                VERTEX_API_HOST?: string | undefined;
+                VERTEX_API_KEY?: string | undefined;
+                VERTEX_PROJECT_ID?: string | undefined;
+                VERTEX_PUBLISHER?: string | undefined;
+                VERTEX_REGION?: string | undefined;
+            } | undefined;
+            metadata?: Record<string, any> | undefined;
+            description?: string | undefined;
+            tests?: string | (string | {
+                options?: ({
+                    prefix?: string | undefined;
+                    suffix?: string | undefined;
+                } & {
+                    transform?: string | undefined;
+                    postprocess?: string | undefined;
+                    transformVars?: string | undefined;
+                    storeOutputAs?: string | undefined;
+                } & {
+                    provider?: any;
+                    rubricPrompt?: string | string[] | {
+                        role: string;
+                        content: string;
+                    }[] | undefined;
+                    factuality?: {
+                        subset?: number | undefined;
+                        superset?: number | undefined;
+                        agree?: number | undefined;
+                        disagree?: number | undefined;
+                        differButFactual?: number | undefined;
+                    } | undefined;
+                } & {
+                    disableVarExpansion?: boolean | undefined;
+                    disableConversationVar?: boolean | undefined;
+                    runSerially?: boolean | undefined;
+                }) | undefined;
+                vars?: Record<string, string | any[] | string[] | {}> | undefined;
+                provider?: string | {
+                    id?: string | undefined;
+                    config?: any;
+                    label?: string | undefined;
+                    prompts?: string[] | undefined;
+                    transform?: string | undefined;
+                    delay?: number | undefined;
+                    env?: {
+                        AI21_API_BASE_URL?: string | undefined;
+                        AI21_API_KEY?: string | undefined;
+                        ANTHROPIC_API_KEY?: string | undefined;
+                        AWS_BEDROCK_REGION?: string | undefined;
+                        AZURE_OPENAI_API_BASE_URL?: string | undefined;
+                        AZURE_OPENAI_API_HOST?: string | undefined;
+                        AZURE_OPENAI_API_KEY?: string | undefined;
+                        AZURE_OPENAI_BASE_URL?: string | undefined;
+                        BAM_API_HOST?: string | undefined;
+                        BAM_API_KEY?: string | undefined;
+                        CLOUDFLARE_ACCOUNT_ID?: string | undefined;
+                        CLOUDFLARE_API_KEY?: string | undefined;
+                        COHERE_API_KEY?: string | undefined;
+                        GOOGLE_API_HOST?: string | undefined;
+                        GOOGLE_API_KEY?: string | undefined;
+                        GROQ_API_KEY?: string | undefined;
+                        LOCALAI_BASE_URL?: string | undefined;
+                        MISTRAL_API_BASE_URL?: string | undefined;
+                        MISTRAL_API_HOST?: string | undefined;
+                        MISTRAL_API_KEY?: string | undefined;
+                        OPENAI_API_BASE_URL?: string | undefined;
+                        OPENAI_API_HOST?: string | undefined;
+                        OPENAI_API_KEY?: string | undefined;
+                        OPENAI_BASE_URL?: string | undefined;
+                        OPENAI_ORGANIZATION?: string | undefined;
+                        PALM_API_HOST?: string | undefined;
+                        PALM_API_KEY?: string | undefined;
+                        REPLICATE_API_KEY?: string | undefined;
+                        REPLICATE_API_TOKEN?: string | undefined;
+                        VERTEX_API_HOST?: string | undefined;
+                        VERTEX_API_KEY?: string | undefined;
+                        VERTEX_PROJECT_ID?: string | undefined;
+                        VERTEX_PUBLISHER?: string | undefined;
+                        VERTEX_REGION?: string | undefined;
+                    } | undefined;
+                } | {
+                    callApi: import("../../src").CallApiFunction;
+                    id: (...args: unknown[]) => string;
+                    config?: any;
+                    label?: string | undefined;
+                    transform?: string | undefined;
+                    delay?: number | undefined;
+                    callEmbeddingApi?: ((args_0: string, ...args: unknown[]) => Promise<import("../../src").ProviderEmbeddingResponse>) | undefined;
+                    callClassificationApi?: ((args_0: string, ...args: unknown[]) => Promise<import("../../src").ProviderClassificationResponse>) | undefined;
+                } | undefined;
+                metadata?: Record<string, any> | undefined;
+                description?: string | undefined;
+                providerOutput?: string | {} | undefined;
+                assert?: ({
+                    type: import("../../src").AssertionType;
+                    config?: Record<string, any> | undefined;
+                    value?: import("../../src").AssertionValue | undefined;
+                    provider?: any;
+                    transform?: string | undefined;
+                    rubricPrompt?: string | string[] | {
+                        role: string;
+                        content: string;
+                    }[] | undefined;
+                    threshold?: number | undefined;
+                    weight?: number | undefined;
+                    metric?: string | undefined;
+                } | {
+                    type: "assert-set";
+                    assert: {
+                        type: import("../../src").AssertionType;
+                        config?: Record<string, any> | undefined;
+                        value?: import("../../src").AssertionValue | undefined;
+                        provider?: any;
+                        transform?: string | undefined;
+                        rubricPrompt?: string | string[] | {
+                            role: string;
+                            content: string;
+                        }[] | undefined;
+                        threshold?: number | undefined;
+                        weight?: number | undefined;
+                        metric?: string | undefined;
+                    }[];
+                    threshold?: number | undefined;
+                    weight?: number | undefined;
+                    metric?: string | undefined;
+                })[] | undefined;
+                threshold?: number | undefined;
+            })[] | undefined;
+            redteam?: import("../../src").RedteamFileConfig | undefined;
+            tags?: Record<string, string> | undefined;
+            scenarios?: (string | {
+                config: {
+                    options?: ({
+                        prefix?: string | undefined;
+                        suffix?: string | undefined;
+                    } & {
+                        transform?: string | undefined;
+                        postprocess?: string | undefined;
+                        transformVars?: string | undefined;
+                        storeOutputAs?: string | undefined;
+                    } & {
+                        provider?: any;
+                        rubricPrompt?: string | string[] | {
+                            role: string;
+                            content: string;
+                        }[] | undefined;
+                        factuality?: {
+                            subset?: number | undefined;
+                            superset?: number | undefined;
+                            agree?: number | undefined;
+                            disagree?: number | undefined;
+                            differButFactual?: number | undefined;
+                        } | undefined;
+                    } & {
+                        disableVarExpansion?: boolean | undefined;
+                        disableConversationVar?: boolean | undefined;
+                        runSerially?: boolean | undefined;
+                    }) | undefined;
+                    vars?: Record<string, string | any[] | string[] | {}> | undefined;
+                    provider?: string | {
+                        id?: string | undefined;
+                        config?: any;
+                        label?: string | undefined;
+                        prompts?: string[] | undefined;
+                        transform?: string | undefined;
+                        delay?: number | undefined;
+                        env?: {
+                            AI21_API_BASE_URL?: string | undefined;
+                            AI21_API_KEY?: string | undefined;
+                            ANTHROPIC_API_KEY?: string | undefined;
+                            AWS_BEDROCK_REGION?: string | undefined;
+                            AZURE_OPENAI_API_BASE_URL?: string | undefined;
+                            AZURE_OPENAI_API_HOST?: string | undefined;
+                            AZURE_OPENAI_API_KEY?: string | undefined;
+                            AZURE_OPENAI_BASE_URL?: string | undefined;
+                            BAM_API_HOST?: string | undefined;
+                            BAM_API_KEY?: string | undefined;
+                            CLOUDFLARE_ACCOUNT_ID?: string | undefined;
+                            CLOUDFLARE_API_KEY?: string | undefined;
+                            COHERE_API_KEY?: string | undefined;
+                            GOOGLE_API_HOST?: string | undefined;
+                            GOOGLE_API_KEY?: string | undefined;
+                            GROQ_API_KEY?: string | undefined;
+                            LOCALAI_BASE_URL?: string | undefined;
+                            MISTRAL_API_BASE_URL?: string | undefined;
+                            MISTRAL_API_HOST?: string | undefined;
+                            MISTRAL_API_KEY?: string | undefined;
+                            OPENAI_API_BASE_URL?: string | undefined;
+                            OPENAI_API_HOST?: string | undefined;
+                            OPENAI_API_KEY?: string | undefined;
+                            OPENAI_BASE_URL?: string | undefined;
+                            OPENAI_ORGANIZATION?: string | undefined;
+                            PALM_API_HOST?: string | undefined;
+                            PALM_API_KEY?: string | undefined;
+                            REPLICATE_API_KEY?: string | undefined;
+                            REPLICATE_API_TOKEN?: string | undefined;
+                            VERTEX_API_HOST?: string | undefined;
+                            VERTEX_API_KEY?: string | undefined;
+                            VERTEX_PROJECT_ID?: string | undefined;
+                            VERTEX_PUBLISHER?: string | undefined;
+                            VERTEX_REGION?: string | undefined;
+                        } | undefined;
+                    } | {
+                        callApi: import("../../src").CallApiFunction;
+                        id: (...args: unknown[]) => string;
+                        config?: any;
+                        label?: string | undefined;
+                        transform?: string | undefined;
+                        delay?: number | undefined;
+                        callEmbeddingApi?: ((args_0: string, ...args: unknown[]) => Promise<import("../../src").ProviderEmbeddingResponse>) | undefined;
+                        callClassificationApi?: ((args_0: string, ...args: unknown[]) => Promise<import("../../src").ProviderClassificationResponse>) | undefined;
+                    } | undefined;
+                    metadata?: Record<string, any> | undefined;
+                    description?: string | undefined;
+                    providerOutput?: string | {} | undefined;
+                    assert?: ({
+                        type: import("../../src").AssertionType;
+                        config?: Record<string, any> | undefined;
+                        value?: import("../../src").AssertionValue | undefined;
+                        provider?: any;
+                        transform?: string | undefined;
+                        rubricPrompt?: string | string[] | {
+                            role: string;
+                            content: string;
+                        }[] | undefined;
+                        threshold?: number | undefined;
+                        weight?: number | undefined;
+                        metric?: string | undefined;
+                    } | {
+                        type: "assert-set";
+                        assert: {
+                            type: import("../../src").AssertionType;
+                            config?: Record<string, any> | undefined;
+                            value?: import("../../src").AssertionValue | undefined;
+                            provider?: any;
+                            transform?: string | undefined;
+                            rubricPrompt?: string | string[] | {
+                                role: string;
+                                content: string;
+                            }[] | undefined;
+                            threshold?: number | undefined;
+                            weight?: number | undefined;
+                            metric?: string | undefined;
+                        }[];
+                        threshold?: number | undefined;
+                        weight?: number | undefined;
+                        metric?: string | undefined;
+                    })[] | undefined;
+                    threshold?: number | undefined;
+                }[];
+                tests: {
+                    options?: ({
+                        prefix?: string | undefined;
+                        suffix?: string | undefined;
+                    } & {
+                        transform?: string | undefined;
+                        postprocess?: string | undefined;
+                        transformVars?: string | undefined;
+                        storeOutputAs?: string | undefined;
+                    } & {
+                        provider?: any;
+                        rubricPrompt?: string | string[] | {
+                            role: string;
+                            content: string;
+                        }[] | undefined;
+                        factuality?: {
+                            subset?: number | undefined;
+                            superset?: number | undefined;
+                            agree?: number | undefined;
+                            disagree?: number | undefined;
+                            differButFactual?: number | undefined;
+                        } | undefined;
+                    } & {
+                        disableVarExpansion?: boolean | undefined;
+                        disableConversationVar?: boolean | undefined;
+                        runSerially?: boolean | undefined;
+                    }) | undefined;
+                    vars?: Record<string, string | any[] | string[] | {}> | undefined;
+                    provider?: string | {
+                        id?: string | undefined;
+                        config?: any;
+                        label?: string | undefined;
+                        prompts?: string[] | undefined;
+                        transform?: string | undefined;
+                        delay?: number | undefined;
+                        env?: {
+                            AI21_API_BASE_URL?: string | undefined;
+                            AI21_API_KEY?: string | undefined;
+                            ANTHROPIC_API_KEY?: string | undefined;
+                            AWS_BEDROCK_REGION?: string | undefined;
+                            AZURE_OPENAI_API_BASE_URL?: string | undefined;
+                            AZURE_OPENAI_API_HOST?: string | undefined;
+                            AZURE_OPENAI_API_KEY?: string | undefined;
+                            AZURE_OPENAI_BASE_URL?: string | undefined;
+                            BAM_API_HOST?: string | undefined;
+                            BAM_API_KEY?: string | undefined;
+                            CLOUDFLARE_ACCOUNT_ID?: string | undefined;
+                            CLOUDFLARE_API_KEY?: string | undefined;
+                            COHERE_API_KEY?: string | undefined;
+                            GOOGLE_API_HOST?: string | undefined;
+                            GOOGLE_API_KEY?: string | undefined;
+                            GROQ_API_KEY?: string | undefined;
+                            LOCALAI_BASE_URL?: string | undefined;
+                            MISTRAL_API_BASE_URL?: string | undefined;
+                            MISTRAL_API_HOST?: string | undefined;
+                            MISTRAL_API_KEY?: string | undefined;
+                            OPENAI_API_BASE_URL?: string | undefined;
+                            OPENAI_API_HOST?: string | undefined;
+                            OPENAI_API_KEY?: string | undefined;
+                            OPENAI_BASE_URL?: string | undefined;
+                            OPENAI_ORGANIZATION?: string | undefined;
+                            PALM_API_HOST?: string | undefined;
+                            PALM_API_KEY?: string | undefined;
+                            REPLICATE_API_KEY?: string | undefined;
+                            REPLICATE_API_TOKEN?: string | undefined;
+                            VERTEX_API_HOST?: string | undefined;
+                            VERTEX_API_KEY?: string | undefined;
+                            VERTEX_PROJECT_ID?: string | undefined;
+                            VERTEX_PUBLISHER?: string | undefined;
+                            VERTEX_REGION?: string | undefined;
+                        } | undefined;
+                    } | {
+                        callApi: import("../../src").CallApiFunction;
+                        id: (...args: unknown[]) => string;
+                        config?: any;
+                        label?: string | undefined;
+                        transform?: string | undefined;
+                        delay?: number | undefined;
+                        callEmbeddingApi?: ((args_0: string, ...args: unknown[]) => Promise<import("../../src").ProviderEmbeddingResponse>) | undefined;
+                        callClassificationApi?: ((args_0: string, ...args: unknown[]) => Promise<import("../../src").ProviderClassificationResponse>) | undefined;
+                    } | undefined;
+                    metadata?: Record<string, any> | undefined;
+                    description?: string | undefined;
+                    providerOutput?: string | {} | undefined;
+                    assert?: ({
+                        type: import("../../src").AssertionType;
+                        config?: Record<string, any> | undefined;
+                        value?: import("../../src").AssertionValue | undefined;
+                        provider?: any;
+                        transform?: string | undefined;
+                        rubricPrompt?: string | string[] | {
+                            role: string;
+                            content: string;
+                        }[] | undefined;
+                        threshold?: number | undefined;
+                        weight?: number | undefined;
+                        metric?: string | undefined;
+                    } | {
+                        type: "assert-set";
+                        assert: {
+                            type: import("../../src").AssertionType;
+                            config?: Record<string, any> | undefined;
+                            value?: import("../../src").AssertionValue | undefined;
+                            provider?: any;
+                            transform?: string | undefined;
+                            rubricPrompt?: string | string[] | {
+                                role: string;
+                                content: string;
+                            }[] | undefined;
+                            threshold?: number | undefined;
+                            weight?: number | undefined;
+                            metric?: string | undefined;
+                        }[];
+                        threshold?: number | undefined;
+                        weight?: number | undefined;
+                        metric?: string | undefined;
+                    })[] | undefined;
+                    threshold?: number | undefined;
+                }[];
+                description?: string | undefined;
+            })[] | undefined;
+            defaultTest?: {
+                options?: ({
+                    prefix?: string | undefined;
+                    suffix?: string | undefined;
+                } & {
+                    transform?: string | undefined;
+                    postprocess?: string | undefined;
+                    transformVars?: string | undefined;
+                    storeOutputAs?: string | undefined;
+                } & {
+                    provider?: any;
+                    rubricPrompt?: string | string[] | {
+                        role: string;
+                        content: string;
+                    }[] | undefined;
+                    factuality?: {
+                        subset?: number | undefined;
+                        superset?: number | undefined;
+                        agree?: number | undefined;
+                        disagree?: number | undefined;
+                        differButFactual?: number | undefined;
+                    } | undefined;
+                } & {
+                    disableVarExpansion?: boolean | undefined;
+                    disableConversationVar?: boolean | undefined;
+                    runSerially?: boolean | undefined;
+                }) | undefined;
+                vars?: Record<string, string | any[] | string[] | {}> | undefined;
+                provider?: string | {
+                    id?: string | undefined;
+                    config?: any;
+                    label?: string | undefined;
+                    prompts?: string[] | undefined;
+                    transform?: string | undefined;
+                    delay?: number | undefined;
+                    env?: {
+                        AI21_API_BASE_URL?: string | undefined;
+                        AI21_API_KEY?: string | undefined;
+                        ANTHROPIC_API_KEY?: string | undefined;
+                        AWS_BEDROCK_REGION?: string | undefined;
+                        AZURE_OPENAI_API_BASE_URL?: string | undefined;
+                        AZURE_OPENAI_API_HOST?: string | undefined;
+                        AZURE_OPENAI_API_KEY?: string | undefined;
+                        AZURE_OPENAI_BASE_URL?: string | undefined;
+                        BAM_API_HOST?: string | undefined;
+                        BAM_API_KEY?: string | undefined;
+                        CLOUDFLARE_ACCOUNT_ID?: string | undefined;
+                        CLOUDFLARE_API_KEY?: string | undefined;
+                        COHERE_API_KEY?: string | undefined;
+                        GOOGLE_API_HOST?: string | undefined;
+                        GOOGLE_API_KEY?: string | undefined;
+                        GROQ_API_KEY?: string | undefined;
+                        LOCALAI_BASE_URL?: string | undefined;
+                        MISTRAL_API_BASE_URL?: string | undefined;
+                        MISTRAL_API_HOST?: string | undefined;
+                        MISTRAL_API_KEY?: string | undefined;
+                        OPENAI_API_BASE_URL?: string | undefined;
+                        OPENAI_API_HOST?: string | undefined;
+                        OPENAI_API_KEY?: string | undefined;
+                        OPENAI_BASE_URL?: string | undefined;
+                        OPENAI_ORGANIZATION?: string | undefined;
+                        PALM_API_HOST?: string | undefined;
+                        PALM_API_KEY?: string | undefined;
+                        REPLICATE_API_KEY?: string | undefined;
+                        REPLICATE_API_TOKEN?: string | undefined;
+                        VERTEX_API_HOST?: string | undefined;
+                        VERTEX_API_KEY?: string | undefined;
+                        VERTEX_PROJECT_ID?: string | undefined;
+                        VERTEX_PUBLISHER?: string | undefined;
+                        VERTEX_REGION?: string | undefined;
+                    } | undefined;
+                } | {
+                    callApi: import("../../src").CallApiFunction;
+                    id: (...args: unknown[]) => string;
+                    config?: any;
+                    label?: string | undefined;
+                    transform?: string | undefined;
+                    delay?: number | undefined;
+                    callEmbeddingApi?: ((args_0: string, ...args: unknown[]) => Promise<import("../../src").ProviderEmbeddingResponse>) | undefined;
+                    callClassificationApi?: ((args_0: string, ...args: unknown[]) => Promise<import("../../src").ProviderClassificationResponse>) | undefined;
+                } | undefined;
+                metadata?: Record<string, any> | undefined;
+                providerOutput?: string | {} | undefined;
+                assert?: ({
+                    type: import("../../src").AssertionType;
+                    config?: Record<string, any> | undefined;
+                    value?: import("../../src").AssertionValue | undefined;
+                    provider?: any;
+                    transform?: string | undefined;
+                    rubricPrompt?: string | string[] | {
+                        role: string;
+                        content: string;
+                    }[] | undefined;
+                    threshold?: number | undefined;
+                    weight?: number | undefined;
+                    metric?: string | undefined;
+                } | {
+                    type: "assert-set";
+                    assert: {
+                        type: import("../../src").AssertionType;
+                        config?: Record<string, any> | undefined;
+                        value?: import("../../src").AssertionValue | undefined;
+                        provider?: any;
+                        transform?: string | undefined;
+                        rubricPrompt?: string | string[] | {
+                            role: string;
+                            content: string;
+                        }[] | undefined;
+                        threshold?: number | undefined;
+                        weight?: number | undefined;
+                        metric?: string | undefined;
+                    }[];
+                    threshold?: number | undefined;
+                    weight?: number | undefined;
+                    metric?: string | undefined;
+                })[] | undefined;
+                threshold?: number | undefined;
+            } | undefined;
+            nunjucksFilters?: Record<string, string> | undefined;
+            derivedMetrics?: {
+                value: string | ((args_0: Record<string, number>, args_1: import("../../src").RunEvalOptions, ...args: unknown[]) => number);
+                name: string;
+            }[] | undefined;
+            extensions?: string[] | undefined;
+            outputPath?: string | string[] | undefined;
+            sharing?: boolean | {
+                apiBaseUrl?: string | undefined;
+                appBaseUrl?: string | undefined;
+            } | undefined;
+            writeLatestResults?: boolean | undefined;
+            evaluateOptions?: {
+                cache?: boolean | undefined;
+                maxConcurrency?: number | undefined;
+                delay?: number | undefined;
+                repeat?: number | undefined;
+                generateSuggestions?: boolean | undefined;
+                eventSource?: string | undefined;
+                interactiveProviders?: boolean | undefined;
+                progressCallback?: ((args_0: number, args_1: number, args_2: number, args_3: import("../../src").RunEvalOptions) => void) | undefined;
+                showProgressBar?: boolean | undefined;
+            } | undefined;
+            commandLineOptions?: {
+                cache?: boolean | undefined;
+                config?: string[] | undefined;
+                maxConcurrency?: number | undefined;
+                output?: string[] | undefined;
+                write?: boolean | undefined;
+                vars?: string | undefined;
+                prompts?: string[] | undefined;
+                delay?: number | undefined;
+                description?: string | undefined;
+                providers?: string[] | undefined;
+                repeat?: number | undefined;
+                tests?: string | undefined;
+                assertions?: string | undefined;
+                modelOutputs?: string | undefined;
+                verbose?: boolean | undefined;
+                grader?: string | undefined;
+                tableCellMaxLength?: number | undefined;
+                table?: boolean | undefined;
+                share?: boolean | undefined;
+                progressBar?: boolean | undefined;
+                watch?: boolean | undefined;
+                filterFailing?: string | undefined;
+                filterFirstN?: number | undefined;
+                filterPattern?: string | undefined;
+                filterProviders?: string | undefined;
+                var?: Record<string, string> | undefined;
+                generateSuggestions?: boolean | undefined;
+                promptPrefix?: string | undefined;
+                promptSuffix?: string | undefined;
+                envPath?: string | undefined;
+            } | undefined;
+        }>;
+        prompts: {
+            provider: string;
+            raw: string;
+            label: string;
+            function?: ((args_0: {
+                vars: Record<string, any>;
+                provider?: import("../../src").ApiProvider | undefined;
+            }, ...args: unknown[]) => Promise<any>) | undefined;
+            id?: string | undefined;
+            config?: any;
+            display?: string | undefined;
+            metrics?: {
+                cost: number;
+                tokenUsage: {
+                    cached?: number | undefined;
+                    completion?: number | undefined;
+                    prompt?: number | undefined;
+                    total?: number | undefined;
+                };
+                score: number;
+                testPassCount: number;
+                testFailCount: number;
+                assertPassCount: number;
+                assertFailCount: number;
+                totalLatencyMs: number;
+                namedScores: Record<string, number>;
+                namedScoresCount: Record<string, number>;
+                redteam?: {
+                    pluginPassCount: Record<string, number>;
+                    pluginFailCount: Record<string, number>;
+                    strategyPassCount: Record<string, number>;
+                    strategyFailCount: Record<string, number>;
+                } | undefined;
+            } | undefined;
+        }[] | null;
+        description: string | null;
+        createdAt: number;
+        author: string | null;
+        results: object | import("../../src").EvaluateSummaryV2;
+    }>;
+}
+//# sourceMappingURL=evalFactory.d.ts.map