npm - promptfoo - Versions diffs - 0.71.1 → 0.72.1 - Mend

promptfoo 0.71.1 → 0.72.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (121) hide show

package/dist/src/database.d.ts CHANGED Viewed

@@ -1,4 +1,4 @@
-import type { EvaluateSummary, UnifiedConfig } from './types';
+import type { EvaluateSummary } from './types';
 export declare const prompts: import("drizzle-orm/sqlite-core").SQLiteTableWithColumns<{
     name: "prompts";
     schema: undefined;
@@ -143,7 +143,976 @@ export declare const evals: import("drizzle-orm/sqlite-core").SQLiteTableWithCol
             tableName: "evals";
             dataType: "json";
             columnType: "SQLiteTextJson";
-            data: Partial<UnifiedConfig>;
+            data: Partial<{
+                prompts: string | Record<string, string> | (string | {
+                    label: string;
+                    raw: string;
+                    function?: ((args_0: {
+                        vars: Record<string, any>;
+                        provider?: {
+                            id: (...args: unknown[]) => string;
+                            callApi: (((args_0: string, args_1: {
+                                vars: Record<string, string | {}>;
+                                logger?: any;
+                                fetchWithCache?: any;
+                                getCache?: any;
+                            } | undefined, args_2: {
+                                includeLogProbs?: boolean | undefined;
+                                originalProvider?: any;
+                            } | undefined, ...args_3: unknown[]) => Promise<{
+                                output?: any;
+                                cached?: boolean | undefined;
+                                cost?: number | undefined;
+                                error?: string | undefined;
+                                logProbs?: number[] | undefined;
+                                metadata?: import("zod").objectOutputType<{
+                                    redteamFinalPrompt: import("zod").ZodOptional<import("zod").ZodString>;
+                                }, import("zod").ZodAny, "strip"> | undefined;
+                                tokenUsage?: {
+                                    prompt?: number | undefined;
+                                    cached?: number | undefined;
+                                    completion?: number | undefined;
+                                    total?: number | undefined;
+                                } | undefined;
+                            }>) & {
+                                label?: string | undefined;
+                            }) | (((args_0: string, args_1: {
+                                includeLogProbs?: boolean | undefined;
+                                originalProvider?: any;
+                            } | undefined, ...args_2: unknown[]) => Promise<{
+                                output?: any;
+                                cached?: boolean | undefined;
+                                cost?: number | undefined;
+                                error?: string | undefined;
+                                logProbs?: number[] | undefined;
+                                metadata?: import("zod").objectOutputType<{
+                                    redteamFinalPrompt: import("zod").ZodOptional<import("zod").ZodString>;
+                                }, import("zod").ZodAny, "strip"> | undefined;
+                                tokenUsage?: {
+                                    prompt?: number | undefined;
+                                    cached?: number | undefined;
+                                    completion?: number | undefined;
+                                    total?: number | undefined;
+                                } | undefined;
+                            }>) & {
+                                label?: string | undefined;
+                            });
+                            delay?: number | undefined;
+                            label?: string | undefined;
+                            transform?: string | undefined;
+                            callEmbeddingApi?: ((args_0: string, ...args_1: unknown[]) => Promise<{
+                                error?: string | undefined;
+                                tokenUsage?: {
+                                    prompt?: number | undefined;
+                                    cached?: number | undefined;
+                                    completion?: number | undefined;
+                                    total?: number | undefined;
+                                } | undefined;
+                                embedding?: number[] | undefined;
+                            }>) | undefined;
+                            callClassificationApi?: ((args_0: string, ...args_1: unknown[]) => Promise<{
+                                error?: string | undefined;
+                                classification?: Record<string, number> | undefined;
+                            }>) | undefined;
+                        } | undefined;
+                    }, ...args_1: unknown[]) => Promise<any>) | undefined;
+                    id?: string | undefined;
+                    display?: string | undefined;
+                })[];
+                providers: string | (((args_0: string, args_1: {
+                    vars: Record<string, string | {}>;
+                    logger?: any;
+                    fetchWithCache?: any;
+                    getCache?: any;
+                } | undefined, args_2: {
+                    includeLogProbs?: boolean | undefined;
+                    originalProvider?: any;
+                } | undefined, ...args_3: unknown[]) => Promise<{
+                    output?: any;
+                    cached?: boolean | undefined;
+                    cost?: number | undefined;
+                    error?: string | undefined;
+                    logProbs?: number[] | undefined;
+                    metadata?: import("zod").objectOutputType<{
+                        redteamFinalPrompt: import("zod").ZodOptional<import("zod").ZodString>;
+                    }, import("zod").ZodAny, "strip"> | undefined;
+                    tokenUsage?: {
+                        prompt?: number | undefined;
+                        cached?: number | undefined;
+                        completion?: number | undefined;
+                        total?: number | undefined;
+                    } | undefined;
+                }>) & {
+                    label?: string | undefined;
+                }) | (((args_0: string, args_1: {
+                    includeLogProbs?: boolean | undefined;
+                    originalProvider?: any;
+                } | undefined, ...args_2: unknown[]) => Promise<{
+                    output?: any;
+                    cached?: boolean | undefined;
+                    cost?: number | undefined;
+                    error?: string | undefined;
+                    logProbs?: number[] | undefined;
+                    metadata?: import("zod").objectOutputType<{
+                        redteamFinalPrompt: import("zod").ZodOptional<import("zod").ZodString>;
+                    }, import("zod").ZodAny, "strip"> | undefined;
+                    tokenUsage?: {
+                        prompt?: number | undefined;
+                        cached?: number | undefined;
+                        completion?: number | undefined;
+                        total?: number | undefined;
+                    } | undefined;
+                }>) & {
+                    label?: string | undefined;
+                }) | (string | {
+                    prompts?: string[] | undefined;
+                    delay?: number | undefined;
+                    config?: any;
+                    id?: string | undefined;
+                    label?: string | undefined;
+                    transform?: string | undefined;
+                    env?: {
+                        ANTHROPIC_API_KEY?: string | undefined;
+                        BAM_API_KEY?: string | undefined;
+                        BAM_API_HOST?: string | undefined;
+                        AZURE_OPENAI_API_HOST?: string | undefined;
+                        AZURE_OPENAI_API_KEY?: string | undefined;
+                        AZURE_OPENAI_API_BASE_URL?: string | undefined;
+                        AZURE_OPENAI_BASE_URL?: string | undefined;
+                        AWS_BEDROCK_REGION?: string | undefined;
+                        COHERE_API_KEY?: string | undefined;
+                        OPENAI_API_KEY?: string | undefined;
+                        OPENAI_API_HOST?: string | undefined;
+                        OPENAI_API_BASE_URL?: string | undefined;
+                        OPENAI_BASE_URL?: string | undefined;
+                        OPENAI_ORGANIZATION?: string | undefined;
+                        REPLICATE_API_KEY?: string | undefined;
+                        REPLICATE_API_TOKEN?: string | undefined;
+                        LOCALAI_BASE_URL?: string | undefined;
+                        MISTRAL_API_HOST?: string | undefined;
+                        MISTRAL_API_BASE_URL?: string | undefined;
+                        PALM_API_KEY?: string | undefined;
+                        PALM_API_HOST?: string | undefined;
+                        GOOGLE_API_KEY?: string | undefined;
+                        GOOGLE_API_HOST?: string | undefined;
+                        VERTEX_API_KEY?: string | undefined;
+                        VERTEX_API_HOST?: string | undefined;
+                        VERTEX_PROJECT_ID?: string | undefined;
+                        VERTEX_REGION?: string | undefined;
+                        VERTEX_PUBLISHER?: string | undefined;
+                        MISTRAL_API_KEY?: string | undefined;
+                        CLOUDFLARE_API_KEY?: string | undefined;
+                        CLOUDFLARE_ACCOUNT_ID?: string | undefined;
+                    } | undefined;
+                } | (((args_0: string, args_1: {
+                    vars: Record<string, string | {}>;
+                    logger?: any;
+                    fetchWithCache?: any;
+                    getCache?: any;
+                } | undefined, args_2: {
+                    includeLogProbs?: boolean | undefined;
+                    originalProvider?: any;
+                } | undefined, ...args_3: unknown[]) => Promise<{
+                    output?: any;
+                    cached?: boolean | undefined;
+                    cost?: number | undefined;
+                    error?: string | undefined;
+                    logProbs?: number[] | undefined;
+                    metadata?: import("zod").objectOutputType<{
+                        redteamFinalPrompt: import("zod").ZodOptional<import("zod").ZodString>;
+                    }, import("zod").ZodAny, "strip"> | undefined;
+                    tokenUsage?: {
+                        prompt?: number | undefined;
+                        cached?: number | undefined;
+                        completion?: number | undefined;
+                        total?: number | undefined;
+                    } | undefined;
+                }>) & {
+                    label?: string | undefined;
+                }) | (((args_0: string, args_1: {
+                    includeLogProbs?: boolean | undefined;
+                    originalProvider?: any;
+                } | undefined, ...args_2: unknown[]) => Promise<{
+                    output?: any;
+                    cached?: boolean | undefined;
+                    cost?: number | undefined;
+                    error?: string | undefined;
+                    logProbs?: number[] | undefined;
+                    metadata?: import("zod").objectOutputType<{
+                        redteamFinalPrompt: import("zod").ZodOptional<import("zod").ZodString>;
+                    }, import("zod").ZodAny, "strip"> | undefined;
+                    tokenUsage?: {
+                        prompt?: number | undefined;
+                        cached?: number | undefined;
+                        completion?: number | undefined;
+                        total?: number | undefined;
+                    } | undefined;
+                }>) & {
+                    label?: string | undefined;
+                }) | Record<string, {
+                    prompts?: string[] | undefined;
+                    delay?: number | undefined;
+                    config?: any;
+                    id?: string | undefined;
+                    label?: string | undefined;
+                    transform?: string | undefined;
+                    env?: {
+                        ANTHROPIC_API_KEY?: string | undefined;
+                        BAM_API_KEY?: string | undefined;
+                        BAM_API_HOST?: string | undefined;
+                        AZURE_OPENAI_API_HOST?: string | undefined;
+                        AZURE_OPENAI_API_KEY?: string | undefined;
+                        AZURE_OPENAI_API_BASE_URL?: string | undefined;
+                        AZURE_OPENAI_BASE_URL?: string | undefined;
+                        AWS_BEDROCK_REGION?: string | undefined;
+                        COHERE_API_KEY?: string | undefined;
+                        OPENAI_API_KEY?: string | undefined;
+                        OPENAI_API_HOST?: string | undefined;
+                        OPENAI_API_BASE_URL?: string | undefined;
+                        OPENAI_BASE_URL?: string | undefined;
+                        OPENAI_ORGANIZATION?: string | undefined;
+                        REPLICATE_API_KEY?: string | undefined;
+                        REPLICATE_API_TOKEN?: string | undefined;
+                        LOCALAI_BASE_URL?: string | undefined;
+                        MISTRAL_API_HOST?: string | undefined;
+                        MISTRAL_API_BASE_URL?: string | undefined;
+                        PALM_API_KEY?: string | undefined;
+                        PALM_API_HOST?: string | undefined;
+                        GOOGLE_API_KEY?: string | undefined;
+                        GOOGLE_API_HOST?: string | undefined;
+                        VERTEX_API_KEY?: string | undefined;
+                        VERTEX_API_HOST?: string | undefined;
+                        VERTEX_PROJECT_ID?: string | undefined;
+                        VERTEX_REGION?: string | undefined;
+                        VERTEX_PUBLISHER?: string | undefined;
+                        MISTRAL_API_KEY?: string | undefined;
+                        CLOUDFLARE_API_KEY?: string | undefined;
+                        CLOUDFLARE_ACCOUNT_ID?: string | undefined;
+                    } | undefined;
+                }>)[];
+                tests: string | (string | {
+                    options?: ({
+                        prefix?: string | undefined;
+                        suffix?: string | undefined;
+                    } & {
+                        transform?: string | undefined;
+                        postprocess?: string | undefined;
+                        storeOutputAs?: string | undefined;
+                    } & {
+                        rubricPrompt?: string | string[] | undefined;
+                        provider?: any;
+                        factuality?: {
+                            subset?: number | undefined;
+                            superset?: number | undefined;
+                            agree?: number | undefined;
+                            disagree?: number | undefined;
+                            differButFactual?: number | undefined;
+                        } | undefined;
+                    } & {
+                        disableVarExpansion?: boolean | undefined;
+                        disableConversationVar?: boolean | undefined;
+                    }) | undefined;
+                    vars?: Record<string, string | string[] | {}> | undefined;
+                    provider?: string | {
+                        prompts?: string[] | undefined;
+                        delay?: number | undefined;
+                        config?: any;
+                        id?: string | undefined;
+                        label?: string | undefined;
+                        transform?: string | undefined;
+                        env?: {
+                            ANTHROPIC_API_KEY?: string | undefined;
+                            BAM_API_KEY?: string | undefined;
+                            BAM_API_HOST?: string | undefined;
+                            AZURE_OPENAI_API_HOST?: string | undefined;
+                            AZURE_OPENAI_API_KEY?: string | undefined;
+                            AZURE_OPENAI_API_BASE_URL?: string | undefined;
+                            AZURE_OPENAI_BASE_URL?: string | undefined;
+                            AWS_BEDROCK_REGION?: string | undefined;
+                            COHERE_API_KEY?: string | undefined;
+                            OPENAI_API_KEY?: string | undefined;
+                            OPENAI_API_HOST?: string | undefined;
+                            OPENAI_API_BASE_URL?: string | undefined;
+                            OPENAI_BASE_URL?: string | undefined;
+                            OPENAI_ORGANIZATION?: string | undefined;
+                            REPLICATE_API_KEY?: string | undefined;
+                            REPLICATE_API_TOKEN?: string | undefined;
+                            LOCALAI_BASE_URL?: string | undefined;
+                            MISTRAL_API_HOST?: string | undefined;
+                            MISTRAL_API_BASE_URL?: string | undefined;
+                            PALM_API_KEY?: string | undefined;
+                            PALM_API_HOST?: string | undefined;
+                            GOOGLE_API_KEY?: string | undefined;
+                            GOOGLE_API_HOST?: string | undefined;
+                            VERTEX_API_KEY?: string | undefined;
+                            VERTEX_API_HOST?: string | undefined;
+                            VERTEX_PROJECT_ID?: string | undefined;
+                            VERTEX_REGION?: string | undefined;
+                            VERTEX_PUBLISHER?: string | undefined;
+                            MISTRAL_API_KEY?: string | undefined;
+                            CLOUDFLARE_API_KEY?: string | undefined;
+                            CLOUDFLARE_ACCOUNT_ID?: string | undefined;
+                        } | undefined;
+                    } | {
+                        id: (...args: unknown[]) => string;
+                        callApi: (((args_0: string, args_1: {
+                            vars: Record<string, string | {}>;
+                            logger?: any;
+                            fetchWithCache?: any;
+                            getCache?: any;
+                        } | undefined, args_2: {
+                            includeLogProbs?: boolean | undefined;
+                            originalProvider?: any;
+                        } | undefined, ...args_3: unknown[]) => Promise<{
+                            output?: any;
+                            cached?: boolean | undefined;
+                            cost?: number | undefined;
+                            error?: string | undefined;
+                            logProbs?: number[] | undefined;
+                            metadata?: import("zod").objectOutputType<{
+                                redteamFinalPrompt: import("zod").ZodOptional<import("zod").ZodString>;
+                            }, import("zod").ZodAny, "strip"> | undefined;
+                            tokenUsage?: {
+                                prompt?: number | undefined;
+                                cached?: number | undefined;
+                                completion?: number | undefined;
+                                total?: number | undefined;
+                            } | undefined;
+                        }>) & {
+                            label?: string | undefined;
+                        }) | (((args_0: string, args_1: {
+                            includeLogProbs?: boolean | undefined;
+                            originalProvider?: any;
+                        } | undefined, ...args_2: unknown[]) => Promise<{
+                            output?: any;
+                            cached?: boolean | undefined;
+                            cost?: number | undefined;
+                            error?: string | undefined;
+                            logProbs?: number[] | undefined;
+                            metadata?: import("zod").objectOutputType<{
+                                redteamFinalPrompt: import("zod").ZodOptional<import("zod").ZodString>;
+                            }, import("zod").ZodAny, "strip"> | undefined;
+                            tokenUsage?: {
+                                prompt?: number | undefined;
+                                cached?: number | undefined;
+                                completion?: number | undefined;
+                                total?: number | undefined;
+                            } | undefined;
+                        }>) & {
+                            label?: string | undefined;
+                        });
+                        delay?: number | undefined;
+                        label?: string | undefined;
+                        transform?: string | undefined;
+                        callEmbeddingApi?: ((args_0: string, ...args_1: unknown[]) => Promise<{
+                            error?: string | undefined;
+                            tokenUsage?: {
+                                prompt?: number | undefined;
+                                cached?: number | undefined;
+                                completion?: number | undefined;
+                                total?: number | undefined;
+                            } | undefined;
+                            embedding?: number[] | undefined;
+                        }>) | undefined;
+                        callClassificationApi?: ((args_0: string, ...args_1: unknown[]) => Promise<{
+                            error?: string | undefined;
+                            classification?: Record<string, number> | undefined;
+                        }>) | undefined;
+                    } | undefined;
+                    description?: string | undefined;
+                    providerOutput?: string | {} | undefined;
+                    assert?: ({
+                        type: import("./types").AssertionType;
+                        value?: import("./types").AssertionValue | undefined;
+                        transform?: string | undefined;
+                        rubricPrompt?: string | string[] | undefined;
+                        provider?: any;
+                        threshold?: number | undefined;
+                        weight?: number | undefined;
+                        metric?: string | undefined;
+                    } | {
+                        type: "assert-set";
+                        assert: {
+                            type: import("./types").AssertionType;
+                            value?: import("./types").AssertionValue | undefined;
+                            transform?: string | undefined;
+                            rubricPrompt?: string | string[] | undefined;
+                            provider?: any;
+                            threshold?: number | undefined;
+                            weight?: number | undefined;
+                            metric?: string | undefined;
+                        }[];
+                        threshold?: number | undefined;
+                        weight?: number | undefined;
+                        metric?: string | undefined;
+                    })[] | undefined;
+                    threshold?: number | undefined;
+                })[];
+                evaluateOptions: {
+                    maxConcurrency?: number | undefined;
+                    repeat?: number | undefined;
+                    delay?: number | undefined;
+                    cache?: boolean | undefined;
+                    interactiveProviders?: boolean | undefined;
+                    generateSuggestions?: boolean | undefined;
+                    showProgressBar?: boolean | undefined;
+                    progressCallback?: ((args_0: number, args_1: number, args_2: number, args_3: import("./types").RunEvalOptions) => void) | undefined;
+                    eventSource?: string | undefined;
+                };
+                commandLineOptions: {
+                    prompts?: string[] | undefined;
+                    providers?: string[] | undefined;
+                    output?: string[] | undefined;
+                    maxConcurrency?: string | undefined;
+                    repeat?: string | undefined;
+                    delay?: string | undefined;
+                    vars?: string | undefined;
+                    tests?: string | undefined;
+                    config?: string[] | undefined;
+                    assertions?: string | undefined;
+                    modelOutputs?: string | undefined;
+                    verbose?: boolean | undefined;
+                    grader?: string | undefined;
+                    tableCellMaxLength?: string | undefined;
+                    write?: boolean | undefined;
+                    cache?: boolean | undefined;
+                    table?: boolean | undefined;
+                    share?: boolean | undefined;
+                    progressBar?: boolean | undefined;
+                    watch?: boolean | undefined;
+                    interactiveProviders?: boolean | undefined;
+                    filterFailing?: string | undefined;
+                    filterFirstN?: string | undefined;
+                    filterPattern?: string | undefined;
+                    filterProviders?: string | undefined;
+                    var?: Record<string, string> | undefined;
+                    generateSuggestions?: boolean | undefined;
+                    promptPrefix?: string | undefined;
+                    promptSuffix?: string | undefined;
+                    envFile?: string | undefined;
+                };
+                env?: {
+                    ANTHROPIC_API_KEY?: string | undefined;
+                    BAM_API_KEY?: string | undefined;
+                    BAM_API_HOST?: string | undefined;
+                    AZURE_OPENAI_API_HOST?: string | undefined;
+                    AZURE_OPENAI_API_KEY?: string | undefined;
+                    AZURE_OPENAI_API_BASE_URL?: string | undefined;
+                    AZURE_OPENAI_BASE_URL?: string | undefined;
+                    AWS_BEDROCK_REGION?: string | undefined;
+                    COHERE_API_KEY?: string | undefined;
+                    OPENAI_API_KEY?: string | undefined;
+                    OPENAI_API_HOST?: string | undefined;
+                    OPENAI_API_BASE_URL?: string | undefined;
+                    OPENAI_BASE_URL?: string | undefined;
+                    OPENAI_ORGANIZATION?: string | undefined;
+                    REPLICATE_API_KEY?: string | undefined;
+                    REPLICATE_API_TOKEN?: string | undefined;
+                    LOCALAI_BASE_URL?: string | undefined;
+                    MISTRAL_API_HOST?: string | undefined;
+                    MISTRAL_API_BASE_URL?: string | undefined;
+                    PALM_API_KEY?: string | undefined;
+                    PALM_API_HOST?: string | undefined;
+                    GOOGLE_API_KEY?: string | undefined;
+                    GOOGLE_API_HOST?: string | undefined;
+                    VERTEX_API_KEY?: string | undefined;
+                    VERTEX_API_HOST?: string | undefined;
+                    VERTEX_PROJECT_ID?: string | undefined;
+                    VERTEX_REGION?: string | undefined;
+                    VERTEX_PUBLISHER?: string | undefined;
+                    MISTRAL_API_KEY?: string | undefined;
+                    CLOUDFLARE_API_KEY?: string | undefined;
+                    CLOUDFLARE_ACCOUNT_ID?: string | undefined;
+                } | undefined;
+                metadata?: Record<string, any> | undefined;
+                description?: string | undefined;
+                scenarios?: {
+                    tests: {
+                        options?: ({
+                            prefix?: string | undefined;
+                            suffix?: string | undefined;
+                        } & {
+                            transform?: string | undefined;
+                            postprocess?: string | undefined;
+                            storeOutputAs?: string | undefined;
+                        } & {
+                            rubricPrompt?: string | string[] | undefined;
+                            provider?: any;
+                            factuality?: {
+                                subset?: number | undefined;
+                                superset?: number | undefined;
+                                agree?: number | undefined;
+                                disagree?: number | undefined;
+                                differButFactual?: number | undefined;
+                            } | undefined;
+                        } & {
+                            disableVarExpansion?: boolean | undefined;
+                            disableConversationVar?: boolean | undefined;
+                        }) | undefined;
+                        vars?: Record<string, string | string[] | {}> | undefined;
+                        provider?: string | {
+                            prompts?: string[] | undefined;
+                            delay?: number | undefined;
+                            config?: any;
+                            id?: string | undefined;
+                            label?: string | undefined;
+                            transform?: string | undefined;
+                            env?: {
+                                ANTHROPIC_API_KEY?: string | undefined;
+                                BAM_API_KEY?: string | undefined;
+                                BAM_API_HOST?: string | undefined;
+                                AZURE_OPENAI_API_HOST?: string | undefined;
+                                AZURE_OPENAI_API_KEY?: string | undefined;
+                                AZURE_OPENAI_API_BASE_URL?: string | undefined;
+                                AZURE_OPENAI_BASE_URL?: string | undefined;
+                                AWS_BEDROCK_REGION?: string | undefined;
+                                COHERE_API_KEY?: string | undefined;
+                                OPENAI_API_KEY?: string | undefined;
+                                OPENAI_API_HOST?: string | undefined;
+                                OPENAI_API_BASE_URL?: string | undefined;
+                                OPENAI_BASE_URL?: string | undefined;
+                                OPENAI_ORGANIZATION?: string | undefined;
+                                REPLICATE_API_KEY?: string | undefined;
+                                REPLICATE_API_TOKEN?: string | undefined;
+                                LOCALAI_BASE_URL?: string | undefined;
+                                MISTRAL_API_HOST?: string | undefined;
+                                MISTRAL_API_BASE_URL?: string | undefined;
+                                PALM_API_KEY?: string | undefined;
+                                PALM_API_HOST?: string | undefined;
+                                GOOGLE_API_KEY?: string | undefined;
+                                GOOGLE_API_HOST?: string | undefined;
+                                VERTEX_API_KEY?: string | undefined;
+                                VERTEX_API_HOST?: string | undefined;
+                                VERTEX_PROJECT_ID?: string | undefined;
+                                VERTEX_REGION?: string | undefined;
+                                VERTEX_PUBLISHER?: string | undefined;
+                                MISTRAL_API_KEY?: string | undefined;
+                                CLOUDFLARE_API_KEY?: string | undefined;
+                                CLOUDFLARE_ACCOUNT_ID?: string | undefined;
+                            } | undefined;
+                        } | {
+                            id: (...args: unknown[]) => string;
+                            callApi: (((args_0: string, args_1: {
+                                vars: Record<string, string | {}>;
+                                logger?: any;
+                                fetchWithCache?: any;
+                                getCache?: any;
+                            } | undefined, args_2: {
+                                includeLogProbs?: boolean | undefined;
+                                originalProvider?: any;
+                            } | undefined, ...args_3: unknown[]) => Promise<{
+                                output?: any;
+                                cached?: boolean | undefined;
+                                cost?: number | undefined;
+                                error?: string | undefined;
+                                logProbs?: number[] | undefined;
+                                metadata?: import("zod").objectOutputType<{
+                                    redteamFinalPrompt: import("zod").ZodOptional<import("zod").ZodString>;
+                                }, import("zod").ZodAny, "strip"> | undefined;
+                                tokenUsage?: {
+                                    prompt?: number | undefined;
+                                    cached?: number | undefined;
+                                    completion?: number | undefined;
+                                    total?: number | undefined;
+                                } | undefined;
+                            }>) & {
+                                label?: string | undefined;
+                            }) | (((args_0: string, args_1: {
+                                includeLogProbs?: boolean | undefined;
+                                originalProvider?: any;
+                            } | undefined, ...args_2: unknown[]) => Promise<{
+                                output?: any;
+                                cached?: boolean | undefined;
+                                cost?: number | undefined;
+                                error?: string | undefined;
+                                logProbs?: number[] | undefined;
+                                metadata?: import("zod").objectOutputType<{
+                                    redteamFinalPrompt: import("zod").ZodOptional<import("zod").ZodString>;
+                                }, import("zod").ZodAny, "strip"> | undefined;
+                                tokenUsage?: {
+                                    prompt?: number | undefined;
+                                    cached?: number | undefined;
+                                    completion?: number | undefined;
+                                    total?: number | undefined;
+                                } | undefined;
+                            }>) & {
+                                label?: string | undefined;
+                            });
+                            delay?: number | undefined;
+                            label?: string | undefined;
+                            transform?: string | undefined;
+                            callEmbeddingApi?: ((args_0: string, ...args_1: unknown[]) => Promise<{
+                                error?: string | undefined;
+                                tokenUsage?: {
+                                    prompt?: number | undefined;
+                                    cached?: number | undefined;
+                                    completion?: number | undefined;
+                                    total?: number | undefined;
+                                } | undefined;
+                                embedding?: number[] | undefined;
+                            }>) | undefined;
+                            callClassificationApi?: ((args_0: string, ...args_1: unknown[]) => Promise<{
+                                error?: string | undefined;
+                                classification?: Record<string, number> | undefined;
+                            }>) | undefined;
+                        } | undefined;
+                        description?: string | undefined;
+                        providerOutput?: string | {} | undefined;
+                        assert?: ({
+                            type: import("./types").AssertionType;
+                            value?: import("./types").AssertionValue | undefined;
+                            transform?: string | undefined;
+                            rubricPrompt?: string | string[] | undefined;
+                            provider?: any;
+                            threshold?: number | undefined;
+                            weight?: number | undefined;
+                            metric?: string | undefined;
+                        } | {
+                            type: "assert-set";
+                            assert: {
+                                type: import("./types").AssertionType;
+                                value?: import("./types").AssertionValue | undefined;
+                                transform?: string | undefined;
+                                rubricPrompt?: string | string[] | undefined;
+                                provider?: any;
+                                threshold?: number | undefined;
+                                weight?: number | undefined;
+                                metric?: string | undefined;
+                            }[];
+                            threshold?: number | undefined;
+                            weight?: number | undefined;
+                            metric?: string | undefined;
+                        })[] | undefined;
+                        threshold?: number | undefined;
+                    }[];
+                    config: {
+                        options?: ({
+                            prefix?: string | undefined;
+                            suffix?: string | undefined;
+                        } & {
+                            transform?: string | undefined;
+                            postprocess?: string | undefined;
+                            storeOutputAs?: string | undefined;
+                        } & {
+                            rubricPrompt?: string | string[] | undefined;
+                            provider?: any;
+                            factuality?: {
+                                subset?: number | undefined;
+                                superset?: number | undefined;
+                                agree?: number | undefined;
+                                disagree?: number | undefined;
+                                differButFactual?: number | undefined;
+                            } | undefined;
+                        } & {
+                            disableVarExpansion?: boolean | undefined;
+                            disableConversationVar?: boolean | undefined;
+                        }) | undefined;
+                        vars?: Record<string, string | string[] | {}> | undefined;
+                        provider?: string | {
+                            prompts?: string[] | undefined;
+                            delay?: number | undefined;
+                            config?: any;
+                            id?: string | undefined;
+                            label?: string | undefined;
+                            transform?: string | undefined;
+                            env?: {
+                                ANTHROPIC_API_KEY?: string | undefined;
+                                BAM_API_KEY?: string | undefined;
+                                BAM_API_HOST?: string | undefined;
+                                AZURE_OPENAI_API_HOST?: string | undefined;
+                                AZURE_OPENAI_API_KEY?: string | undefined;
+                                AZURE_OPENAI_API_BASE_URL?: string | undefined;
+                                AZURE_OPENAI_BASE_URL?: string | undefined;
+                                AWS_BEDROCK_REGION?: string | undefined;
+                                COHERE_API_KEY?: string | undefined;
+                                OPENAI_API_KEY?: string | undefined;
+                                OPENAI_API_HOST?: string | undefined;
+                                OPENAI_API_BASE_URL?: string | undefined;
+                                OPENAI_BASE_URL?: string | undefined;
+                                OPENAI_ORGANIZATION?: string | undefined;
+                                REPLICATE_API_KEY?: string | undefined;
+                                REPLICATE_API_TOKEN?: string | undefined;
+                                LOCALAI_BASE_URL?: string | undefined;
+                                MISTRAL_API_HOST?: string | undefined;
+                                MISTRAL_API_BASE_URL?: string | undefined;
+                                PALM_API_KEY?: string | undefined;
+                                PALM_API_HOST?: string | undefined;
+                                GOOGLE_API_KEY?: string | undefined;
+                                GOOGLE_API_HOST?: string | undefined;
+                                VERTEX_API_KEY?: string | undefined;
+                                VERTEX_API_HOST?: string | undefined;
+                                VERTEX_PROJECT_ID?: string | undefined;
+                                VERTEX_REGION?: string | undefined;
+                                VERTEX_PUBLISHER?: string | undefined;
+                                MISTRAL_API_KEY?: string | undefined;
+                                CLOUDFLARE_API_KEY?: string | undefined;
+                                CLOUDFLARE_ACCOUNT_ID?: string | undefined;
+                            } | undefined;
+                        } | {
+                            id: (...args: unknown[]) => string;
+                            callApi: (((args_0: string, args_1: {
+                                vars: Record<string, string | {}>;
+                                logger?: any;
+                                fetchWithCache?: any;
+                                getCache?: any;
+                            } | undefined, args_2: {
+                                includeLogProbs?: boolean | undefined;
+                                originalProvider?: any;
+                            } | undefined, ...args_3: unknown[]) => Promise<{
+                                output?: any;
+                                cached?: boolean | undefined;
+                                cost?: number | undefined;
+                                error?: string | undefined;
+                                logProbs?: number[] | undefined;
+                                metadata?: import("zod").objectOutputType<{
+                                    redteamFinalPrompt: import("zod").ZodOptional<import("zod").ZodString>;
+                                }, import("zod").ZodAny, "strip"> | undefined;
+                                tokenUsage?: {
+                                    prompt?: number | undefined;
+                                    cached?: number | undefined;
+                                    completion?: number | undefined;
+                                    total?: number | undefined;
+                                } | undefined;
+                            }>) & {
+                                label?: string | undefined;
+                            }) | (((args_0: string, args_1: {
+                                includeLogProbs?: boolean | undefined;
+                                originalProvider?: any;
+                            } | undefined, ...args_2: unknown[]) => Promise<{
+                                output?: any;
+                                cached?: boolean | undefined;
+                                cost?: number | undefined;
+                                error?: string | undefined;
+                                logProbs?: number[] | undefined;
+                                metadata?: import("zod").objectOutputType<{
+                                    redteamFinalPrompt: import("zod").ZodOptional<import("zod").ZodString>;
+                                }, import("zod").ZodAny, "strip"> | undefined;
+                                tokenUsage?: {
+                                    prompt?: number | undefined;
+                                    cached?: number | undefined;
+                                    completion?: number | undefined;
+                                    total?: number | undefined;
+                                } | undefined;
+                            }>) & {
+                                label?: string | undefined;
+                            });
+                            delay?: number | undefined;
+                            label?: string | undefined;
+                            transform?: string | undefined;
+                            callEmbeddingApi?: ((args_0: string, ...args_1: unknown[]) => Promise<{
+                                error?: string | undefined;
+                                tokenUsage?: {
+                                    prompt?: number | undefined;
+                                    cached?: number | undefined;
+                                    completion?: number | undefined;
+                                    total?: number | undefined;
+                                } | undefined;
+                                embedding?: number[] | undefined;
+                            }>) | undefined;
+                            callClassificationApi?: ((args_0: string, ...args_1: unknown[]) => Promise<{
+                                error?: string | undefined;
+                                classification?: Record<string, number> | undefined;
+                            }>) | undefined;
+                        } | undefined;
+                        description?: string | undefined;
+                        providerOutput?: string | {} | undefined;
+                        assert?: ({
+                            type: import("./types").AssertionType;
+                            value?: import("./types").AssertionValue | undefined;
+                            transform?: string | undefined;
+                            rubricPrompt?: string | string[] | undefined;
+                            provider?: any;
+                            threshold?: number | undefined;
+                            weight?: number | undefined;
+                            metric?: string | undefined;
+                        } | {
+                            type: "assert-set";
+                            assert: {
+                                type: import("./types").AssertionType;
+                                value?: import("./types").AssertionValue | undefined;
+                                transform?: string | undefined;
+                                rubricPrompt?: string | string[] | undefined;
+                                provider?: any;
+                                threshold?: number | undefined;
+                                weight?: number | undefined;
+                                metric?: string | undefined;
+                            }[];
+                            threshold?: number | undefined;
+                            weight?: number | undefined;
+                            metric?: string | undefined;
+                        })[] | undefined;
+                        threshold?: number | undefined;
+                    }[];
+                    description?: string | undefined;
+                }[] | undefined;
+                defaultTest?: {
+                    options?: ({
+                        prefix?: string | undefined;
+                        suffix?: string | undefined;
+                    } & {
+                        transform?: string | undefined;
+                        postprocess?: string | undefined;
+                        storeOutputAs?: string | undefined;
+                    } & {
+                        rubricPrompt?: string | string[] | undefined;
+                        provider?: any;
+                        factuality?: {
+                            subset?: number | undefined;
+                            superset?: number | undefined;
+                            agree?: number | undefined;
+                            disagree?: number | undefined;
+                            differButFactual?: number | undefined;
+                        } | undefined;
+                    } & {
+                        disableVarExpansion?: boolean | undefined;
+                        disableConversationVar?: boolean | undefined;
+                    }) | undefined;
+                    vars?: Record<string, string | string[] | {}> | undefined;
+                    provider?: string | {
+                        prompts?: string[] | undefined;
+                        delay?: number | undefined;
+                        config?: any;
+                        id?: string | undefined;
+                        label?: string | undefined;
+                        transform?: string | undefined;
+                        env?: {
+                            ANTHROPIC_API_KEY?: string | undefined;
+                            BAM_API_KEY?: string | undefined;
+                            BAM_API_HOST?: string | undefined;
+                            AZURE_OPENAI_API_HOST?: string | undefined;
+                            AZURE_OPENAI_API_KEY?: string | undefined;
+                            AZURE_OPENAI_API_BASE_URL?: string | undefined;
+                            AZURE_OPENAI_BASE_URL?: string | undefined;
+                            AWS_BEDROCK_REGION?: string | undefined;
+                            COHERE_API_KEY?: string | undefined;
+                            OPENAI_API_KEY?: string | undefined;
+                            OPENAI_API_HOST?: string | undefined;
+                            OPENAI_API_BASE_URL?: string | undefined;
+                            OPENAI_BASE_URL?: string | undefined;
+                            OPENAI_ORGANIZATION?: string | undefined;
+                            REPLICATE_API_KEY?: string | undefined;
+                            REPLICATE_API_TOKEN?: string | undefined;
+                            LOCALAI_BASE_URL?: string | undefined;
+                            MISTRAL_API_HOST?: string | undefined;
+                            MISTRAL_API_BASE_URL?: string | undefined;
+                            PALM_API_KEY?: string | undefined;
+                            PALM_API_HOST?: string | undefined;
+                            GOOGLE_API_KEY?: string | undefined;
+                            GOOGLE_API_HOST?: string | undefined;
+                            VERTEX_API_KEY?: string | undefined;
+                            VERTEX_API_HOST?: string | undefined;
+                            VERTEX_PROJECT_ID?: string | undefined;
+                            VERTEX_REGION?: string | undefined;
+                            VERTEX_PUBLISHER?: string | undefined;
+                            MISTRAL_API_KEY?: string | undefined;
+                            CLOUDFLARE_API_KEY?: string | undefined;
+                            CLOUDFLARE_ACCOUNT_ID?: string | undefined;
+                        } | undefined;
+                    } | {
+                        id: (...args: unknown[]) => string;
+                        callApi: (((args_0: string, args_1: {
+                            vars: Record<string, string | {}>;
+                            logger?: any;
+                            fetchWithCache?: any;
+                            getCache?: any;
+                        } | undefined, args_2: {
+                            includeLogProbs?: boolean | undefined;
+                            originalProvider?: any;
+                        } | undefined, ...args_3: unknown[]) => Promise<{
+                            output?: any;
+                            cached?: boolean | undefined;
+                            cost?: number | undefined;
+                            error?: string | undefined;
+                            logProbs?: number[] | undefined;
+                            metadata?: import("zod").objectOutputType<{
+                                redteamFinalPrompt: import("zod").ZodOptional<import("zod").ZodString>;
+                            }, import("zod").ZodAny, "strip"> | undefined;
+                            tokenUsage?: {
+                                prompt?: number | undefined;
+                                cached?: number | undefined;
+                                completion?: number | undefined;
+                                total?: number | undefined;
+                            } | undefined;
+                        }>) & {
+                            label?: string | undefined;
+                        }) | (((args_0: string, args_1: {
+                            includeLogProbs?: boolean | undefined;
+                            originalProvider?: any;
+                        } | undefined, ...args_2: unknown[]) => Promise<{
+                            output?: any;
+                            cached?: boolean | undefined;
+                            cost?: number | undefined;
+                            error?: string | undefined;
+                            logProbs?: number[] | undefined;
+                            metadata?: import("zod").objectOutputType<{
+                                redteamFinalPrompt: import("zod").ZodOptional<import("zod").ZodString>;
+                            }, import("zod").ZodAny, "strip"> | undefined;
+                            tokenUsage?: {
+                                prompt?: number | undefined;
+                                cached?: number | undefined;
+                                completion?: number | undefined;
+                                total?: number | undefined;
+                            } | undefined;
+                        }>) & {
+                            label?: string | undefined;
+                        });
+                        delay?: number | undefined;
+                        label?: string | undefined;
+                        transform?: string | undefined;
+                        callEmbeddingApi?: ((args_0: string, ...args_1: unknown[]) => Promise<{
+                            error?: string | undefined;
+                            tokenUsage?: {
+                                prompt?: number | undefined;
+                                cached?: number | undefined;
+                                completion?: number | undefined;
+                                total?: number | undefined;
+                            } | undefined;
+                            embedding?: number[] | undefined;
+                        }>) | undefined;
+                        callClassificationApi?: ((args_0: string, ...args_1: unknown[]) => Promise<{
+                            error?: string | undefined;
+                            classification?: Record<string, number> | undefined;
+                        }>) | undefined;
+                    } | undefined;
+                    providerOutput?: string | {} | undefined;
+                    assert?: ({
+                        type: import("./types").AssertionType;
+                        value?: import("./types").AssertionValue | undefined;
+                        transform?: string | undefined;
+                        rubricPrompt?: string | string[] | undefined;
+                        provider?: any;
+                        threshold?: number | undefined;
+                        weight?: number | undefined;
+                        metric?: string | undefined;
+                    } | {
+                        type: "assert-set";
+                        assert: {
+                            type: import("./types").AssertionType;
+                            value?: import("./types").AssertionValue | undefined;
+                            transform?: string | undefined;
+                            rubricPrompt?: string | string[] | undefined;
+                            provider?: any;
+                            threshold?: number | undefined;
+                            weight?: number | undefined;
+                            metric?: string | undefined;
+                        }[];
+                        threshold?: number | undefined;
+                        weight?: number | undefined;
+                        metric?: string | undefined;
+                    })[] | undefined;
+                    threshold?: number | undefined;
+                } | undefined;
+                nunjucksFilters?: Record<string, string> | undefined;
+                derivedMetrics?: {
+                    value: string | ((args_0: Record<string, number>, args_1: import("./types").RunEvalOptions, ...args_2: unknown[]) => number);
+                    name: string;
+                }[] | undefined;
+                outputPath?: string | string[] | undefined;
+                sharing?: boolean | {
+                    apiBaseUrl?: string | undefined;
+                    appBaseUrl?: string | undefined;
+                } | undefined;
+            }>;
             driverParam: string;
             notNull: true;
             hasDefault: false;
@@ -224,7 +1193,164 @@ export declare const datasets: import("drizzle-orm/sqlite-core").SQLiteTableWith
             tableName: "datasets";
             dataType: "json";
             columnType: "SQLiteTextJson";
-            data: string | (string | import("./types").TestCase<Record<string, string | object | string[]>>)[];
+            data: string | (string | {
+                options?: ({
+                    prefix?: string | undefined;
+                    suffix?: string | undefined;
+                } & {
+                    transform?: string | undefined;
+                    postprocess?: string | undefined;
+                    storeOutputAs?: string | undefined;
+                } & {
+                    rubricPrompt?: string | string[] | undefined;
+                    provider?: any;
+                    factuality?: {
+                        subset?: number | undefined;
+                        superset?: number | undefined;
+                        agree?: number | undefined;
+                        disagree?: number | undefined;
+                        differButFactual?: number | undefined;
+                    } | undefined;
+                } & {
+                    disableVarExpansion?: boolean | undefined;
+                    disableConversationVar?: boolean | undefined;
+                }) | undefined;
+                vars?: Record<string, string | string[] | {}> | undefined;
+                provider?: string | {
+                    prompts?: string[] | undefined;
+                    delay?: number | undefined;
+                    config?: any;
+                    id?: string | undefined;
+                    label?: string | undefined;
+                    transform?: string | undefined;
+                    env?: {
+                        ANTHROPIC_API_KEY?: string | undefined;
+                        BAM_API_KEY?: string | undefined;
+                        BAM_API_HOST?: string | undefined;
+                        AZURE_OPENAI_API_HOST?: string | undefined;
+                        AZURE_OPENAI_API_KEY?: string | undefined;
+                        AZURE_OPENAI_API_BASE_URL?: string | undefined;
+                        AZURE_OPENAI_BASE_URL?: string | undefined;
+                        AWS_BEDROCK_REGION?: string | undefined;
+                        COHERE_API_KEY?: string | undefined;
+                        OPENAI_API_KEY?: string | undefined;
+                        OPENAI_API_HOST?: string | undefined;
+                        OPENAI_API_BASE_URL?: string | undefined;
+                        OPENAI_BASE_URL?: string | undefined;
+                        OPENAI_ORGANIZATION?: string | undefined;
+                        REPLICATE_API_KEY?: string | undefined;
+                        REPLICATE_API_TOKEN?: string | undefined;
+                        LOCALAI_BASE_URL?: string | undefined;
+                        MISTRAL_API_HOST?: string | undefined;
+                        MISTRAL_API_BASE_URL?: string | undefined;
+                        PALM_API_KEY?: string | undefined;
+                        PALM_API_HOST?: string | undefined;
+                        GOOGLE_API_KEY?: string | undefined;
+                        GOOGLE_API_HOST?: string | undefined;
+                        VERTEX_API_KEY?: string | undefined;
+                        VERTEX_API_HOST?: string | undefined;
+                        VERTEX_PROJECT_ID?: string | undefined;
+                        VERTEX_REGION?: string | undefined;
+                        VERTEX_PUBLISHER?: string | undefined;
+                        MISTRAL_API_KEY?: string | undefined;
+                        CLOUDFLARE_API_KEY?: string | undefined;
+                        CLOUDFLARE_ACCOUNT_ID?: string | undefined;
+                    } | undefined;
+                } | {
+                    id: (...args: unknown[]) => string;
+                    callApi: (((args_0: string, args_1: {
+                        vars: Record<string, string | {}>;
+                        logger?: any;
+                        fetchWithCache?: any;
+                        getCache?: any;
+                    } | undefined, args_2: {
+                        includeLogProbs?: boolean | undefined;
+                        originalProvider?: any;
+                    } | undefined, ...args_3: unknown[]) => Promise<{
+                        output?: any;
+                        cached?: boolean | undefined;
+                        cost?: number | undefined;
+                        error?: string | undefined;
+                        logProbs?: number[] | undefined;
+                        metadata?: import("zod").objectOutputType<{
+                            redteamFinalPrompt: import("zod").ZodOptional<import("zod").ZodString>;
+                        }, import("zod").ZodAny, "strip"> | undefined;
+                        tokenUsage?: {
+                            prompt?: number | undefined;
+                            cached?: number | undefined;
+                            completion?: number | undefined;
+                            total?: number | undefined;
+                        } | undefined;
+                    }>) & {
+                        label?: string | undefined;
+                    }) | (((args_0: string, args_1: {
+                        includeLogProbs?: boolean | undefined;
+                        originalProvider?: any;
+                    } | undefined, ...args_2: unknown[]) => Promise<{
+                        output?: any;
+                        cached?: boolean | undefined;
+                        cost?: number | undefined;
+                        error?: string | undefined;
+                        logProbs?: number[] | undefined;
+                        metadata?: import("zod").objectOutputType<{
+                            redteamFinalPrompt: import("zod").ZodOptional<import("zod").ZodString>;
+                        }, import("zod").ZodAny, "strip"> | undefined;
+                        tokenUsage?: {
+                            prompt?: number | undefined;
+                            cached?: number | undefined;
+                            completion?: number | undefined;
+                            total?: number | undefined;
+                        } | undefined;
+                    }>) & {
+                        label?: string | undefined;
+                    });
+                    delay?: number | undefined;
+                    label?: string | undefined;
+                    transform?: string | undefined;
+                    callEmbeddingApi?: ((args_0: string, ...args_1: unknown[]) => Promise<{
+                        error?: string | undefined;
+                        tokenUsage?: {
+                            prompt?: number | undefined;
+                            cached?: number | undefined;
+                            completion?: number | undefined;
+                            total?: number | undefined;
+                        } | undefined;
+                        embedding?: number[] | undefined;
+                    }>) | undefined;
+                    callClassificationApi?: ((args_0: string, ...args_1: unknown[]) => Promise<{
+                        error?: string | undefined;
+                        classification?: Record<string, number> | undefined;
+                    }>) | undefined;
+                } | undefined;
+                description?: string | undefined;
+                providerOutput?: string | {} | undefined;
+                assert?: ({
+                    type: import("./types").AssertionType;
+                    value?: import("./types").AssertionValue | undefined;
+                    transform?: string | undefined;
+                    rubricPrompt?: string | string[] | undefined;
+                    provider?: any;
+                    threshold?: number | undefined;
+                    weight?: number | undefined;
+                    metric?: string | undefined;
+                } | {
+                    type: "assert-set";
+                    assert: {
+                        type: import("./types").AssertionType;
+                        value?: import("./types").AssertionValue | undefined;
+                        transform?: string | undefined;
+                        rubricPrompt?: string | string[] | undefined;
+                        provider?: any;
+                        threshold?: number | undefined;
+                        weight?: number | undefined;
+                        metric?: string | undefined;
+                    }[];
+                    threshold?: number | undefined;
+                    weight?: number | undefined;
+                    metric?: string | undefined;
+                })[] | undefined;
+                threshold?: number | undefined;
+            })[];
             driverParam: string;
             notNull: false;
             hasDefault: false;