promptfoo 0.73.8 → 0.74.0

package/dist/src/commands/share.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"share.d.ts","sourceRoot":"","sources":["../../../src/commands/share.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAcpC,wBAAgB,YAAY,CAAC,OAAO,EAAE,OAAO,QAiE5C"}
+{"version":3,"file":"share.d.ts","sourceRoot":"","sources":["../../../src/commands/share.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAczC,wBAAgB,YAAY,CAAC,OAAO,EAAE,OAAO,QAiE5C"}

package/dist/src/commands/show.d.ts

@@ -1,3 +1,3 @@
-import { Command } from 'commander';
+import type { Command } from 'commander';
 export declare function showCommand(program: Command): Promise<void>;
 //# sourceMappingURL=show.d.ts.map

package/dist/src/commands/show.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"show.d.ts","sourceRoot":"","sources":["../../../src/commands/show.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AA6IpC,wBAAsB,WAAW,CAAC,OAAO,EAAE,OAAO,iBA6CjD"}
+{"version":3,"file":"show.d.ts","sourceRoot":"","sources":["../../../src/commands/show.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AA6IzC,wBAAsB,WAAW,CAAC,OAAO,EAAE,OAAO,iBA6CjD"}

package/dist/src/commands/version.d.ts

@@ -1,3 +1,3 @@
-import { Command } from 'commander';
+import type { Command } from 'commander';
 export declare function versionCommand(program: Command): void;
 //# sourceMappingURL=version.d.ts.map

package/dist/src/commands/version.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"version.d.ts","sourceRoot":"","sources":["../../../src/commands/version.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAMpC,wBAAgB,cAAc,CAAC,OAAO,EAAE,OAAO,QAQ9C"}
+{"version":3,"file":"version.d.ts","sourceRoot":"","sources":["../../../src/commands/version.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAMzC,wBAAgB,cAAc,CAAC,OAAO,EAAE,OAAO,QAQ9C"}

package/dist/src/commands/view.d.ts

@@ -1,3 +1,3 @@
-import { Command } from 'commander';
+import type { Command } from 'commander';
 export declare function viewCommand(program: Command): void;
 //# sourceMappingURL=view.d.ts.map

package/dist/src/commands/view.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"view.d.ts","sourceRoot":"","sources":["../../../src/commands/view.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAMpC,wBAAgB,WAAW,CAAC,OAAO,EAAE,OAAO,QA8C3C"}
+{"version":3,"file":"view.d.ts","sourceRoot":"","sources":["../../../src/commands/view.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAMzC,wBAAgB,WAAW,CAAC,OAAO,EAAE,OAAO,QA8C3C"}

package/dist/src/config.d.ts

@@ -1,4 +1,4 @@
-import { CommandLineOptions, TestSuite, UnifiedConfig } from './types';
+import type { CommandLineOptions, TestSuite, UnifiedConfig } from './types';
 export declare function dereferenceConfig(rawConfig: UnifiedConfig): Promise<UnifiedConfig>;
 export declare function readConfig(configPath: string): Promise<UnifiedConfig>;
 export declare function maybeReadConfig(configPath: string): Promise<UnifiedConfig | undefined>;

package/dist/src/config.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../../src/config.ts"],"names":[],"mappings":"AAcA,OAAO,EACL,kBAAkB,EAIlB,SAAS,EACT,aAAa,EACd,MAAM,SAAS,CAAC;AAGjB,wBAAsB,iBAAiB,CAAC,SAAS,EAAE,aAAa,GAAG,OAAO,CAAC,aAAa,CAAC,CA4GxF;AAED,wBAAsB,UAAU,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,CAAC,CAkB3E;AAED,wBAAsB,eAAe,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,GAAG,SAAS,CAAC,CAK5F;AAED;;;;;GAKG;AACH,wBAAsB,WAAW,CAAC,WAAW,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,aAAa,CAAC,CAkI/E;AAED,wBAAsB,cAAc,CAClC,MAAM,EAAE,OAAO,CAAC,kBAAkB,CAAC,EACnC,aAAa,EAAE,OAAO,CAAC,aAAa,CAAC,GACpC,OAAO,CAAC;IAAE,SAAS,EAAE,SAAS,CAAC;IAAC,MAAM,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAA;CAAE,CAAC,CAkJrF"}
+{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../../src/config.ts"],"names":[],"mappings":"AAcA,OAAO,KAAK,EACV,kBAAkB,EAIlB,SAAS,EACT,aAAa,EACd,MAAM,SAAS,CAAC;AAGjB,wBAAsB,iBAAiB,CAAC,SAAS,EAAE,aAAa,GAAG,OAAO,CAAC,aAAa,CAAC,CA4GxF;AAED,wBAAsB,UAAU,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,CAAC,CAkB3E;AAED,wBAAsB,eAAe,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,GAAG,SAAS,CAAC,CAK5F;AAED;;;;;GAKG;AACH,wBAAsB,WAAW,CAAC,WAAW,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,aAAa,CAAC,CAkI/E;AAED,wBAAsB,cAAc,CAClC,MAAM,EAAE,OAAO,CAAC,kBAAkB,CAAC,EACnC,aAAa,EAAE,OAAO,CAAC,aAAa,CAAC,GACpC,OAAO,CAAC;IAAE,SAAS,EAAE,SAAS,CAAC;IAAC,MAAM,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAA;CAAE,CAAC,CAkJrF"}

package/dist/src/database/operations.d.ts

@@ -145,34 +145,11 @@ export declare const evals: import("drizzle-orm/sqlite-core").SQLiteTableWithCol
             columnType: "SQLiteTextJson";
             data: Partial<{
                 prompts: string | Record<string, string> | (string | {
-                    label: string;
                     raw: string;
+                    label: string;
                     function?: ((args_0: {
                         vars: Record<string, any>;
-                        provider?: {
-                            id: (...args: unknown[]) => string;
-                            callApi: {
-                                (prompt: string, context?: import("../types").CallApiContextParams, options?: import("../types").CallApiOptionsParams): Promise<import("../types").ProviderResponse>;
-                                label?: string;
-                            };
-                            delay?: number | undefined;
-                            label?: string | undefined;
-                            transform?: string | undefined;
-                            callEmbeddingApi?: ((args_0: string, ...args_1: unknown[]) => Promise<{
-                                error?: string | undefined;
-                                embedding?: number[] | undefined;
-                                tokenUsage?: {
-                                    cached?: number | undefined;
-                                    completion?: number | undefined;
-                                    prompt?: number | undefined;
-                                    total?: number | undefined;
-                                } | undefined;
-                            }>) | undefined;
-                            callClassificationApi?: ((args_0: string, ...args_1: unknown[]) => Promise<{
-                                error?: string | undefined;
-                                classification?: Record<string, number> | undefined;
-                            }>) | undefined;
-                        } | undefined;
+                        provider?: import("../types").ApiProvider | undefined;
                     }, ...args_1: unknown[]) => Promise<any>) | undefined;
                     id?: string | undefined;
                     display?: string | undefined;
@@ -180,34 +157,11 @@ export declare const evals: import("drizzle-orm/sqlite-core").SQLiteTableWithCol
                 providers: string | (((args_0: string, args_1: {
                     vars: Record<string, string | {}>;
                     prompt: {
-                        label: string;
                         raw: string;
+                        label: string;
                         function?: ((args_0: {
                             vars: Record<string, any>;
-                            provider?: {
-                                id: (...args: unknown[]) => string;
-                                callApi: {
-                                    (prompt: string, context?: import("../types").CallApiContextParams, options?: import("../types").CallApiOptionsParams): Promise<import("../types").ProviderResponse>;
-                                    label?: string;
-                                };
-                                delay?: number | undefined;
-                                label?: string | undefined;
-                                transform?: string | undefined;
-                                callEmbeddingApi?: ((args_0: string, ...args_1: unknown[]) => Promise<{
-                                    error?: string | undefined;
-                                    embedding?: number[] | undefined;
-                                    tokenUsage?: {
-                                        cached?: number | undefined;
-                                        completion?: number | undefined;
-                                        prompt?: number | undefined;
-                                        total?: number | undefined;
-                                    } | undefined;
-                                }>) | undefined;
-                                callClassificationApi?: ((args_0: string, ...args_1: unknown[]) => Promise<{
-                                    error?: string | undefined;
-                                    classification?: Record<string, number> | undefined;
-                                }>) | undefined;
-                            } | undefined;
+                            provider?: import("../types").ApiProvider | undefined;
                         }, ...args_1: unknown[]) => Promise<any>) | undefined;
                         id?: string | undefined;
                         display?: string | undefined;
@@ -219,30 +173,15 @@ export declare const evals: import("drizzle-orm/sqlite-core").SQLiteTableWithCol
                     originalProvider?: any;
                 } | undefined, args_2: {
                     includeLogProbs?: boolean | undefined;
-                } | undefined, ...args_3: unknown[]) => Promise<{
-                    output?: any;
-                    error?: string | undefined;
-                    cached?: boolean | undefined;
-                    tokenUsage?: {
-                        cached?: number | undefined;
-                        completion?: number | undefined;
-                        prompt?: number | undefined;
-                        total?: number | undefined;
-                    } | undefined;
-                    cost?: number | undefined;
-                    logProbs?: number[] | undefined;
-                    metadata?: import("zod").objectOutputType<{
-                        redteamFinalPrompt: import("zod").ZodOptional<import("zod").ZodString>;
-                    }, import("zod").ZodAny, "strip"> | undefined;
-                }>) & {
+                } | undefined, ...args_3: unknown[]) => Promise<import("../types").ProviderResponse>) & {
                     label?: string | undefined;
                 }) | (string | {
                     id?: string | undefined;
+                    label?: string | undefined;
                     config?: any;
                     prompts?: string[] | undefined;
-                    delay?: number | undefined;
-                    label?: string | undefined;
                     transform?: string | undefined;
+                    delay?: number | undefined;
                     env?: {
                         ANTHROPIC_API_KEY?: string | undefined;
                         BAM_API_KEY?: string | undefined;
@@ -279,34 +218,11 @@ export declare const evals: import("drizzle-orm/sqlite-core").SQLiteTableWithCol
                 } | (((args_0: string, args_1: {
                     vars: Record<string, string | {}>;
                     prompt: {
-                        label: string;
                         raw: string;
+                        label: string;
                         function?: ((args_0: {
                             vars: Record<string, any>;
-                            provider?: {
-                                id: (...args: unknown[]) => string;
-                                callApi: {
-                                    (prompt: string, context?: import("../types").CallApiContextParams, options?: import("../types").CallApiOptionsParams): Promise<import("../types").ProviderResponse>;
-                                    label?: string;
-                                };
-                                delay?: number | undefined;
-                                label?: string | undefined;
-                                transform?: string | undefined;
-                                callEmbeddingApi?: ((args_0: string, ...args_1: unknown[]) => Promise<{
-                                    error?: string | undefined;
-                                    embedding?: number[] | undefined;
-                                    tokenUsage?: {
-                                        cached?: number | undefined;
-                                        completion?: number | undefined;
-                                        prompt?: number | undefined;
-                                        total?: number | undefined;
-                                    } | undefined;
-                                }>) | undefined;
-                                callClassificationApi?: ((args_0: string, ...args_1: unknown[]) => Promise<{
-                                    error?: string | undefined;
-                                    classification?: Record<string, number> | undefined;
-                                }>) | undefined;
-                            } | undefined;
+                            provider?: import("../types").ApiProvider | undefined;
                         }, ...args_1: unknown[]) => Promise<any>) | undefined;
                         id?: string | undefined;
                         display?: string | undefined;
@@ -318,30 +234,15 @@ export declare const evals: import("drizzle-orm/sqlite-core").SQLiteTableWithCol
                     originalProvider?: any;
                 } | undefined, args_2: {
                     includeLogProbs?: boolean | undefined;
-                } | undefined, ...args_3: unknown[]) => Promise<{
-                    output?: any;
-                    error?: string | undefined;
-                    cached?: boolean | undefined;
-                    tokenUsage?: {
-                        cached?: number | undefined;
-                        completion?: number | undefined;
-                        prompt?: number | undefined;
-                        total?: number | undefined;
-                    } | undefined;
-                    cost?: number | undefined;
-                    logProbs?: number[] | undefined;
-                    metadata?: import("zod").objectOutputType<{
-                        redteamFinalPrompt: import("zod").ZodOptional<import("zod").ZodString>;
-                    }, import("zod").ZodAny, "strip"> | undefined;
-                }>) & {
+                } | undefined, ...args_3: unknown[]) => Promise<import("../types").ProviderResponse>) & {
                     label?: string | undefined;
                 }) | Record<string, {
                     id?: string | undefined;
+                    label?: string | undefined;
                     config?: any;
                     prompts?: string[] | undefined;
-                    delay?: number | undefined;
-                    label?: string | undefined;
                     transform?: string | undefined;
+                    delay?: number | undefined;
                     env?: {
                         ANTHROPIC_API_KEY?: string | undefined;
                         BAM_API_KEY?: string | undefined;
@@ -401,13 +302,14 @@ export declare const evals: import("drizzle-orm/sqlite-core").SQLiteTableWithCol
                         disableVarExpansion?: boolean | undefined;
                         disableConversationVar?: boolean | undefined;
                     }) | undefined;
+                    vars?: Record<string, string | any[] | string[] | {}> | undefined;
                     provider?: string | {
                         id?: string | undefined;
+                        label?: string | undefined;
                         config?: any;
                         prompts?: string[] | undefined;
-                        delay?: number | undefined;
-                        label?: string | undefined;
                         transform?: string | undefined;
+                        delay?: number | undefined;
                         env?: {
                             ANTHROPIC_API_KEY?: string | undefined;
                             BAM_API_KEY?: string | undefined;
@@ -442,30 +344,14 @@ export declare const evals: import("drizzle-orm/sqlite-core").SQLiteTableWithCol
                             CLOUDFLARE_ACCOUNT_ID?: string | undefined;
                         } | undefined;
                     } | {
+                        callApi: import("../types").CallApiFunction;
                         id: (...args: unknown[]) => string;
-                        callApi: {
-                            (prompt: string, context?: import("../types").CallApiContextParams, options?: import("../types").CallApiOptionsParams): Promise<import("../types").ProviderResponse>;
-                            label?: string;
-                        };
-                        delay?: number | undefined;
                         label?: string | undefined;
                         transform?: string | undefined;
-                        callEmbeddingApi?: ((args_0: string, ...args_1: unknown[]) => Promise<{
-                            error?: string | undefined;
-                            embedding?: number[] | undefined;
-                            tokenUsage?: {
-                                cached?: number | undefined;
-                                completion?: number | undefined;
-                                prompt?: number | undefined;
-                                total?: number | undefined;
-                            } | undefined;
-                        }>) | undefined;
-                        callClassificationApi?: ((args_0: string, ...args_1: unknown[]) => Promise<{
-                            error?: string | undefined;
-                            classification?: Record<string, number> | undefined;
-                        }>) | undefined;
+                        delay?: number | undefined;
+                        callEmbeddingApi?: ((args_0: string, ...args_1: unknown[]) => Promise<import("../types").ProviderEmbeddingResponse>) | undefined;
+                        callClassificationApi?: ((args_0: string, ...args_1: unknown[]) => Promise<import("../types").ProviderClassificationResponse>) | undefined;
                     } | undefined;
-                    vars?: Record<string, string | any[] | string[] | {}> | undefined;
                     metadata?: Record<string, any> | undefined;
                     description?: string | undefined;
                     providerOutput?: string | {} | undefined;
@@ -563,13 +449,14 @@ export declare const evals: import("drizzle-orm/sqlite-core").SQLiteTableWithCol
                             disableVarExpansion?: boolean | undefined;
                             disableConversationVar?: boolean | undefined;
                         }) | undefined;
+                        vars?: Record<string, string | any[] | string[] | {}> | undefined;
                         provider?: string | {
                             id?: string | undefined;
+                            label?: string | undefined;
                             config?: any;
                             prompts?: string[] | undefined;
-                            delay?: number | undefined;
-                            label?: string | undefined;
                             transform?: string | undefined;
+                            delay?: number | undefined;
                             env?: {
                                 ANTHROPIC_API_KEY?: string | undefined;
                                 BAM_API_KEY?: string | undefined;
@@ -604,30 +491,14 @@ export declare const evals: import("drizzle-orm/sqlite-core").SQLiteTableWithCol
                                 CLOUDFLARE_ACCOUNT_ID?: string | undefined;
                             } | undefined;
                         } | {
+                            callApi: import("../types").CallApiFunction;
                             id: (...args: unknown[]) => string;
-                            callApi: {
-                                (prompt: string, context?: import("../types").CallApiContextParams, options?: import("../types").CallApiOptionsParams): Promise<import("../types").ProviderResponse>;
-                                label?: string;
-                            };
-                            delay?: number | undefined;
                             label?: string | undefined;
                             transform?: string | undefined;
-                            callEmbeddingApi?: ((args_0: string, ...args_1: unknown[]) => Promise<{
-                                error?: string | undefined;
-                                embedding?: number[] | undefined;
-                                tokenUsage?: {
-                                    cached?: number | undefined;
-                                    completion?: number | undefined;
-                                    prompt?: number | undefined;
-                                    total?: number | undefined;
-                                } | undefined;
-                            }>) | undefined;
-                            callClassificationApi?: ((args_0: string, ...args_1: unknown[]) => Promise<{
-                                error?: string | undefined;
-                                classification?: Record<string, number> | undefined;
-                            }>) | undefined;
+                            delay?: number | undefined;
+                            callEmbeddingApi?: ((args_0: string, ...args_1: unknown[]) => Promise<import("../types").ProviderEmbeddingResponse>) | undefined;
+                            callClassificationApi?: ((args_0: string, ...args_1: unknown[]) => Promise<import("../types").ProviderClassificationResponse>) | undefined;
                         } | undefined;
-                        vars?: Record<string, string | any[] | string[] | {}> | undefined;
                         metadata?: Record<string, any> | undefined;
                         description?: string | undefined;
                         providerOutput?: string | {} | undefined;
@@ -689,13 +560,14 @@ export declare const evals: import("drizzle-orm/sqlite-core").SQLiteTableWithCol
                             disableVarExpansion?: boolean | undefined;
                             disableConversationVar?: boolean | undefined;
                         }) | undefined;
+                        vars?: Record<string, string | any[] | string[] | {}> | undefined;
                         provider?: string | {
                             id?: string | undefined;
+                            label?: string | undefined;
                             config?: any;
                             prompts?: string[] | undefined;
-                            delay?: number | undefined;
-                            label?: string | undefined;
                             transform?: string | undefined;
+                            delay?: number | undefined;
                             env?: {
                                 ANTHROPIC_API_KEY?: string | undefined;
                                 BAM_API_KEY?: string | undefined;
@@ -730,30 +602,14 @@ export declare const evals: import("drizzle-orm/sqlite-core").SQLiteTableWithCol
                                 CLOUDFLARE_ACCOUNT_ID?: string | undefined;
                             } | undefined;
                         } | {
+                            callApi: import("../types").CallApiFunction;
                             id: (...args: unknown[]) => string;
-                            callApi: {
-                                (prompt: string, context?: import("../types").CallApiContextParams, options?: import("../types").CallApiOptionsParams): Promise<import("../types").ProviderResponse>;
-                                label?: string;
-                            };
-                            delay?: number | undefined;
                             label?: string | undefined;
                             transform?: string | undefined;
-                            callEmbeddingApi?: ((args_0: string, ...args_1: unknown[]) => Promise<{
-                                error?: string | undefined;
-                                embedding?: number[] | undefined;
-                                tokenUsage?: {
-                                    cached?: number | undefined;
-                                    completion?: number | undefined;
-                                    prompt?: number | undefined;
-                                    total?: number | undefined;
-                                } | undefined;
-                            }>) | undefined;
-                            callClassificationApi?: ((args_0: string, ...args_1: unknown[]) => Promise<{
-                                error?: string | undefined;
-                                classification?: Record<string, number> | undefined;
-                            }>) | undefined;
+                            delay?: number | undefined;
+                            callEmbeddingApi?: ((args_0: string, ...args_1: unknown[]) => Promise<import("../types").ProviderEmbeddingResponse>) | undefined;
+                            callClassificationApi?: ((args_0: string, ...args_1: unknown[]) => Promise<import("../types").ProviderClassificationResponse>) | undefined;
                         } | undefined;
-                        vars?: Record<string, string | any[] | string[] | {}> | undefined;
                         metadata?: Record<string, any> | undefined;
                         description?: string | undefined;
                         providerOutput?: string | {} | undefined;
@@ -817,13 +673,14 @@ export declare const evals: import("drizzle-orm/sqlite-core").SQLiteTableWithCol
                         disableVarExpansion?: boolean | undefined;
                         disableConversationVar?: boolean | undefined;
                     }) | undefined;
+                    vars?: Record<string, string | any[] | string[] | {}> | undefined;
                     provider?: string | {
                         id?: string | undefined;
+                        label?: string | undefined;
                         config?: any;
                         prompts?: string[] | undefined;
-                        delay?: number | undefined;
-                        label?: string | undefined;
                         transform?: string | undefined;
+                        delay?: number | undefined;
                         env?: {
                             ANTHROPIC_API_KEY?: string | undefined;
                             BAM_API_KEY?: string | undefined;
@@ -858,30 +715,14 @@ export declare const evals: import("drizzle-orm/sqlite-core").SQLiteTableWithCol
                             CLOUDFLARE_ACCOUNT_ID?: string | undefined;
                         } | undefined;
                     } | {
+                        callApi: import("../types").CallApiFunction;
                         id: (...args: unknown[]) => string;
-                        callApi: {
-                            (prompt: string, context?: import("../types").CallApiContextParams, options?: import("../types").CallApiOptionsParams): Promise<import("../types").ProviderResponse>;
-                            label?: string;
-                        };
-                        delay?: number | undefined;
                         label?: string | undefined;
                         transform?: string | undefined;
-                        callEmbeddingApi?: ((args_0: string, ...args_1: unknown[]) => Promise<{
-                            error?: string | undefined;
-                            embedding?: number[] | undefined;
-                            tokenUsage?: {
-                                cached?: number | undefined;
-                                completion?: number | undefined;
-                                prompt?: number | undefined;
-                                total?: number | undefined;
-                            } | undefined;
-                        }>) | undefined;
-                        callClassificationApi?: ((args_0: string, ...args_1: unknown[]) => Promise<{
-                            error?: string | undefined;
-                            classification?: Record<string, number> | undefined;
-                        }>) | undefined;
+                        delay?: number | undefined;
+                        callEmbeddingApi?: ((args_0: string, ...args_1: unknown[]) => Promise<import("../types").ProviderEmbeddingResponse>) | undefined;
+                        callClassificationApi?: ((args_0: string, ...args_1: unknown[]) => Promise<import("../types").ProviderClassificationResponse>) | undefined;
                     } | undefined;
-                    vars?: Record<string, string | any[] | string[] | {}> | undefined;
                     metadata?: Record<string, any> | undefined;
                     providerOutput?: string | {} | undefined;
                     assert?: ({
@@ -922,17 +763,7 @@ export declare const evals: import("drizzle-orm/sqlite-core").SQLiteTableWithCol
                     value: string | ((args_0: Record<string, number>, args_1: import("../types").RunEvalOptions, ...args_2: unknown[]) => number);
                     name: string;
                 }[] | undefined;
-                redteam?: {
-                    plugins: any[];
-                    strategies: ({
-                        id: string;
-                    } | {
-                        id: string;
-                    })[];
-                    provider?: string | undefined;
-                    injectVar?: string | undefined;
-                    purpose?: string | undefined;
-                } | undefined;
+                redteam?: import("../types").RedteamConfig | undefined;
                 outputPath?: string | string[] | undefined;
                 sharing?: boolean | {
                     apiBaseUrl?: string | undefined;
@@ -940,9 +771,9 @@ export declare const evals: import("drizzle-orm/sqlite-core").SQLiteTableWithCol
                 } | undefined;
                 evaluateOptions?: {
                     repeat?: number | undefined;
-                    cache?: boolean | undefined;
-                    maxConcurrency?: number | undefined;
                     delay?: number | undefined;
+                    maxConcurrency?: number | undefined;
+                    cache?: boolean | undefined;
                     interactiveProviders?: boolean | undefined;
                     generateSuggestions?: boolean | undefined;
                     showProgressBar?: boolean | undefined;
@@ -951,22 +782,21 @@ export declare const evals: import("drizzle-orm/sqlite-core").SQLiteTableWithCol
                 } | undefined;
                 commandLineOptions?: {
                     repeat?: string | undefined;
-                    cache?: boolean | undefined;
+                    vars?: string | undefined;
                     config?: string[] | undefined;
-                    envFile?: string | undefined;
-                    output?: string[] | undefined;
-                    write?: boolean | undefined;
                     prompts?: string[] | undefined;
+                    delay?: string | undefined;
+                    output?: string[] | undefined;
                     providers?: string[] | undefined;
                     maxConcurrency?: string | undefined;
-                    delay?: string | undefined;
-                    vars?: string | undefined;
                     tests?: string | undefined;
                     assertions?: string | undefined;
                     modelOutputs?: string | undefined;
                     verbose?: boolean | undefined;
                     grader?: string | undefined;
                     tableCellMaxLength?: string | undefined;
+                    write?: boolean | undefined;
+                    cache?: boolean | undefined;
                     table?: boolean | undefined;
                     share?: boolean | undefined;
                     progressBar?: boolean | undefined;
@@ -980,6 +810,7 @@ export declare const evals: import("drizzle-orm/sqlite-core").SQLiteTableWithCol
                     generateSuggestions?: boolean | undefined;
                     promptPrefix?: string | undefined;
                     promptSuffix?: string | undefined;
+                    envFile?: string | undefined;
                 } | undefined;
             }>;
             driverParam: string;
@@ -1087,13 +918,14 @@ export declare const datasets: import("drizzle-orm/sqlite-core").SQLiteTableWith
                     disableVarExpansion?: boolean | undefined;
                     disableConversationVar?: boolean | undefined;
                 }) | undefined;
+                vars?: Record<string, string | any[] | string[] | {}> | undefined;
                 provider?: string | {
                     id?: string | undefined;
+                    label?: string | undefined;
                     config?: any;
                     prompts?: string[] | undefined;
-                    delay?: number | undefined;
-                    label?: string | undefined;
                     transform?: string | undefined;
+                    delay?: number | undefined;
                     env?: {
                         ANTHROPIC_API_KEY?: string | undefined;
                         BAM_API_KEY?: string | undefined;
@@ -1128,30 +960,14 @@ export declare const datasets: import("drizzle-orm/sqlite-core").SQLiteTableWith
                         CLOUDFLARE_ACCOUNT_ID?: string | undefined;
                     } | undefined;
                 } | {
+                    callApi: import("../types").CallApiFunction;
                     id: (...args: unknown[]) => string;
-                    callApi: {
-                        (prompt: string, context?: import("../types").CallApiContextParams, options?: import("../types").CallApiOptionsParams): Promise<import("../types").ProviderResponse>;
-                        label?: string;
-                    };
-                    delay?: number | undefined;
                     label?: string | undefined;
                     transform?: string | undefined;
-                    callEmbeddingApi?: ((args_0: string, ...args_1: unknown[]) => Promise<{
-                        error?: string | undefined;
-                        embedding?: number[] | undefined;
-                        tokenUsage?: {
-                            cached?: number | undefined;
-                            completion?: number | undefined;
-                            prompt?: number | undefined;
-                            total?: number | undefined;
-                        } | undefined;
-                    }>) | undefined;
-                    callClassificationApi?: ((args_0: string, ...args_1: unknown[]) => Promise<{
-                        error?: string | undefined;
-                        classification?: Record<string, number> | undefined;
-                    }>) | undefined;
+                    delay?: number | undefined;
+                    callEmbeddingApi?: ((args_0: string, ...args_1: unknown[]) => Promise<import("../types").ProviderEmbeddingResponse>) | undefined;
+                    callClassificationApi?: ((args_0: string, ...args_1: unknown[]) => Promise<import("../types").ProviderClassificationResponse>) | undefined;
                 } | undefined;
-                vars?: Record<string, string | any[] | string[] | {}> | undefined;
                 metadata?: Record<string, any> | undefined;
                 description?: string | undefined;
                 providerOutput?: string | {} | undefined;

package/dist/src/database/operations.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"operations.d.ts","sourceRoot":"","sources":["../../../src/database/operations.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,eAAe,EAAiB,MAAM,UAAU,CAAC;AAI/D,eAAO,MAAM,OAAO;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAMlB,CAAC;AAEH,eAAO,MAAM,KAAK;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAShB,CAAC;AAEH,eAAO,MAAM,cAAc;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAe1B,CAAC;AAEF,eAAO,MAAM,gBAAgB;;EAE1B,CAAC;AAIJ,eAAO,MAAM,QAAQ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAMnB,CAAC;AAEH,eAAO,MAAM,eAAe;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAe3B,CAAC;AAEF,eAAO,MAAM,iBAAiB;;EAE3B,CAAC;AAIJ,eAAO,MAAM,cAAc;;;EAGxB,CAAC;AAEJ,eAAO,MAAM,uBAAuB;;;EASjC,CAAC;AAEJ,eAAO,MAAM,wBAAwB;;;EASlC,CAAC"}
+{"version":3,"file":"operations.d.ts","sourceRoot":"","sources":["../../../src/database/operations.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,eAAe,EAAiB,MAAM,UAAU,CAAC;AAI/D,eAAO,MAAM,OAAO;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAMlB,CAAC;AAEH,eAAO,MAAM,KAAK;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAShB,CAAC;AAEH,eAAO,MAAM,cAAc;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAe1B,CAAC;AAEF,eAAO,MAAM,gBAAgB;;EAE1B,CAAC;AAIJ,eAAO,MAAM,QAAQ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAMnB,CAAC;AAEH,eAAO,MAAM,eAAe;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAe3B,CAAC;AAEF,eAAO,MAAM,iBAAiB;;EAE3B,CAAC;AAIJ,eAAO,MAAM,cAAc;;;EAGxB,CAAC;AAEJ,eAAO,MAAM,uBAAuB;;;EASjC,CAAC;AAEJ,eAAO,MAAM,wBAAwB;;;EASlC,CAAC"}

package/dist/src/database/types.d.ts

@@ -1,4 +1,4 @@
-import { EvaluateSummary, UnifiedConfig } from '../types';
+import type { EvaluateSummary, UnifiedConfig } from '../types';
 export interface ResultsFile {
     version: number;
     createdAt: string;

package/dist/src/database/types.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/database/types.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAE1D,MAAM,WAAW,WAAW;IAC1B,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,OAAO,EAAE,eAAe,CAAC;IACzB,MAAM,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;IAC/B,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,gBAAgB;IAC/B,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,IAAI,CAAC;IACX,MAAM,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;IAC/B,OAAO,EAAE,eAAe,CAAC;IACzB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB"}
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/database/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAE/D,MAAM,WAAW,WAAW;IAC1B,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,OAAO,EAAE,eAAe,CAAC;IACzB,MAAM,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;IAC/B,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,gBAAgB;IAC/B,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,IAAI,CAAC;IACX,MAAM,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;IAC/B,OAAO,EAAE,eAAe,CAAC;IACzB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB"}

package/dist/src/evaluator.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"evaluator.d.ts","sourceRoot":"","sources":["../../src/evaluator.ts"],"names":[],"mappings":"AAeA,OAAO,KAAK,EAGV,eAAe,EAGf,eAAe,EAEf,MAAM,EAEN,SAAS,EAGV,MAAM,SAAS,CAAC;AAIjB,eAAO,MAAM,uBAAuB,IAAI,CAAC;AAEzC;;;;;;;;;;;;;;;;GAgBG;AACH,wBAAgB,eAAe,CAAC,MAAM,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,EAAE,GAAG,SAAS,GAAG,OAAO,CAM7F;AAED,wBAAgB,uBAAuB,CACrC,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,GAAG,CAAC,GAC5C,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,GAAG,EAAE,CAAC,EAAE,CAqClC;AA4xBD,wBAAgB,QAAQ,CAAC,SAAS,EAAE,SAAS,EAAE,OAAO,EAAE,eAAe,4BAGtE"}
+{"version":3,"file":"evaluator.d.ts","sourceRoot":"","sources":["../../src/evaluator.ts"],"names":[],"mappings":"AAeA,OAAO,KAAK,EAGV,eAAe,EAGf,eAAe,EAEf,MAAM,EAEN,SAAS,EAGV,MAAM,SAAS,CAAC;AAIjB,eAAO,MAAM,uBAAuB,IAAI,CAAC;AAEzC;;;;;;;;;;;;;;;;GAgBG;AACH,wBAAgB,eAAe,CAAC,MAAM,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,EAAE,GAAG,SAAS,GAAG,OAAO,CAM7F;AAED,wBAAgB,uBAAuB,CACrC,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,GAAG,CAAC,GAC5C,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,GAAG,EAAE,CAAC,EAAE,CAqClC;AAoyBD,wBAAgB,QAAQ,CAAC,SAAS,EAAE,SAAS,EAAE,OAAO,EAAE,eAAe,4BAGtE"}

package/dist/src/evaluator.js

@@ -220,9 +220,15 @@ class Evaluator {
             else if (response.output) {
                 // Create a copy of response so we can potentially mutate it.
                 const processedResponse = { ...response };
-                const shouldTransform = test.options?.transform || test.options?.postprocess || provider.transform;
-                if (shouldTransform) {
-                    processedResponse.output = await (0, transform_1.transform)(shouldTransform, processedResponse.output, {
+                const transforms = [
+                    provider.transform, // Apply provider transform first
+                    // NOTE: postprocess is deprecated. Use the first defined transform.
+                    [test.options?.transform, test.options?.postprocess].find((s) => s),
+                ]
+                    .flat()
+                    .filter((s) => typeof s === 'string');
+                for (const t of transforms) {
+                    processedResponse.output = await (0, transform_1.transform)(t, processedResponse.output, {
                         vars,
                         prompt,
                     });
@@ -474,7 +480,10 @@ class Evaluator {
         const table = {
             head: {
                 prompts,
-                vars: Array.from(varNames).sort(),
+                vars: [
+                    ...Object.keys(testSuite.defaultTest?.vars || {}).sort(),
+                    ...Array.from(varNames).sort(),
+                ],
             },
             body: [],
         };