npm - promptfoo - Versions diffs - 0.78.3 → 0.80.0 - Mend

promptfoo 0.78.3 → 0.80.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (232) hide show

package/dist/src/types/index.d.ts CHANGED Viewed

@@ -29,7 +29,6 @@ export declare const CommandLineOptionsSchema: z.ZodObject<{
     share: z.ZodOptional<z.ZodBoolean>;
     progressBar: z.ZodOptional<z.ZodBoolean>;
     watch: z.ZodOptional<z.ZodBoolean>;
-    interactiveProviders: z.ZodOptional<z.ZodBoolean>;
     filterFailing: z.ZodOptional<z.ZodString>;
     filterFirstN: z.ZodOptional<z.ZodString>;
     filterPattern: z.ZodOptional<z.ZodString>;
@@ -61,7 +60,6 @@ export declare const CommandLineOptionsSchema: z.ZodObject<{
     share?: boolean | undefined;
     progressBar?: boolean | undefined;
     watch?: boolean | undefined;
-    interactiveProviders?: boolean | undefined;
     filterFailing?: string | undefined;
     filterFirstN?: string | undefined;
     filterPattern?: string | undefined;
@@ -93,7 +91,6 @@ export declare const CommandLineOptionsSchema: z.ZodObject<{
     share?: boolean | undefined;
     progressBar?: boolean | undefined;
     watch?: boolean | undefined;
-    interactiveProviders?: boolean | undefined;
     filterFailing?: string | undefined;
     filterFirstN?: string | undefined;
     filterPattern?: string | undefined;
@@ -197,35 +194,40 @@ export interface RunEvalOptions {
     repeatIndex: number;
 }
 declare const EvaluateOptionsSchema: z.ZodObject<{
-    maxConcurrency: z.ZodOptional<z.ZodNumber>;
-    showProgressBar: z.ZodOptional<z.ZodBoolean>;
-    progressCallback: z.ZodOptional<z.ZodFunction<z.ZodTuple<[z.ZodNumber, z.ZodNumber, z.ZodNumber, z.ZodType<RunEvalOptions, z.ZodTypeDef, RunEvalOptions>], null>, z.ZodVoid>>;
-    generateSuggestions: z.ZodOptional<z.ZodBoolean>;
-    repeat: z.ZodOptional<z.ZodNumber>;
-    delay: z.ZodOptional<z.ZodNumber>;
     cache: z.ZodOptional<z.ZodBoolean>;
+    delay: z.ZodOptional<z.ZodNumber>;
     eventSource: z.ZodOptional<z.ZodString>;
+    generateSuggestions: z.ZodOptional<z.ZodBoolean>;
+    /**
+     * @deprecated This option has been removed as of 2024-08-21.
+     * @description Use `maxConcurrency: 1` or the CLI option `-j 1` instead to run evaluations serially.
+     * @author mldangelo
+     */
     interactiveProviders: z.ZodOptional<z.ZodBoolean>;
+    maxConcurrency: z.ZodOptional<z.ZodNumber>;
+    progressCallback: z.ZodOptional<z.ZodFunction<z.ZodTuple<[z.ZodNumber, z.ZodNumber, z.ZodNumber, z.ZodType<RunEvalOptions, z.ZodTypeDef, RunEvalOptions>], null>, z.ZodVoid>>;
+    repeat: z.ZodOptional<z.ZodNumber>;
+    showProgressBar: z.ZodOptional<z.ZodBoolean>;
 }, "strip", z.ZodTypeAny, {
     delay?: number | undefined;
     maxConcurrency?: number | undefined;
     repeat?: number | undefined;
     cache?: boolean | undefined;
-    interactiveProviders?: boolean | undefined;
     generateSuggestions?: boolean | undefined;
-    showProgressBar?: boolean | undefined;
-    progressCallback?: ((args_0: number, args_1: number, args_2: number, args_3: RunEvalOptions) => void) | undefined;
     eventSource?: string | undefined;
+    interactiveProviders?: boolean | undefined;
+    progressCallback?: ((args_0: number, args_1: number, args_2: number, args_3: RunEvalOptions) => void) | undefined;
+    showProgressBar?: boolean | undefined;
 }, {
     delay?: number | undefined;
     maxConcurrency?: number | undefined;
     repeat?: number | undefined;
     cache?: boolean | undefined;
-    interactiveProviders?: boolean | undefined;
     generateSuggestions?: boolean | undefined;
-    showProgressBar?: boolean | undefined;
-    progressCallback?: ((args_0: number, args_1: number, args_2: number, args_3: RunEvalOptions) => void) | undefined;
     eventSource?: string | undefined;
+    interactiveProviders?: boolean | undefined;
+    progressCallback?: ((args_0: number, args_1: number, args_2: number, args_3: RunEvalOptions) => void) | undefined;
+    showProgressBar?: boolean | undefined;
 }>;
 export type EvaluateOptions = z.infer<typeof EvaluateOptionsSchema>;
 export declare const CompletedPromptSchema: z.ZodObject<z.objectUtil.extendShape<{
@@ -10424,6 +10426,18 @@ export declare const TestSuiteConfigSchema: z.ZodObject<{
         label?: string | undefined;
     }>>]>, "many">]>;
     prompts: z.ZodUnion<[z.ZodString, z.ZodArray<z.ZodUnion<[z.ZodString, z.ZodObject<{
+        id: z.ZodString;
+        label: z.ZodOptional<z.ZodString>;
+        raw: z.ZodOptional<z.ZodString>;
+    }, "strip", z.ZodTypeAny, {
+        id: string;
+        raw?: string | undefined;
+        label?: string | undefined;
+    }, {
+        id: string;
+        raw?: string | undefined;
+        label?: string | undefined;
+    }>, z.ZodObject<{
         id: z.ZodOptional<z.ZodString>;
         raw: z.ZodString;
         display: z.ZodOptional<z.ZodString>;
@@ -10460,7 +10474,7 @@ export declare const TestSuiteConfigSchema: z.ZodObject<{
         display?: string | undefined;
         config?: any;
     }>]>, "many">, z.ZodRecord<z.ZodString, z.ZodString>]>;
-    tests: z.ZodUnion<[z.ZodString, z.ZodArray<z.ZodUnion<[z.ZodString, z.ZodObject<{
+    tests: z.ZodOptional<z.ZodUnion<[z.ZodString, z.ZodArray<z.ZodUnion<[z.ZodString, z.ZodObject<{
         description: z.ZodOptional<z.ZodString>;
         vars: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnion<[z.ZodString, z.ZodEffects<z.ZodNumber, string, number>, z.ZodEffects<z.ZodBoolean, string, boolean>, z.ZodArray<z.ZodUnion<[z.ZodString, z.ZodEffects<z.ZodNumber, string, number>, z.ZodEffects<z.ZodBoolean, string, boolean>]>, "many">, z.ZodObject<{}, "strip", z.ZodTypeAny, {}, {}>, z.ZodArray<z.ZodAny, "many">]>>>;
         provider: z.ZodOptional<z.ZodUnion<[z.ZodString, z.ZodObject<{
@@ -11107,7 +11121,7 @@ export declare const TestSuiteConfigSchema: z.ZodObject<{
             metric?: string | undefined;
         })[] | undefined;
         threshold?: number | undefined;
-    }>]>, "many">]>;
+    }>]>, "many">]>>;
     scenarios: z.ZodOptional<z.ZodArray<z.ZodObject<{
         description: z.ZodOptional<z.ZodString>;
         config: z.ZodArray<z.ZodObject<{
@@ -13634,6 +13648,10 @@ export declare const TestSuiteConfigSchema: z.ZodObject<{
         id?: string | undefined;
         display?: string | undefined;
         config?: any;
+    } | {
+        id: string;
+        raw?: string | undefined;
+        label?: string | undefined;
     })[];
     providers: string | (((args_0: string, args_1: {
         vars: Record<string, string | {}>;
@@ -13760,7 +13778,42 @@ export declare const TestSuiteConfigSchema: z.ZodObject<{
             CLOUDFLARE_ACCOUNT_ID?: string | undefined;
         } | undefined;
     }>)[];
-    tests: string | (string | {
+    env?: {
+        ANTHROPIC_API_KEY?: string | undefined;
+        BAM_API_KEY?: string | undefined;
+        BAM_API_HOST?: string | undefined;
+        AZURE_OPENAI_API_HOST?: string | undefined;
+        AZURE_OPENAI_API_KEY?: string | undefined;
+        AZURE_OPENAI_API_BASE_URL?: string | undefined;
+        AZURE_OPENAI_BASE_URL?: string | undefined;
+        AWS_BEDROCK_REGION?: string | undefined;
+        COHERE_API_KEY?: string | undefined;
+        OPENAI_API_KEY?: string | undefined;
+        OPENAI_API_HOST?: string | undefined;
+        OPENAI_API_BASE_URL?: string | undefined;
+        OPENAI_BASE_URL?: string | undefined;
+        OPENAI_ORGANIZATION?: string | undefined;
+        REPLICATE_API_KEY?: string | undefined;
+        REPLICATE_API_TOKEN?: string | undefined;
+        LOCALAI_BASE_URL?: string | undefined;
+        MISTRAL_API_HOST?: string | undefined;
+        MISTRAL_API_BASE_URL?: string | undefined;
+        PALM_API_KEY?: string | undefined;
+        PALM_API_HOST?: string | undefined;
+        GOOGLE_API_KEY?: string | undefined;
+        GOOGLE_API_HOST?: string | undefined;
+        VERTEX_API_KEY?: string | undefined;
+        VERTEX_API_HOST?: string | undefined;
+        VERTEX_PROJECT_ID?: string | undefined;
+        VERTEX_REGION?: string | undefined;
+        VERTEX_PUBLISHER?: string | undefined;
+        MISTRAL_API_KEY?: string | undefined;
+        CLOUDFLARE_API_KEY?: string | undefined;
+        CLOUDFLARE_ACCOUNT_ID?: string | undefined;
+    } | undefined;
+    metadata?: Record<string, any> | undefined;
+    description?: string | undefined;
+    tests?: string | (string | {
         options?: ({
             prefix?: string | undefined;
             suffix?: string | undefined;
@@ -13870,42 +13923,7 @@ export declare const TestSuiteConfigSchema: z.ZodObject<{
             metric?: string | undefined;
         })[] | undefined;
         threshold?: number | undefined;
-    })[];
-    env?: {
-        ANTHROPIC_API_KEY?: string | undefined;
-        BAM_API_KEY?: string | undefined;
-        BAM_API_HOST?: string | undefined;
-        AZURE_OPENAI_API_HOST?: string | undefined;
-        AZURE_OPENAI_API_KEY?: string | undefined;
-        AZURE_OPENAI_API_BASE_URL?: string | undefined;
-        AZURE_OPENAI_BASE_URL?: string | undefined;
-        AWS_BEDROCK_REGION?: string | undefined;
-        COHERE_API_KEY?: string | undefined;
-        OPENAI_API_KEY?: string | undefined;
-        OPENAI_API_HOST?: string | undefined;
-        OPENAI_API_BASE_URL?: string | undefined;
-        OPENAI_BASE_URL?: string | undefined;
-        OPENAI_ORGANIZATION?: string | undefined;
-        REPLICATE_API_KEY?: string | undefined;
-        REPLICATE_API_TOKEN?: string | undefined;
-        LOCALAI_BASE_URL?: string | undefined;
-        MISTRAL_API_HOST?: string | undefined;
-        MISTRAL_API_BASE_URL?: string | undefined;
-        PALM_API_KEY?: string | undefined;
-        PALM_API_HOST?: string | undefined;
-        GOOGLE_API_KEY?: string | undefined;
-        GOOGLE_API_HOST?: string | undefined;
-        VERTEX_API_KEY?: string | undefined;
-        VERTEX_API_HOST?: string | undefined;
-        VERTEX_PROJECT_ID?: string | undefined;
-        VERTEX_REGION?: string | undefined;
-        VERTEX_PUBLISHER?: string | undefined;
-        MISTRAL_API_KEY?: string | undefined;
-        CLOUDFLARE_API_KEY?: string | undefined;
-        CLOUDFLARE_ACCOUNT_ID?: string | undefined;
-    } | undefined;
-    metadata?: Record<string, any> | undefined;
-    description?: string | undefined;
+    })[] | undefined;
     scenarios?: {
         config: {
             options?: ({
@@ -14264,6 +14282,10 @@ export declare const TestSuiteConfigSchema: z.ZodObject<{
         id?: string | undefined;
         display?: string | undefined;
         config?: any;
+    } | {
+        id: string;
+        raw?: string | undefined;
+        label?: string | undefined;
     })[];
     providers: string | (((args_0: string, args_1: {
         vars: Record<string, string | {}>;
@@ -14390,7 +14412,42 @@ export declare const TestSuiteConfigSchema: z.ZodObject<{
             CLOUDFLARE_ACCOUNT_ID?: string | undefined;
         } | undefined;
     }>)[];
-    tests: string | (string | {
+    env?: {
+        ANTHROPIC_API_KEY?: string | undefined;
+        BAM_API_KEY?: string | undefined;
+        BAM_API_HOST?: string | undefined;
+        AZURE_OPENAI_API_HOST?: string | undefined;
+        AZURE_OPENAI_API_KEY?: string | undefined;
+        AZURE_OPENAI_API_BASE_URL?: string | undefined;
+        AZURE_OPENAI_BASE_URL?: string | undefined;
+        AWS_BEDROCK_REGION?: string | undefined;
+        COHERE_API_KEY?: string | undefined;
+        OPENAI_API_KEY?: string | undefined;
+        OPENAI_API_HOST?: string | undefined;
+        OPENAI_API_BASE_URL?: string | undefined;
+        OPENAI_BASE_URL?: string | undefined;
+        OPENAI_ORGANIZATION?: string | undefined;
+        REPLICATE_API_KEY?: string | undefined;
+        REPLICATE_API_TOKEN?: string | undefined;
+        LOCALAI_BASE_URL?: string | undefined;
+        MISTRAL_API_HOST?: string | undefined;
+        MISTRAL_API_BASE_URL?: string | undefined;
+        PALM_API_KEY?: string | undefined;
+        PALM_API_HOST?: string | undefined;
+        GOOGLE_API_KEY?: string | undefined;
+        GOOGLE_API_HOST?: string | undefined;
+        VERTEX_API_KEY?: string | undefined;
+        VERTEX_API_HOST?: string | undefined;
+        VERTEX_PROJECT_ID?: string | undefined;
+        VERTEX_REGION?: string | undefined;
+        VERTEX_PUBLISHER?: string | undefined;
+        MISTRAL_API_KEY?: string | undefined;
+        CLOUDFLARE_API_KEY?: string | undefined;
+        CLOUDFLARE_ACCOUNT_ID?: string | undefined;
+    } | undefined;
+    metadata?: Record<string, any> | undefined;
+    description?: string | undefined;
+    tests?: string | (string | {
         options?: ({
             prefix?: string | undefined;
             suffix?: string | undefined;
@@ -14500,42 +14557,7 @@ export declare const TestSuiteConfigSchema: z.ZodObject<{
             metric?: string | undefined;
         })[] | undefined;
         threshold?: number | undefined;
-    })[];
-    env?: {
-        ANTHROPIC_API_KEY?: string | undefined;
-        BAM_API_KEY?: string | undefined;
-        BAM_API_HOST?: string | undefined;
-        AZURE_OPENAI_API_HOST?: string | undefined;
-        AZURE_OPENAI_API_KEY?: string | undefined;
-        AZURE_OPENAI_API_BASE_URL?: string | undefined;
-        AZURE_OPENAI_BASE_URL?: string | undefined;
-        AWS_BEDROCK_REGION?: string | undefined;
-        COHERE_API_KEY?: string | undefined;
-        OPENAI_API_KEY?: string | undefined;
-        OPENAI_API_HOST?: string | undefined;
-        OPENAI_API_BASE_URL?: string | undefined;
-        OPENAI_BASE_URL?: string | undefined;
-        OPENAI_ORGANIZATION?: string | undefined;
-        REPLICATE_API_KEY?: string | undefined;
-        REPLICATE_API_TOKEN?: string | undefined;
-        LOCALAI_BASE_URL?: string | undefined;
-        MISTRAL_API_HOST?: string | undefined;
-        MISTRAL_API_BASE_URL?: string | undefined;
-        PALM_API_KEY?: string | undefined;
-        PALM_API_HOST?: string | undefined;
-        GOOGLE_API_KEY?: string | undefined;
-        GOOGLE_API_HOST?: string | undefined;
-        VERTEX_API_KEY?: string | undefined;
-        VERTEX_API_HOST?: string | undefined;
-        VERTEX_PROJECT_ID?: string | undefined;
-        VERTEX_REGION?: string | undefined;
-        VERTEX_PUBLISHER?: string | undefined;
-        MISTRAL_API_KEY?: string | undefined;
-        CLOUDFLARE_API_KEY?: string | undefined;
-        CLOUDFLARE_ACCOUNT_ID?: string | undefined;
-    } | undefined;
-    metadata?: Record<string, any> | undefined;
-    description?: string | undefined;
+    })[] | undefined;
     scenarios?: {
         config: {
             options?: ({
@@ -15441,6 +15463,18 @@ export declare const UnifiedConfigSchema: z.ZodObject<z.objectUtil.extendShape<{
         label?: string | undefined;
     }>>]>, "many">]>;
     prompts: z.ZodUnion<[z.ZodString, z.ZodArray<z.ZodUnion<[z.ZodString, z.ZodObject<{
+        id: z.ZodString;
+        label: z.ZodOptional<z.ZodString>;
+        raw: z.ZodOptional<z.ZodString>;
+    }, "strip", z.ZodTypeAny, {
+        id: string;
+        raw?: string | undefined;
+        label?: string | undefined;
+    }, {
+        id: string;
+        raw?: string | undefined;
+        label?: string | undefined;
+    }>, z.ZodObject<{
         id: z.ZodOptional<z.ZodString>;
         raw: z.ZodString;
         display: z.ZodOptional<z.ZodString>;
@@ -15477,7 +15511,7 @@ export declare const UnifiedConfigSchema: z.ZodObject<z.objectUtil.extendShape<{
         display?: string | undefined;
         config?: any;
     }>]>, "many">, z.ZodRecord<z.ZodString, z.ZodString>]>;
-    tests: z.ZodUnion<[z.ZodString, z.ZodArray<z.ZodUnion<[z.ZodString, z.ZodObject<{
+    tests: z.ZodOptional<z.ZodUnion<[z.ZodString, z.ZodArray<z.ZodUnion<[z.ZodString, z.ZodObject<{
         description: z.ZodOptional<z.ZodString>;
         vars: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnion<[z.ZodString, z.ZodEffects<z.ZodNumber, string, number>, z.ZodEffects<z.ZodBoolean, string, boolean>, z.ZodArray<z.ZodUnion<[z.ZodString, z.ZodEffects<z.ZodNumber, string, number>, z.ZodEffects<z.ZodBoolean, string, boolean>]>, "many">, z.ZodObject<{}, "strip", z.ZodTypeAny, {}, {}>, z.ZodArray<z.ZodAny, "many">]>>>;
         provider: z.ZodOptional<z.ZodUnion<[z.ZodString, z.ZodObject<{
@@ -16124,7 +16158,7 @@ export declare const UnifiedConfigSchema: z.ZodObject<z.objectUtil.extendShape<{
             metric?: string | undefined;
         })[] | undefined;
         threshold?: number | undefined;
-    }>]>, "many">]>;
+    }>]>, "many">]>>;
     scenarios: z.ZodOptional<z.ZodArray<z.ZodObject<{
         description: z.ZodOptional<z.ZodString>;
         config: z.ZodArray<z.ZodObject<{
@@ -18642,35 +18676,40 @@ export declare const UnifiedConfigSchema: z.ZodObject<z.objectUtil.extendShape<{
     redteam: z.ZodOptional<z.ZodType<RedteamConfig, z.ZodTypeDef, RedteamConfig>>;
 }, {
     evaluateOptions: z.ZodOptional<z.ZodObject<{
-        maxConcurrency: z.ZodOptional<z.ZodNumber>;
-        showProgressBar: z.ZodOptional<z.ZodBoolean>;
-        progressCallback: z.ZodOptional<z.ZodFunction<z.ZodTuple<[z.ZodNumber, z.ZodNumber, z.ZodNumber, z.ZodType<RunEvalOptions, z.ZodTypeDef, RunEvalOptions>], null>, z.ZodVoid>>;
-        generateSuggestions: z.ZodOptional<z.ZodBoolean>;
-        repeat: z.ZodOptional<z.ZodNumber>;
-        delay: z.ZodOptional<z.ZodNumber>;
         cache: z.ZodOptional<z.ZodBoolean>;
+        delay: z.ZodOptional<z.ZodNumber>;
         eventSource: z.ZodOptional<z.ZodString>;
+        generateSuggestions: z.ZodOptional<z.ZodBoolean>;
+        /**
+         * @deprecated This option has been removed as of 2024-08-21.
+         * @description Use `maxConcurrency: 1` or the CLI option `-j 1` instead to run evaluations serially.
+         * @author mldangelo
+         */
         interactiveProviders: z.ZodOptional<z.ZodBoolean>;
+        maxConcurrency: z.ZodOptional<z.ZodNumber>;
+        progressCallback: z.ZodOptional<z.ZodFunction<z.ZodTuple<[z.ZodNumber, z.ZodNumber, z.ZodNumber, z.ZodType<RunEvalOptions, z.ZodTypeDef, RunEvalOptions>], null>, z.ZodVoid>>;
+        repeat: z.ZodOptional<z.ZodNumber>;
+        showProgressBar: z.ZodOptional<z.ZodBoolean>;
     }, "strip", z.ZodTypeAny, {
         delay?: number | undefined;
         maxConcurrency?: number | undefined;
         repeat?: number | undefined;
         cache?: boolean | undefined;
-        interactiveProviders?: boolean | undefined;
         generateSuggestions?: boolean | undefined;
-        showProgressBar?: boolean | undefined;
-        progressCallback?: ((args_0: number, args_1: number, args_2: number, args_3: RunEvalOptions) => void) | undefined;
         eventSource?: string | undefined;
+        interactiveProviders?: boolean | undefined;
+        progressCallback?: ((args_0: number, args_1: number, args_2: number, args_3: RunEvalOptions) => void) | undefined;
+        showProgressBar?: boolean | undefined;
     }, {
         delay?: number | undefined;
         maxConcurrency?: number | undefined;
         repeat?: number | undefined;
         cache?: boolean | undefined;
-        interactiveProviders?: boolean | undefined;
         generateSuggestions?: boolean | undefined;
-        showProgressBar?: boolean | undefined;
-        progressCallback?: ((args_0: number, args_1: number, args_2: number, args_3: RunEvalOptions) => void) | undefined;
         eventSource?: string | undefined;
+        interactiveProviders?: boolean | undefined;
+        progressCallback?: ((args_0: number, args_1: number, args_2: number, args_3: RunEvalOptions) => void) | undefined;
+        showProgressBar?: boolean | undefined;
     }>>;
     commandLineOptions: z.ZodOptional<z.ZodObject<{
         description: z.ZodOptional<z.ZodOptional<z.ZodString>>;
@@ -18694,7 +18733,6 @@ export declare const UnifiedConfigSchema: z.ZodObject<z.objectUtil.extendShape<{
         share: z.ZodOptional<z.ZodOptional<z.ZodBoolean>>;
         progressBar: z.ZodOptional<z.ZodOptional<z.ZodBoolean>>;
         watch: z.ZodOptional<z.ZodOptional<z.ZodBoolean>>;
-        interactiveProviders: z.ZodOptional<z.ZodOptional<z.ZodBoolean>>;
         filterFailing: z.ZodOptional<z.ZodOptional<z.ZodString>>;
         filterFirstN: z.ZodOptional<z.ZodOptional<z.ZodString>>;
         filterPattern: z.ZodOptional<z.ZodOptional<z.ZodString>>;
@@ -18726,7 +18764,6 @@ export declare const UnifiedConfigSchema: z.ZodObject<z.objectUtil.extendShape<{
         share?: boolean | undefined;
         progressBar?: boolean | undefined;
         watch?: boolean | undefined;
-        interactiveProviders?: boolean | undefined;
         filterFailing?: string | undefined;
         filterFirstN?: string | undefined;
         filterPattern?: string | undefined;
@@ -18758,7 +18795,6 @@ export declare const UnifiedConfigSchema: z.ZodObject<z.objectUtil.extendShape<{
         share?: boolean | undefined;
         progressBar?: boolean | undefined;
         watch?: boolean | undefined;
-        interactiveProviders?: boolean | undefined;
         filterFailing?: string | undefined;
         filterFirstN?: string | undefined;
         filterPattern?: string | undefined;
@@ -18780,6 +18816,10 @@ export declare const UnifiedConfigSchema: z.ZodObject<z.objectUtil.extendShape<{
         id?: string | undefined;
         display?: string | undefined;
         config?: any;
+    } | {
+        id: string;
+        raw?: string | undefined;
+        label?: string | undefined;
     })[];
     providers: string | (((args_0: string, args_1: {
         vars: Record<string, string | {}>;
@@ -18906,7 +18946,42 @@ export declare const UnifiedConfigSchema: z.ZodObject<z.objectUtil.extendShape<{
             CLOUDFLARE_ACCOUNT_ID?: string | undefined;
         } | undefined;
     }>)[];
-    tests: string | (string | {
+    env?: {
+        ANTHROPIC_API_KEY?: string | undefined;
+        BAM_API_KEY?: string | undefined;
+        BAM_API_HOST?: string | undefined;
+        AZURE_OPENAI_API_HOST?: string | undefined;
+        AZURE_OPENAI_API_KEY?: string | undefined;
+        AZURE_OPENAI_API_BASE_URL?: string | undefined;
+        AZURE_OPENAI_BASE_URL?: string | undefined;
+        AWS_BEDROCK_REGION?: string | undefined;
+        COHERE_API_KEY?: string | undefined;
+        OPENAI_API_KEY?: string | undefined;
+        OPENAI_API_HOST?: string | undefined;
+        OPENAI_API_BASE_URL?: string | undefined;
+        OPENAI_BASE_URL?: string | undefined;
+        OPENAI_ORGANIZATION?: string | undefined;
+        REPLICATE_API_KEY?: string | undefined;
+        REPLICATE_API_TOKEN?: string | undefined;
+        LOCALAI_BASE_URL?: string | undefined;
+        MISTRAL_API_HOST?: string | undefined;
+        MISTRAL_API_BASE_URL?: string | undefined;
+        PALM_API_KEY?: string | undefined;
+        PALM_API_HOST?: string | undefined;
+        GOOGLE_API_KEY?: string | undefined;
+        GOOGLE_API_HOST?: string | undefined;
+        VERTEX_API_KEY?: string | undefined;
+        VERTEX_API_HOST?: string | undefined;
+        VERTEX_PROJECT_ID?: string | undefined;
+        VERTEX_REGION?: string | undefined;
+        VERTEX_PUBLISHER?: string | undefined;
+        MISTRAL_API_KEY?: string | undefined;
+        CLOUDFLARE_API_KEY?: string | undefined;
+        CLOUDFLARE_ACCOUNT_ID?: string | undefined;
+    } | undefined;
+    metadata?: Record<string, any> | undefined;
+    description?: string | undefined;
+    tests?: string | (string | {
         options?: ({
             prefix?: string | undefined;
             suffix?: string | undefined;
@@ -19016,42 +19091,7 @@ export declare const UnifiedConfigSchema: z.ZodObject<z.objectUtil.extendShape<{
             metric?: string | undefined;
         })[] | undefined;
         threshold?: number | undefined;
-    })[];
-    env?: {
-        ANTHROPIC_API_KEY?: string | undefined;
-        BAM_API_KEY?: string | undefined;
-        BAM_API_HOST?: string | undefined;
-        AZURE_OPENAI_API_HOST?: string | undefined;
-        AZURE_OPENAI_API_KEY?: string | undefined;
-        AZURE_OPENAI_API_BASE_URL?: string | undefined;
-        AZURE_OPENAI_BASE_URL?: string | undefined;
-        AWS_BEDROCK_REGION?: string | undefined;
-        COHERE_API_KEY?: string | undefined;
-        OPENAI_API_KEY?: string | undefined;
-        OPENAI_API_HOST?: string | undefined;
-        OPENAI_API_BASE_URL?: string | undefined;
-        OPENAI_BASE_URL?: string | undefined;
-        OPENAI_ORGANIZATION?: string | undefined;
-        REPLICATE_API_KEY?: string | undefined;
-        REPLICATE_API_TOKEN?: string | undefined;
-        LOCALAI_BASE_URL?: string | undefined;
-        MISTRAL_API_HOST?: string | undefined;
-        MISTRAL_API_BASE_URL?: string | undefined;
-        PALM_API_KEY?: string | undefined;
-        PALM_API_HOST?: string | undefined;
-        GOOGLE_API_KEY?: string | undefined;
-        GOOGLE_API_HOST?: string | undefined;
-        VERTEX_API_KEY?: string | undefined;
-        VERTEX_API_HOST?: string | undefined;
-        VERTEX_PROJECT_ID?: string | undefined;
-        VERTEX_REGION?: string | undefined;
-        VERTEX_PUBLISHER?: string | undefined;
-        MISTRAL_API_KEY?: string | undefined;
-        CLOUDFLARE_API_KEY?: string | undefined;
-        CLOUDFLARE_ACCOUNT_ID?: string | undefined;
-    } | undefined;
-    metadata?: Record<string, any> | undefined;
-    description?: string | undefined;
+    })[] | undefined;
     scenarios?: {
         config: {
             options?: ({
@@ -19404,11 +19444,11 @@ export declare const UnifiedConfigSchema: z.ZodObject<z.objectUtil.extendShape<{
         maxConcurrency?: number | undefined;
         repeat?: number | undefined;
         cache?: boolean | undefined;
-        interactiveProviders?: boolean | undefined;
         generateSuggestions?: boolean | undefined;
-        showProgressBar?: boolean | undefined;
-        progressCallback?: ((args_0: number, args_1: number, args_2: number, args_3: RunEvalOptions) => void) | undefined;
         eventSource?: string | undefined;
+        interactiveProviders?: boolean | undefined;
+        progressCallback?: ((args_0: number, args_1: number, args_2: number, args_3: RunEvalOptions) => void) | undefined;
+        showProgressBar?: boolean | undefined;
     } | undefined;
     commandLineOptions?: {
         vars?: string | undefined;
@@ -19432,7 +19472,6 @@ export declare const UnifiedConfigSchema: z.ZodObject<z.objectUtil.extendShape<{
         share?: boolean | undefined;
         progressBar?: boolean | undefined;
         watch?: boolean | undefined;
-        interactiveProviders?: boolean | undefined;
         filterFailing?: string | undefined;
         filterFirstN?: string | undefined;
         filterPattern?: string | undefined;
@@ -19454,6 +19493,10 @@ export declare const UnifiedConfigSchema: z.ZodObject<z.objectUtil.extendShape<{
         id?: string | undefined;
         display?: string | undefined;
         config?: any;
+    } | {
+        id: string;
+        raw?: string | undefined;
+        label?: string | undefined;
     })[];
     providers: string | (((args_0: string, args_1: {
         vars: Record<string, string | {}>;
@@ -19580,7 +19623,42 @@ export declare const UnifiedConfigSchema: z.ZodObject<z.objectUtil.extendShape<{
             CLOUDFLARE_ACCOUNT_ID?: string | undefined;
         } | undefined;
     }>)[];
-    tests: string | (string | {
+    env?: {
+        ANTHROPIC_API_KEY?: string | undefined;
+        BAM_API_KEY?: string | undefined;
+        BAM_API_HOST?: string | undefined;
+        AZURE_OPENAI_API_HOST?: string | undefined;
+        AZURE_OPENAI_API_KEY?: string | undefined;
+        AZURE_OPENAI_API_BASE_URL?: string | undefined;
+        AZURE_OPENAI_BASE_URL?: string | undefined;
+        AWS_BEDROCK_REGION?: string | undefined;
+        COHERE_API_KEY?: string | undefined;
+        OPENAI_API_KEY?: string | undefined;
+        OPENAI_API_HOST?: string | undefined;
+        OPENAI_API_BASE_URL?: string | undefined;
+        OPENAI_BASE_URL?: string | undefined;
+        OPENAI_ORGANIZATION?: string | undefined;
+        REPLICATE_API_KEY?: string | undefined;
+        REPLICATE_API_TOKEN?: string | undefined;
+        LOCALAI_BASE_URL?: string | undefined;
+        MISTRAL_API_HOST?: string | undefined;
+        MISTRAL_API_BASE_URL?: string | undefined;
+        PALM_API_KEY?: string | undefined;
+        PALM_API_HOST?: string | undefined;
+        GOOGLE_API_KEY?: string | undefined;
+        GOOGLE_API_HOST?: string | undefined;
+        VERTEX_API_KEY?: string | undefined;
+        VERTEX_API_HOST?: string | undefined;
+        VERTEX_PROJECT_ID?: string | undefined;
+        VERTEX_REGION?: string | undefined;
+        VERTEX_PUBLISHER?: string | undefined;
+        MISTRAL_API_KEY?: string | undefined;
+        CLOUDFLARE_API_KEY?: string | undefined;
+        CLOUDFLARE_ACCOUNT_ID?: string | undefined;
+    } | undefined;
+    metadata?: Record<string, any> | undefined;
+    description?: string | undefined;
+    tests?: string | (string | {
         options?: ({
             prefix?: string | undefined;
             suffix?: string | undefined;
@@ -19690,42 +19768,7 @@ export declare const UnifiedConfigSchema: z.ZodObject<z.objectUtil.extendShape<{
             metric?: string | undefined;
         })[] | undefined;
         threshold?: number | undefined;
-    })[];
-    env?: {
-        ANTHROPIC_API_KEY?: string | undefined;
-        BAM_API_KEY?: string | undefined;
-        BAM_API_HOST?: string | undefined;
-        AZURE_OPENAI_API_HOST?: string | undefined;
-        AZURE_OPENAI_API_KEY?: string | undefined;
-        AZURE_OPENAI_API_BASE_URL?: string | undefined;
-        AZURE_OPENAI_BASE_URL?: string | undefined;
-        AWS_BEDROCK_REGION?: string | undefined;
-        COHERE_API_KEY?: string | undefined;
-        OPENAI_API_KEY?: string | undefined;
-        OPENAI_API_HOST?: string | undefined;
-        OPENAI_API_BASE_URL?: string | undefined;
-        OPENAI_BASE_URL?: string | undefined;
-        OPENAI_ORGANIZATION?: string | undefined;
-        REPLICATE_API_KEY?: string | undefined;
-        REPLICATE_API_TOKEN?: string | undefined;
-        LOCALAI_BASE_URL?: string | undefined;
-        MISTRAL_API_HOST?: string | undefined;
-        MISTRAL_API_BASE_URL?: string | undefined;
-        PALM_API_KEY?: string | undefined;
-        PALM_API_HOST?: string | undefined;
-        GOOGLE_API_KEY?: string | undefined;
-        GOOGLE_API_HOST?: string | undefined;
-        VERTEX_API_KEY?: string | undefined;
-        VERTEX_API_HOST?: string | undefined;
-        VERTEX_PROJECT_ID?: string | undefined;
-        VERTEX_REGION?: string | undefined;
-        VERTEX_PUBLISHER?: string | undefined;
-        MISTRAL_API_KEY?: string | undefined;
-        CLOUDFLARE_API_KEY?: string | undefined;
-        CLOUDFLARE_ACCOUNT_ID?: string | undefined;
-    } | undefined;
-    metadata?: Record<string, any> | undefined;
-    description?: string | undefined;
+    })[] | undefined;
     scenarios?: {
         config: {
             options?: ({
@@ -20078,11 +20121,11 @@ export declare const UnifiedConfigSchema: z.ZodObject<z.objectUtil.extendShape<{
         maxConcurrency?: number | undefined;
         repeat?: number | undefined;
         cache?: boolean | undefined;
-        interactiveProviders?: boolean | undefined;
         generateSuggestions?: boolean | undefined;
-        showProgressBar?: boolean | undefined;
-        progressCallback?: ((args_0: number, args_1: number, args_2: number, args_3: RunEvalOptions) => void) | undefined;
         eventSource?: string | undefined;
+        interactiveProviders?: boolean | undefined;
+        progressCallback?: ((args_0: number, args_1: number, args_2: number, args_3: RunEvalOptions) => void) | undefined;
+        showProgressBar?: boolean | undefined;
     } | undefined;
     commandLineOptions?: {
         vars?: string | undefined;
@@ -20106,7 +20149,6 @@ export declare const UnifiedConfigSchema: z.ZodObject<z.objectUtil.extendShape<{
         share?: boolean | undefined;
         progressBar?: boolean | undefined;
         watch?: boolean | undefined;
-        interactiveProviders?: boolean | undefined;
         filterFailing?: string | undefined;
         filterFirstN?: string | undefined;
         filterPattern?: string | undefined;