promptfoo 0.91.3 → 0.92.1

package/dist/src/types/index.d.ts

@@ -192,8 +192,8 @@ export interface RunEvalOptions {
     test: AtomicTestCase;
     nunjucksFilters?: NunjucksFilterMap;
     evaluateOptions: EvaluateOptions;
-    rowIndex: number;
-    colIndex: number;
+    testIdx: number;
+    promptIdx: number;
     repeatIndex: number;
 }
 declare const EvaluateOptionsSchema: z.ZodObject<{
@@ -421,20 +421,27 @@ export interface PromptWithMetadata {
     count: number;
 }
 export interface EvaluateResult {
+    id?: string;
+    description?: string;
+    promptIdx: number;
+    testIdx: number;
+    testCase: AtomicTestCase;
+    promptId: string;
     provider: Pick<ProviderOptions, 'id' | 'label'>;
     prompt: Prompt;
     vars: Record<string, string | object>;
     response?: ProviderResponse;
-    error?: string;
+    error?: string | null;
     success: boolean;
     score: number;
     latencyMs: number;
-    gradingResult?: GradingResult;
+    gradingResult?: GradingResult | null;
     namedScores: Record<string, number>;
     cost?: number;
     metadata?: Record<string, any>;
 }
 export interface EvaluateTableOutput {
+    id: string;
     pass: boolean;
     score: number;
     namedScores: Record<string, number>;
@@ -443,7 +450,7 @@ export interface EvaluateTableOutput {
     latencyMs: number;
     provider?: string;
     tokenUsage?: Partial<TokenUsage>;
-    gradingResult?: GradingResult;
+    gradingResult?: GradingResult | null;
     cost: number;
     metadata?: Record<string, any>;
 }
@@ -465,7 +472,14 @@ export interface EvaluateStats {
     failures: number;
     tokenUsage: Required<TokenUsage>;
 }
-export interface EvaluateSummary {
+export interface EvaluateSummaryV3 {
+    version: 3;
+    timestamp: string;
+    results: EvaluateResult[];
+    prompts: CompletedPrompt[];
+    stats: EvaluateStats;
+}
+export interface EvaluateSummaryV2 {
     version: number;
     timestamp: string;
     results: EvaluateResult[];
@@ -1105,9 +1119,11 @@ export declare const TestCaseSchema: z.ZodObject<{
         label: z.ZodOptional<z.ZodType<string, z.ZodTypeDef, string>>;
         transform: z.ZodOptional<z.ZodString>;
         delay: z.ZodOptional<z.ZodNumber>;
+        config: z.ZodOptional<z.ZodAny>;
     }, "strip", z.ZodTypeAny, {
         callApi: import("./providers").CallApiFunction;
         id: (...args: unknown[]) => string;
+        config?: any;
         label?: string | undefined;
         transform?: string | undefined;
         delay?: number | undefined;
@@ -1116,6 +1132,7 @@ export declare const TestCaseSchema: z.ZodObject<{
     }, {
         callApi: import("./providers").CallApiFunction;
         id: (...args: unknown[]) => string;
+        config?: any;
         label?: string | undefined;
         transform?: string | undefined;
         delay?: number | undefined;
@@ -1425,6 +1442,7 @@ export declare const TestCaseSchema: z.ZodObject<{
     } | {
         callApi: import("./providers").CallApiFunction;
         id: (...args: unknown[]) => string;
+        config?: any;
         label?: string | undefined;
         transform?: string | undefined;
         delay?: number | undefined;
@@ -1542,6 +1560,7 @@ export declare const TestCaseSchema: z.ZodObject<{
     } | {
         callApi: import("./providers").CallApiFunction;
         id: (...args: unknown[]) => string;
+        config?: any;
         label?: string | undefined;
         transform?: string | undefined;
         delay?: number | undefined;
@@ -1802,9 +1821,11 @@ export declare const TestCaseWithVarsFileSchema: z.ZodObject<z.objectUtil.extend
         label: z.ZodOptional<z.ZodType<string, z.ZodTypeDef, string>>;
         transform: z.ZodOptional<z.ZodString>;
         delay: z.ZodOptional<z.ZodNumber>;
+        config: z.ZodOptional<z.ZodAny>;
     }, "strip", z.ZodTypeAny, {
         callApi: import("./providers").CallApiFunction;
         id: (...args: unknown[]) => string;
+        config?: any;
         label?: string | undefined;
         transform?: string | undefined;
         delay?: number | undefined;
@@ -1813,6 +1834,7 @@ export declare const TestCaseWithVarsFileSchema: z.ZodObject<z.objectUtil.extend
     }, {
         callApi: import("./providers").CallApiFunction;
         id: (...args: unknown[]) => string;
+        config?: any;
         label?: string | undefined;
         transform?: string | undefined;
         delay?: number | undefined;
@@ -2124,6 +2146,7 @@ export declare const TestCaseWithVarsFileSchema: z.ZodObject<z.objectUtil.extend
     } | {
         callApi: import("./providers").CallApiFunction;
         id: (...args: unknown[]) => string;
+        config?: any;
         label?: string | undefined;
         transform?: string | undefined;
         delay?: number | undefined;
@@ -2241,6 +2264,7 @@ export declare const TestCaseWithVarsFileSchema: z.ZodObject<z.objectUtil.extend
     } | {
         callApi: import("./providers").CallApiFunction;
         id: (...args: unknown[]) => string;
+        config?: any;
         label?: string | undefined;
         transform?: string | undefined;
         delay?: number | undefined;
@@ -2498,9 +2522,11 @@ export declare const TestCasesWithMetadataSchema: z.ZodObject<{
             label: z.ZodOptional<z.ZodType<string, z.ZodTypeDef, string>>;
             transform: z.ZodOptional<z.ZodString>;
             delay: z.ZodOptional<z.ZodNumber>;
+            config: z.ZodOptional<z.ZodAny>;
         }, "strip", z.ZodTypeAny, {
             callApi: import("./providers").CallApiFunction;
             id: (...args: unknown[]) => string;
+            config?: any;
             label?: string | undefined;
             transform?: string | undefined;
             delay?: number | undefined;
@@ -2509,6 +2535,7 @@ export declare const TestCasesWithMetadataSchema: z.ZodObject<{
         }, {
             callApi: import("./providers").CallApiFunction;
             id: (...args: unknown[]) => string;
+            config?: any;
             label?: string | undefined;
             transform?: string | undefined;
             delay?: number | undefined;
@@ -2818,6 +2845,7 @@ export declare const TestCasesWithMetadataSchema: z.ZodObject<{
         } | {
             callApi: import("./providers").CallApiFunction;
             id: (...args: unknown[]) => string;
+            config?: any;
             label?: string | undefined;
             transform?: string | undefined;
             delay?: number | undefined;
@@ -2935,6 +2963,7 @@ export declare const TestCasesWithMetadataSchema: z.ZodObject<{
         } | {
             callApi: import("./providers").CallApiFunction;
             id: (...args: unknown[]) => string;
+            config?: any;
             label?: string | undefined;
             transform?: string | undefined;
             delay?: number | undefined;
@@ -3351,6 +3380,7 @@ export declare const TestCasesWithMetadataSchema: z.ZodObject<{
         } | {
             callApi: import("./providers").CallApiFunction;
             id: (...args: unknown[]) => string;
+            config?: any;
             label?: string | undefined;
             transform?: string | undefined;
             delay?: number | undefined;
@@ -3513,6 +3543,7 @@ export declare const TestCasesWithMetadataSchema: z.ZodObject<{
         } | {
             callApi: import("./providers").CallApiFunction;
             id: (...args: unknown[]) => string;
+            config?: any;
             label?: string | undefined;
             transform?: string | undefined;
             delay?: number | undefined;
@@ -3774,9 +3805,11 @@ export declare const ScenarioSchema: z.ZodObject<{
             label: z.ZodOptional<z.ZodType<string, z.ZodTypeDef, string>>;
             transform: z.ZodOptional<z.ZodString>;
             delay: z.ZodOptional<z.ZodNumber>;
+            config: z.ZodOptional<z.ZodAny>;
         }, "strip", z.ZodTypeAny, {
             callApi: import("./providers").CallApiFunction;
             id: (...args: unknown[]) => string;
+            config?: any;
             label?: string | undefined;
             transform?: string | undefined;
             delay?: number | undefined;
@@ -3785,6 +3818,7 @@ export declare const ScenarioSchema: z.ZodObject<{
         }, {
             callApi: import("./providers").CallApiFunction;
             id: (...args: unknown[]) => string;
+            config?: any;
             label?: string | undefined;
             transform?: string | undefined;
             delay?: number | undefined;
@@ -4094,6 +4128,7 @@ export declare const ScenarioSchema: z.ZodObject<{
         } | {
             callApi: import("./providers").CallApiFunction;
             id: (...args: unknown[]) => string;
+            config?: any;
             label?: string | undefined;
             transform?: string | undefined;
             delay?: number | undefined;
@@ -4211,6 +4246,7 @@ export declare const ScenarioSchema: z.ZodObject<{
         } | {
             callApi: import("./providers").CallApiFunction;
             id: (...args: unknown[]) => string;
+            config?: any;
             label?: string | undefined;
             transform?: string | undefined;
             delay?: number | undefined;
@@ -4465,9 +4501,11 @@ export declare const ScenarioSchema: z.ZodObject<{
             label: z.ZodOptional<z.ZodType<string, z.ZodTypeDef, string>>;
             transform: z.ZodOptional<z.ZodString>;
             delay: z.ZodOptional<z.ZodNumber>;
+            config: z.ZodOptional<z.ZodAny>;
         }, "strip", z.ZodTypeAny, {
             callApi: import("./providers").CallApiFunction;
             id: (...args: unknown[]) => string;
+            config?: any;
             label?: string | undefined;
             transform?: string | undefined;
             delay?: number | undefined;
@@ -4476,6 +4514,7 @@ export declare const ScenarioSchema: z.ZodObject<{
         }, {
             callApi: import("./providers").CallApiFunction;
             id: (...args: unknown[]) => string;
+            config?: any;
             label?: string | undefined;
             transform?: string | undefined;
             delay?: number | undefined;
@@ -4785,6 +4824,7 @@ export declare const ScenarioSchema: z.ZodObject<{
         } | {
             callApi: import("./providers").CallApiFunction;
             id: (...args: unknown[]) => string;
+            config?: any;
             label?: string | undefined;
             transform?: string | undefined;
             delay?: number | undefined;
@@ -4902,6 +4942,7 @@ export declare const ScenarioSchema: z.ZodObject<{
         } | {
             callApi: import("./providers").CallApiFunction;
             id: (...args: unknown[]) => string;
+            config?: any;
             label?: string | undefined;
             transform?: string | undefined;
             delay?: number | undefined;
@@ -5021,6 +5062,7 @@ export declare const ScenarioSchema: z.ZodObject<{
         } | {
             callApi: import("./providers").CallApiFunction;
             id: (...args: unknown[]) => string;
+            config?: any;
             label?: string | undefined;
             transform?: string | undefined;
             delay?: number | undefined;
@@ -5139,6 +5181,7 @@ export declare const ScenarioSchema: z.ZodObject<{
         } | {
             callApi: import("./providers").CallApiFunction;
             id: (...args: unknown[]) => string;
+            config?: any;
             label?: string | undefined;
             transform?: string | undefined;
             delay?: number | undefined;
@@ -5259,6 +5302,7 @@ export declare const ScenarioSchema: z.ZodObject<{
         } | {
             callApi: import("./providers").CallApiFunction;
             id: (...args: unknown[]) => string;
+            config?: any;
             label?: string | undefined;
             transform?: string | undefined;
             delay?: number | undefined;
@@ -5377,6 +5421,7 @@ export declare const ScenarioSchema: z.ZodObject<{
         } | {
             callApi: import("./providers").CallApiFunction;
             id: (...args: unknown[]) => string;
+            config?: any;
             label?: string | undefined;
             transform?: string | undefined;
             delay?: number | undefined;
@@ -5634,9 +5679,11 @@ export declare const AtomicTestCaseSchema: z.ZodObject<z.objectUtil.extendShape<
         label: z.ZodOptional<z.ZodType<string, z.ZodTypeDef, string>>;
         transform: z.ZodOptional<z.ZodString>;
         delay: z.ZodOptional<z.ZodNumber>;
+        config: z.ZodOptional<z.ZodAny>;
     }, "strip", z.ZodTypeAny, {
         callApi: import("./providers").CallApiFunction;
         id: (...args: unknown[]) => string;
+        config?: any;
         label?: string | undefined;
         transform?: string | undefined;
         delay?: number | undefined;
@@ -5645,6 +5692,7 @@ export declare const AtomicTestCaseSchema: z.ZodObject<z.objectUtil.extendShape<
     }, {
         callApi: import("./providers").CallApiFunction;
         id: (...args: unknown[]) => string;
+        config?: any;
         label?: string | undefined;
         transform?: string | undefined;
         delay?: number | undefined;
@@ -5956,6 +6004,7 @@ export declare const AtomicTestCaseSchema: z.ZodObject<z.objectUtil.extendShape<
     } | {
         callApi: import("./providers").CallApiFunction;
         id: (...args: unknown[]) => string;
+        config?: any;
         label?: string | undefined;
         transform?: string | undefined;
         delay?: number | undefined;
@@ -6073,6 +6122,7 @@ export declare const AtomicTestCaseSchema: z.ZodObject<z.objectUtil.extendShape<
     } | {
         callApi: import("./providers").CallApiFunction;
         id: (...args: unknown[]) => string;
+        config?: any;
         label?: string | undefined;
         transform?: string | undefined;
         delay?: number | undefined;
@@ -6140,9 +6190,11 @@ export declare const TestSuiteSchema: z.ZodObject<{
         label: z.ZodOptional<z.ZodType<string, z.ZodTypeDef, string>>;
         transform: z.ZodOptional<z.ZodString>;
         delay: z.ZodOptional<z.ZodNumber>;
+        config: z.ZodOptional<z.ZodAny>;
     }, "strip", z.ZodTypeAny, {
         callApi: import("./providers").CallApiFunction;
         id: (...args: unknown[]) => string;
+        config?: any;
         label?: string | undefined;
         transform?: string | undefined;
         delay?: number | undefined;
@@ -6151,6 +6203,7 @@ export declare const TestSuiteSchema: z.ZodObject<{
     }, {
         callApi: import("./providers").CallApiFunction;
         id: (...args: unknown[]) => string;
+        config?: any;
         label?: string | undefined;
         transform?: string | undefined;
         delay?: number | undefined;
@@ -6405,9 +6458,11 @@ export declare const TestSuiteSchema: z.ZodObject<{
             label: z.ZodOptional<z.ZodType<string, z.ZodTypeDef, string>>;
             transform: z.ZodOptional<z.ZodString>;
             delay: z.ZodOptional<z.ZodNumber>;
+            config: z.ZodOptional<z.ZodAny>;
         }, "strip", z.ZodTypeAny, {
             callApi: import("./providers").CallApiFunction;
             id: (...args: unknown[]) => string;
+            config?: any;
             label?: string | undefined;
             transform?: string | undefined;
             delay?: number | undefined;
@@ -6416,6 +6471,7 @@ export declare const TestSuiteSchema: z.ZodObject<{
         }, {
             callApi: import("./providers").CallApiFunction;
             id: (...args: unknown[]) => string;
+            config?: any;
             label?: string | undefined;
             transform?: string | undefined;
             delay?: number | undefined;
@@ -6725,6 +6781,7 @@ export declare const TestSuiteSchema: z.ZodObject<{
         } | {
             callApi: import("./providers").CallApiFunction;
             id: (...args: unknown[]) => string;
+            config?: any;
             label?: string | undefined;
             transform?: string | undefined;
             delay?: number | undefined;
@@ -6842,6 +6899,7 @@ export declare const TestSuiteSchema: z.ZodObject<{
         } | {
             callApi: import("./providers").CallApiFunction;
             id: (...args: unknown[]) => string;
+            config?: any;
             label?: string | undefined;
             transform?: string | undefined;
             delay?: number | undefined;
@@ -7098,9 +7156,11 @@ export declare const TestSuiteSchema: z.ZodObject<{
                 label: z.ZodOptional<z.ZodType<string, z.ZodTypeDef, string>>;
                 transform: z.ZodOptional<z.ZodString>;
                 delay: z.ZodOptional<z.ZodNumber>;
+                config: z.ZodOptional<z.ZodAny>;
             }, "strip", z.ZodTypeAny, {
                 callApi: import("./providers").CallApiFunction;
                 id: (...args: unknown[]) => string;
+                config?: any;
                 label?: string | undefined;
                 transform?: string | undefined;
                 delay?: number | undefined;
@@ -7109,6 +7169,7 @@ export declare const TestSuiteSchema: z.ZodObject<{
             }, {
                 callApi: import("./providers").CallApiFunction;
                 id: (...args: unknown[]) => string;
+                config?: any;
                 label?: string | undefined;
                 transform?: string | undefined;
                 delay?: number | undefined;
@@ -7418,6 +7479,7 @@ export declare const TestSuiteSchema: z.ZodObject<{
             } | {
                 callApi: import("./providers").CallApiFunction;
                 id: (...args: unknown[]) => string;
+                config?: any;
                 label?: string | undefined;
                 transform?: string | undefined;
                 delay?: number | undefined;
@@ -7535,6 +7597,7 @@ export declare const TestSuiteSchema: z.ZodObject<{
             } | {
                 callApi: import("./providers").CallApiFunction;
                 id: (...args: unknown[]) => string;
+                config?: any;
                 label?: string | undefined;
                 transform?: string | undefined;
                 delay?: number | undefined;
@@ -7789,9 +7852,11 @@ export declare const TestSuiteSchema: z.ZodObject<{
                 label: z.ZodOptional<z.ZodType<string, z.ZodTypeDef, string>>;
                 transform: z.ZodOptional<z.ZodString>;
                 delay: z.ZodOptional<z.ZodNumber>;
+                config: z.ZodOptional<z.ZodAny>;
             }, "strip", z.ZodTypeAny, {
                 callApi: import("./providers").CallApiFunction;
                 id: (...args: unknown[]) => string;
+                config?: any;
                 label?: string | undefined;
                 transform?: string | undefined;
                 delay?: number | undefined;
@@ -7800,6 +7865,7 @@ export declare const TestSuiteSchema: z.ZodObject<{
             }, {
                 callApi: import("./providers").CallApiFunction;
                 id: (...args: unknown[]) => string;
+                config?: any;
                 label?: string | undefined;
                 transform?: string | undefined;
                 delay?: number | undefined;
@@ -8109,6 +8175,7 @@ export declare const TestSuiteSchema: z.ZodObject<{
             } | {
                 callApi: import("./providers").CallApiFunction;
                 id: (...args: unknown[]) => string;
+                config?: any;
                 label?: string | undefined;
                 transform?: string | undefined;
                 delay?: number | undefined;
@@ -8226,6 +8293,7 @@ export declare const TestSuiteSchema: z.ZodObject<{
             } | {
                 callApi: import("./providers").CallApiFunction;
                 id: (...args: unknown[]) => string;
+                config?: any;
                 label?: string | undefined;
                 transform?: string | undefined;
                 delay?: number | undefined;
@@ -8345,6 +8413,7 @@ export declare const TestSuiteSchema: z.ZodObject<{
             } | {
                 callApi: import("./providers").CallApiFunction;
                 id: (...args: unknown[]) => string;
+                config?: any;
                 label?: string | undefined;
                 transform?: string | undefined;
                 delay?: number | undefined;
@@ -8463,6 +8532,7 @@ export declare const TestSuiteSchema: z.ZodObject<{
             } | {
                 callApi: import("./providers").CallApiFunction;
                 id: (...args: unknown[]) => string;
+                config?: any;
                 label?: string | undefined;
                 transform?: string | undefined;
                 delay?: number | undefined;
@@ -8583,6 +8653,7 @@ export declare const TestSuiteSchema: z.ZodObject<{
             } | {
                 callApi: import("./providers").CallApiFunction;
                 id: (...args: unknown[]) => string;
+                config?: any;
                 label?: string | undefined;
                 transform?: string | undefined;
                 delay?: number | undefined;
@@ -8701,6 +8772,7 @@ export declare const TestSuiteSchema: z.ZodObject<{
             } | {
                 callApi: import("./providers").CallApiFunction;
                 id: (...args: unknown[]) => string;
+                config?: any;
                 label?: string | undefined;
                 transform?: string | undefined;
                 delay?: number | undefined;
@@ -8957,9 +9029,11 @@ export declare const TestSuiteSchema: z.ZodObject<{
             label: z.ZodOptional<z.ZodType<string, z.ZodTypeDef, string>>;
             transform: z.ZodOptional<z.ZodString>;
             delay: z.ZodOptional<z.ZodNumber>;
+            config: z.ZodOptional<z.ZodAny>;
         }, "strip", z.ZodTypeAny, {
             callApi: import("./providers").CallApiFunction;
             id: (...args: unknown[]) => string;
+            config?: any;
             label?: string | undefined;
             transform?: string | undefined;
             delay?: number | undefined;
@@ -8968,6 +9042,7 @@ export declare const TestSuiteSchema: z.ZodObject<{
         }, {
             callApi: import("./providers").CallApiFunction;
             id: (...args: unknown[]) => string;
+            config?: any;
             label?: string | undefined;
             transform?: string | undefined;
             delay?: number | undefined;
@@ -9277,6 +9352,7 @@ export declare const TestSuiteSchema: z.ZodObject<{
         } | {
             callApi: import("./providers").CallApiFunction;
             id: (...args: unknown[]) => string;
+            config?: any;
             label?: string | undefined;
             transform?: string | undefined;
             delay?: number | undefined;
@@ -9394,6 +9470,7 @@ export declare const TestSuiteSchema: z.ZodObject<{
         } | {
             callApi: import("./providers").CallApiFunction;
             id: (...args: unknown[]) => string;
+            config?: any;
             label?: string | undefined;
             transform?: string | undefined;
             delay?: number | undefined;
@@ -9555,7 +9632,7 @@ export declare const TestSuiteSchema: z.ZodObject<{
         value: string | ((args_0: Record<string, number>, args_1: RunEvalOptions, ...args: unknown[]) => number);
         name: string;
     }>, "many">>;
-    extensions: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+    extensions: z.ZodOptional<z.ZodArray<z.ZodEffects<z.ZodEffects<z.ZodEffects<z.ZodString, string, string>, string, string>, string, string>, "many">>;
     redteam: z.ZodOptional<z.ZodType<RedteamFileConfig, z.ZodTypeDef, RedteamFileConfig>>;
 }, "strip", z.ZodTypeAny, {
     prompts: {
@@ -9572,6 +9649,7 @@ export declare const TestSuiteSchema: z.ZodObject<{
     providers: {
         callApi: import("./providers").CallApiFunction;
         id: (...args: unknown[]) => string;
+        config?: any;
         label?: string | undefined;
         transform?: string | undefined;
         delay?: number | undefined;
@@ -9689,6 +9767,7 @@ export declare const TestSuiteSchema: z.ZodObject<{
         } | {
             callApi: import("./providers").CallApiFunction;
             id: (...args: unknown[]) => string;
+            config?: any;
             label?: string | undefined;
             transform?: string | undefined;
             delay?: number | undefined;
@@ -9811,6 +9890,7 @@ export declare const TestSuiteSchema: z.ZodObject<{
             } | {
                 callApi: import("./providers").CallApiFunction;
                 id: (...args: unknown[]) => string;
+                config?: any;
                 label?: string | undefined;
                 transform?: string | undefined;
                 delay?: number | undefined;
@@ -9929,6 +10009,7 @@ export declare const TestSuiteSchema: z.ZodObject<{
             } | {
                 callApi: import("./providers").CallApiFunction;
                 id: (...args: unknown[]) => string;
+                config?: any;
                 label?: string | undefined;
                 transform?: string | undefined;
                 delay?: number | undefined;
@@ -10049,6 +10130,7 @@ export declare const TestSuiteSchema: z.ZodObject<{
         } | {
             callApi: import("./providers").CallApiFunction;
             id: (...args: unknown[]) => string;
+            config?: any;
             label?: string | undefined;
             transform?: string | undefined;
             delay?: number | undefined;
@@ -10114,6 +10196,7 @@ export declare const TestSuiteSchema: z.ZodObject<{
     providers: {
         callApi: import("./providers").CallApiFunction;
         id: (...args: unknown[]) => string;
+        config?: any;
         label?: string | undefined;
         transform?: string | undefined;
         delay?: number | undefined;
@@ -10231,6 +10314,7 @@ export declare const TestSuiteSchema: z.ZodObject<{
         } | {
             callApi: import("./providers").CallApiFunction;
             id: (...args: unknown[]) => string;
+            config?: any;
             label?: string | undefined;
             transform?: string | undefined;
             delay?: number | undefined;
@@ -10353,6 +10437,7 @@ export declare const TestSuiteSchema: z.ZodObject<{
             } | {
                 callApi: import("./providers").CallApiFunction;
                 id: (...args: unknown[]) => string;
+                config?: any;
                 label?: string | undefined;
                 transform?: string | undefined;
                 delay?: number | undefined;
@@ -10471,6 +10556,7 @@ export declare const TestSuiteSchema: z.ZodObject<{
             } | {
                 callApi: import("./providers").CallApiFunction;
                 id: (...args: unknown[]) => string;
+                config?: any;
                 label?: string | undefined;
                 transform?: string | undefined;
                 delay?: number | undefined;
@@ -10591,6 +10677,7 @@ export declare const TestSuiteSchema: z.ZodObject<{
         } | {
             callApi: import("./providers").CallApiFunction;
             id: (...args: unknown[]) => string;
+            config?: any;
             label?: string | undefined;
             transform?: string | undefined;
             delay?: number | undefined;
@@ -11488,9 +11575,11 @@ export declare const TestSuiteConfigSchema: z.ZodObject<{
             label: z.ZodOptional<z.ZodType<string, z.ZodTypeDef, string>>;
             transform: z.ZodOptional<z.ZodString>;
             delay: z.ZodOptional<z.ZodNumber>;
+            config: z.ZodOptional<z.ZodAny>;
         }, "strip", z.ZodTypeAny, {
             callApi: import("./providers").CallApiFunction;
             id: (...args: unknown[]) => string;
+            config?: any;
             label?: string | undefined;
             transform?: string | undefined;
             delay?: number | undefined;
@@ -11499,6 +11588,7 @@ export declare const TestSuiteConfigSchema: z.ZodObject<{
         }, {
             callApi: import("./providers").CallApiFunction;
             id: (...args: unknown[]) => string;
+            config?: any;
             label?: string | undefined;
             transform?: string | undefined;
             delay?: number | undefined;
@@ -11808,6 +11898,7 @@ export declare const TestSuiteConfigSchema: z.ZodObject<{
         } | {
             callApi: import("./providers").CallApiFunction;
             id: (...args: unknown[]) => string;
+            config?: any;
             label?: string | undefined;
             transform?: string | undefined;
             delay?: number | undefined;
@@ -11925,6 +12016,7 @@ export declare const TestSuiteConfigSchema: z.ZodObject<{
         } | {
             callApi: import("./providers").CallApiFunction;
             id: (...args: unknown[]) => string;
+            config?: any;
             label?: string | undefined;
             transform?: string | undefined;
             delay?: number | undefined;
@@ -12181,9 +12273,11 @@ export declare const TestSuiteConfigSchema: z.ZodObject<{
                 label: z.ZodOptional<z.ZodType<string, z.ZodTypeDef, string>>;
                 transform: z.ZodOptional<z.ZodString>;
                 delay: z.ZodOptional<z.ZodNumber>;
+                config: z.ZodOptional<z.ZodAny>;
             }, "strip", z.ZodTypeAny, {
                 callApi: import("./providers").CallApiFunction;
                 id: (...args: unknown[]) => string;
+                config?: any;
                 label?: string | undefined;
                 transform?: string | undefined;
                 delay?: number | undefined;
@@ -12192,6 +12286,7 @@ export declare const TestSuiteConfigSchema: z.ZodObject<{
             }, {
                 callApi: import("./providers").CallApiFunction;
                 id: (...args: unknown[]) => string;
+                config?: any;
                 label?: string | undefined;
                 transform?: string | undefined;
                 delay?: number | undefined;
@@ -12501,6 +12596,7 @@ export declare const TestSuiteConfigSchema: z.ZodObject<{
             } | {
                 callApi: import("./providers").CallApiFunction;
                 id: (...args: unknown[]) => string;
+                config?: any;
                 label?: string | undefined;
                 transform?: string | undefined;
                 delay?: number | undefined;
@@ -12618,6 +12714,7 @@ export declare const TestSuiteConfigSchema: z.ZodObject<{
             } | {
                 callApi: import("./providers").CallApiFunction;
                 id: (...args: unknown[]) => string;
+                config?: any;
                 label?: string | undefined;
                 transform?: string | undefined;
                 delay?: number | undefined;
@@ -12872,9 +12969,11 @@ export declare const TestSuiteConfigSchema: z.ZodObject<{
                 label: z.ZodOptional<z.ZodType<string, z.ZodTypeDef, string>>;
                 transform: z.ZodOptional<z.ZodString>;
                 delay: z.ZodOptional<z.ZodNumber>;
+                config: z.ZodOptional<z.ZodAny>;
             }, "strip", z.ZodTypeAny, {
                 callApi: import("./providers").CallApiFunction;
                 id: (...args: unknown[]) => string;
+                config?: any;
                 label?: string | undefined;
                 transform?: string | undefined;
                 delay?: number | undefined;
@@ -12883,6 +12982,7 @@ export declare const TestSuiteConfigSchema: z.ZodObject<{
             }, {
                 callApi: import("./providers").CallApiFunction;
                 id: (...args: unknown[]) => string;
+                config?: any;
                 label?: string | undefined;
                 transform?: string | undefined;
                 delay?: number | undefined;
@@ -13192,6 +13292,7 @@ export declare const TestSuiteConfigSchema: z.ZodObject<{
             } | {
                 callApi: import("./providers").CallApiFunction;
                 id: (...args: unknown[]) => string;
+                config?: any;
                 label?: string | undefined;
                 transform?: string | undefined;
                 delay?: number | undefined;
@@ -13309,6 +13410,7 @@ export declare const TestSuiteConfigSchema: z.ZodObject<{
             } | {
                 callApi: import("./providers").CallApiFunction;
                 id: (...args: unknown[]) => string;
+                config?: any;
                 label?: string | undefined;
                 transform?: string | undefined;
                 delay?: number | undefined;
@@ -13428,6 +13530,7 @@ export declare const TestSuiteConfigSchema: z.ZodObject<{
             } | {
                 callApi: import("./providers").CallApiFunction;
                 id: (...args: unknown[]) => string;
+                config?: any;
                 label?: string | undefined;
                 transform?: string | undefined;
                 delay?: number | undefined;
@@ -13546,6 +13649,7 @@ export declare const TestSuiteConfigSchema: z.ZodObject<{
             } | {
                 callApi: import("./providers").CallApiFunction;
                 id: (...args: unknown[]) => string;
+                config?: any;
                 label?: string | undefined;
                 transform?: string | undefined;
                 delay?: number | undefined;
@@ -13666,6 +13770,7 @@ export declare const TestSuiteConfigSchema: z.ZodObject<{
             } | {
                 callApi: import("./providers").CallApiFunction;
                 id: (...args: unknown[]) => string;
+                config?: any;
                 label?: string | undefined;
                 transform?: string | undefined;
                 delay?: number | undefined;
@@ -13784,6 +13889,7 @@ export declare const TestSuiteConfigSchema: z.ZodObject<{
             } | {
                 callApi: import("./providers").CallApiFunction;
                 id: (...args: unknown[]) => string;
+                config?: any;
                 label?: string | undefined;
                 transform?: string | undefined;
                 delay?: number | undefined;
@@ -14040,9 +14146,11 @@ export declare const TestSuiteConfigSchema: z.ZodObject<{
             label: z.ZodOptional<z.ZodType<string, z.ZodTypeDef, string>>;
             transform: z.ZodOptional<z.ZodString>;
             delay: z.ZodOptional<z.ZodNumber>;
+            config: z.ZodOptional<z.ZodAny>;
         }, "strip", z.ZodTypeAny, {
             callApi: import("./providers").CallApiFunction;
             id: (...args: unknown[]) => string;
+            config?: any;
             label?: string | undefined;
             transform?: string | undefined;
             delay?: number | undefined;
@@ -14051,6 +14159,7 @@ export declare const TestSuiteConfigSchema: z.ZodObject<{
         }, {
             callApi: import("./providers").CallApiFunction;
             id: (...args: unknown[]) => string;
+            config?: any;
             label?: string | undefined;
             transform?: string | undefined;
             delay?: number | undefined;
@@ -14360,6 +14469,7 @@ export declare const TestSuiteConfigSchema: z.ZodObject<{
         } | {
             callApi: import("./providers").CallApiFunction;
             id: (...args: unknown[]) => string;
+            config?: any;
             label?: string | undefined;
             transform?: string | undefined;
             delay?: number | undefined;
@@ -14476,6 +14586,7 @@ export declare const TestSuiteConfigSchema: z.ZodObject<{
         } | {
             callApi: import("./providers").CallApiFunction;
             id: (...args: unknown[]) => string;
+            config?: any;
             label?: string | undefined;
             transform?: string | undefined;
             delay?: number | undefined;
@@ -14910,6 +15021,7 @@ export declare const TestSuiteConfigSchema: z.ZodObject<{
         } | {
             callApi: import("./providers").CallApiFunction;
             id: (...args: unknown[]) => string;
+            config?: any;
             label?: string | undefined;
             transform?: string | undefined;
             delay?: number | undefined;
@@ -15031,6 +15143,7 @@ export declare const TestSuiteConfigSchema: z.ZodObject<{
             } | {
                 callApi: import("./providers").CallApiFunction;
                 id: (...args: unknown[]) => string;
+                config?: any;
                 label?: string | undefined;
                 transform?: string | undefined;
                 delay?: number | undefined;
@@ -15149,6 +15262,7 @@ export declare const TestSuiteConfigSchema: z.ZodObject<{
             } | {
                 callApi: import("./providers").CallApiFunction;
                 id: (...args: unknown[]) => string;
+                config?: any;
                 label?: string | undefined;
                 transform?: string | undefined;
                 delay?: number | undefined;
@@ -15269,6 +15383,7 @@ export declare const TestSuiteConfigSchema: z.ZodObject<{
         } | {
             callApi: import("./providers").CallApiFunction;
             id: (...args: unknown[]) => string;
+            config?: any;
             label?: string | undefined;
             transform?: string | undefined;
             delay?: number | undefined;
@@ -15583,6 +15698,7 @@ export declare const TestSuiteConfigSchema: z.ZodObject<{
         } | {
             callApi: import("./providers").CallApiFunction;
             id: (...args: unknown[]) => string;
+            config?: any;
             label?: string | undefined;
             transform?: string | undefined;
             delay?: number | undefined;
@@ -15704,6 +15820,7 @@ export declare const TestSuiteConfigSchema: z.ZodObject<{
             } | {
                 callApi: import("./providers").CallApiFunction;
                 id: (...args: unknown[]) => string;
+                config?: any;
                 label?: string | undefined;
                 transform?: string | undefined;
                 delay?: number | undefined;
@@ -15822,6 +15939,7 @@ export declare const TestSuiteConfigSchema: z.ZodObject<{
             } | {
                 callApi: import("./providers").CallApiFunction;
                 id: (...args: unknown[]) => string;
+                config?: any;
                 label?: string | undefined;
                 transform?: string | undefined;
                 delay?: number | undefined;
@@ -15942,6 +16060,7 @@ export declare const TestSuiteConfigSchema: z.ZodObject<{
         } | {
             callApi: import("./providers").CallApiFunction;
             id: (...args: unknown[]) => string;
+            config?: any;
             label?: string | undefined;
             transform?: string | undefined;
             delay?: number | undefined;
@@ -16844,9 +16963,11 @@ export declare const UnifiedConfigSchema: z.ZodObject<z.objectUtil.extendShape<{
             label: z.ZodOptional<z.ZodType<string, z.ZodTypeDef, string>>;
             transform: z.ZodOptional<z.ZodString>;
             delay: z.ZodOptional<z.ZodNumber>;
+            config: z.ZodOptional<z.ZodAny>;
         }, "strip", z.ZodTypeAny, {
             callApi: import("./providers").CallApiFunction;
             id: (...args: unknown[]) => string;
+            config?: any;
             label?: string | undefined;
             transform?: string | undefined;
             delay?: number | undefined;
@@ -16855,6 +16976,7 @@ export declare const UnifiedConfigSchema: z.ZodObject<z.objectUtil.extendShape<{
         }, {
             callApi: import("./providers").CallApiFunction;
             id: (...args: unknown[]) => string;
+            config?: any;
             label?: string | undefined;
             transform?: string | undefined;
             delay?: number | undefined;
@@ -17164,6 +17286,7 @@ export declare const UnifiedConfigSchema: z.ZodObject<z.objectUtil.extendShape<{
         } | {
             callApi: import("./providers").CallApiFunction;
             id: (...args: unknown[]) => string;
+            config?: any;
             label?: string | undefined;
             transform?: string | undefined;
             delay?: number | undefined;
@@ -17281,6 +17404,7 @@ export declare const UnifiedConfigSchema: z.ZodObject<z.objectUtil.extendShape<{
         } | {
             callApi: import("./providers").CallApiFunction;
             id: (...args: unknown[]) => string;
+            config?: any;
             label?: string | undefined;
             transform?: string | undefined;
             delay?: number | undefined;
@@ -17537,9 +17661,11 @@ export declare const UnifiedConfigSchema: z.ZodObject<z.objectUtil.extendShape<{
                 label: z.ZodOptional<z.ZodType<string, z.ZodTypeDef, string>>;
                 transform: z.ZodOptional<z.ZodString>;
                 delay: z.ZodOptional<z.ZodNumber>;
+                config: z.ZodOptional<z.ZodAny>;
             }, "strip", z.ZodTypeAny, {
                 callApi: import("./providers").CallApiFunction;
                 id: (...args: unknown[]) => string;
+                config?: any;
                 label?: string | undefined;
                 transform?: string | undefined;
                 delay?: number | undefined;
@@ -17548,6 +17674,7 @@ export declare const UnifiedConfigSchema: z.ZodObject<z.objectUtil.extendShape<{
             }, {
                 callApi: import("./providers").CallApiFunction;
                 id: (...args: unknown[]) => string;
+                config?: any;
                 label?: string | undefined;
                 transform?: string | undefined;
                 delay?: number | undefined;
@@ -17857,6 +17984,7 @@ export declare const UnifiedConfigSchema: z.ZodObject<z.objectUtil.extendShape<{
             } | {
                 callApi: import("./providers").CallApiFunction;
                 id: (...args: unknown[]) => string;
+                config?: any;
                 label?: string | undefined;
                 transform?: string | undefined;
                 delay?: number | undefined;
@@ -17974,6 +18102,7 @@ export declare const UnifiedConfigSchema: z.ZodObject<z.objectUtil.extendShape<{
             } | {
                 callApi: import("./providers").CallApiFunction;
                 id: (...args: unknown[]) => string;
+                config?: any;
                 label?: string | undefined;
                 transform?: string | undefined;
                 delay?: number | undefined;
@@ -18228,9 +18357,11 @@ export declare const UnifiedConfigSchema: z.ZodObject<z.objectUtil.extendShape<{
                 label: z.ZodOptional<z.ZodType<string, z.ZodTypeDef, string>>;
                 transform: z.ZodOptional<z.ZodString>;
                 delay: z.ZodOptional<z.ZodNumber>;
+                config: z.ZodOptional<z.ZodAny>;
             }, "strip", z.ZodTypeAny, {
                 callApi: import("./providers").CallApiFunction;
                 id: (...args: unknown[]) => string;
+                config?: any;
                 label?: string | undefined;
                 transform?: string | undefined;
                 delay?: number | undefined;
@@ -18239,6 +18370,7 @@ export declare const UnifiedConfigSchema: z.ZodObject<z.objectUtil.extendShape<{
             }, {
                 callApi: import("./providers").CallApiFunction;
                 id: (...args: unknown[]) => string;
+                config?: any;
                 label?: string | undefined;
                 transform?: string | undefined;
                 delay?: number | undefined;
@@ -18548,6 +18680,7 @@ export declare const UnifiedConfigSchema: z.ZodObject<z.objectUtil.extendShape<{
             } | {
                 callApi: import("./providers").CallApiFunction;
                 id: (...args: unknown[]) => string;
+                config?: any;
                 label?: string | undefined;
                 transform?: string | undefined;
                 delay?: number | undefined;
@@ -18665,6 +18798,7 @@ export declare const UnifiedConfigSchema: z.ZodObject<z.objectUtil.extendShape<{
             } | {
                 callApi: import("./providers").CallApiFunction;
                 id: (...args: unknown[]) => string;
+                config?: any;
                 label?: string | undefined;
                 transform?: string | undefined;
                 delay?: number | undefined;
@@ -18784,6 +18918,7 @@ export declare const UnifiedConfigSchema: z.ZodObject<z.objectUtil.extendShape<{
             } | {
                 callApi: import("./providers").CallApiFunction;
                 id: (...args: unknown[]) => string;
+                config?: any;
                 label?: string | undefined;
                 transform?: string | undefined;
                 delay?: number | undefined;
@@ -18902,6 +19037,7 @@ export declare const UnifiedConfigSchema: z.ZodObject<z.objectUtil.extendShape<{
             } | {
                 callApi: import("./providers").CallApiFunction;
                 id: (...args: unknown[]) => string;
+                config?: any;
                 label?: string | undefined;
                 transform?: string | undefined;
                 delay?: number | undefined;
@@ -19022,6 +19158,7 @@ export declare const UnifiedConfigSchema: z.ZodObject<z.objectUtil.extendShape<{
             } | {
                 callApi: import("./providers").CallApiFunction;
                 id: (...args: unknown[]) => string;
+                config?: any;
                 label?: string | undefined;
                 transform?: string | undefined;
                 delay?: number | undefined;
@@ -19140,6 +19277,7 @@ export declare const UnifiedConfigSchema: z.ZodObject<z.objectUtil.extendShape<{
             } | {
                 callApi: import("./providers").CallApiFunction;
                 id: (...args: unknown[]) => string;
+                config?: any;
                 label?: string | undefined;
                 transform?: string | undefined;
                 delay?: number | undefined;
@@ -19396,9 +19534,11 @@ export declare const UnifiedConfigSchema: z.ZodObject<z.objectUtil.extendShape<{
             label: z.ZodOptional<z.ZodType<string, z.ZodTypeDef, string>>;
             transform: z.ZodOptional<z.ZodString>;
             delay: z.ZodOptional<z.ZodNumber>;
+            config: z.ZodOptional<z.ZodAny>;
         }, "strip", z.ZodTypeAny, {
             callApi: import("./providers").CallApiFunction;
             id: (...args: unknown[]) => string;
+            config?: any;
             label?: string | undefined;
             transform?: string | undefined;
             delay?: number | undefined;
@@ -19407,6 +19547,7 @@ export declare const UnifiedConfigSchema: z.ZodObject<z.objectUtil.extendShape<{
         }, {
             callApi: import("./providers").CallApiFunction;
             id: (...args: unknown[]) => string;
+            config?: any;
             label?: string | undefined;
             transform?: string | undefined;
             delay?: number | undefined;
@@ -19716,6 +19857,7 @@ export declare const UnifiedConfigSchema: z.ZodObject<z.objectUtil.extendShape<{
         } | {
             callApi: import("./providers").CallApiFunction;
             id: (...args: unknown[]) => string;
+            config?: any;
             label?: string | undefined;
             transform?: string | undefined;
             delay?: number | undefined;
@@ -19832,6 +19974,7 @@ export declare const UnifiedConfigSchema: z.ZodObject<z.objectUtil.extendShape<{
         } | {
             callApi: import("./providers").CallApiFunction;
             id: (...args: unknown[]) => string;
+            config?: any;
             label?: string | undefined;
             transform?: string | undefined;
             delay?: number | undefined;
@@ -20397,6 +20540,7 @@ export declare const UnifiedConfigSchema: z.ZodObject<z.objectUtil.extendShape<{
         } | {
             callApi: import("./providers").CallApiFunction;
             id: (...args: unknown[]) => string;
+            config?: any;
             label?: string | undefined;
             transform?: string | undefined;
             delay?: number | undefined;
@@ -20518,6 +20662,7 @@ export declare const UnifiedConfigSchema: z.ZodObject<z.objectUtil.extendShape<{
             } | {
                 callApi: import("./providers").CallApiFunction;
                 id: (...args: unknown[]) => string;
+                config?: any;
                 label?: string | undefined;
                 transform?: string | undefined;
                 delay?: number | undefined;
@@ -20636,6 +20781,7 @@ export declare const UnifiedConfigSchema: z.ZodObject<z.objectUtil.extendShape<{
             } | {
                 callApi: import("./providers").CallApiFunction;
                 id: (...args: unknown[]) => string;
+                config?: any;
                 label?: string | undefined;
                 transform?: string | undefined;
                 delay?: number | undefined;
@@ -20756,6 +20902,7 @@ export declare const UnifiedConfigSchema: z.ZodObject<z.objectUtil.extendShape<{
         } | {
             callApi: import("./providers").CallApiFunction;
             id: (...args: unknown[]) => string;
+            config?: any;
             label?: string | undefined;
             transform?: string | undefined;
             delay?: number | undefined;
@@ -21113,6 +21260,7 @@ export declare const UnifiedConfigSchema: z.ZodObject<z.objectUtil.extendShape<{
         } | {
             callApi: import("./providers").CallApiFunction;
             id: (...args: unknown[]) => string;
+            config?: any;
             label?: string | undefined;
             transform?: string | undefined;
             delay?: number | undefined;
@@ -21234,6 +21382,7 @@ export declare const UnifiedConfigSchema: z.ZodObject<z.objectUtil.extendShape<{
             } | {
                 callApi: import("./providers").CallApiFunction;
                 id: (...args: unknown[]) => string;
+                config?: any;
                 label?: string | undefined;
                 transform?: string | undefined;
                 delay?: number | undefined;
@@ -21352,6 +21501,7 @@ export declare const UnifiedConfigSchema: z.ZodObject<z.objectUtil.extendShape<{
             } | {
                 callApi: import("./providers").CallApiFunction;
                 id: (...args: unknown[]) => string;
+                config?: any;
                 label?: string | undefined;
                 transform?: string | undefined;
                 delay?: number | undefined;
@@ -21472,6 +21622,7 @@ export declare const UnifiedConfigSchema: z.ZodObject<z.objectUtil.extendShape<{
         } | {
             callApi: import("./providers").CallApiFunction;
             id: (...args: unknown[]) => string;
+            config?: any;
             label?: string | undefined;
             transform?: string | undefined;
             delay?: number | undefined;
@@ -21576,7 +21727,7 @@ export interface EvalWithMetadata {
     id: string;
     date: Date;
     config: Partial<UnifiedConfig>;
-    results: EvaluateSummary;
+    results: EvaluateSummaryV3;
     description?: string;
 }
 export type EvaluateTestSuite = {
@@ -21592,9 +21743,10 @@ export interface SharedResults {
 export interface ResultsFile {
     version: number;
     createdAt: string;
-    results: EvaluateSummary;
+    results: EvaluateSummaryV3 | EvaluateSummaryV2;
     config: Partial<UnifiedConfig>;
     author: string | null;
+    prompts?: CompletedPrompt[];
     datasetId?: string | null;
 }
 export interface ResultLightweight {
@@ -21609,7 +21761,7 @@ export type ResultLightweightWithLabel = ResultLightweight & {
 };
 export interface OutputFile {
     evalId: string | null;
-    results: EvaluateSummary;
+    results: EvaluateSummaryV3 | EvaluateSummaryV2;
     config: Partial<UnifiedConfig>;
     shareableUrl: string | null;
 }
@@ -21617,7 +21769,7 @@ export interface Job {
     status: 'in-progress' | 'complete';
     progress: number;
     total: number;
-    result: EvaluateSummary | null;
+    result: EvaluateSummaryV3 | EvaluateSummaryV2 | null;
 }
 export declare const OutputFileExtension: z.ZodEnum<["csv", "html", "json", "txt", "yaml", "yml"]>;
 export type OutputFileExtension = z.infer<typeof OutputFileExtension>;