promptfoo 0.112.4 → 0.112.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (134) hide show
  1. package/dist/package.json +40 -35
  2. package/dist/src/app/assets/{index-CjtcgXub.js → index-1DW9E3Nc.js} +125 -125
  3. package/dist/src/app/assets/{index.es-CTVmcuv3.js → index.es-CRVd21wS.js} +1 -1
  4. package/dist/src/app/assets/{source-map-support-DcTymX4v.js → source-map-support-BfNnIDrr.js} +1 -1
  5. package/dist/src/app/assets/{sync-DT4gZaAv.js → sync-B6RviJxk.js} +1 -1
  6. package/dist/src/app/index.html +1 -1
  7. package/dist/src/commands/eval.d.ts.map +1 -1
  8. package/dist/src/commands/eval.js +5 -5
  9. package/dist/src/commands/eval.js.map +1 -1
  10. package/dist/src/commands/init.d.ts.map +1 -1
  11. package/dist/src/commands/init.js +0 -2
  12. package/dist/src/commands/init.js.map +1 -1
  13. package/dist/src/commands/modelScan.d.ts.map +1 -1
  14. package/dist/src/commands/modelScan.js +0 -4
  15. package/dist/src/commands/modelScan.js.map +1 -1
  16. package/dist/src/commands/share.d.ts.map +1 -1
  17. package/dist/src/commands/share.js +0 -3
  18. package/dist/src/commands/share.js.map +1 -1
  19. package/dist/src/evaluator.d.ts.map +1 -1
  20. package/dist/src/evaluator.js +10 -1
  21. package/dist/src/evaluator.js.map +1 -1
  22. package/dist/src/evaluatorHelpers.d.ts.map +1 -1
  23. package/dist/src/evaluatorHelpers.js +18 -1
  24. package/dist/src/evaluatorHelpers.js.map +1 -1
  25. package/dist/src/fetch.d.ts +0 -1
  26. package/dist/src/fetch.d.ts.map +1 -1
  27. package/dist/src/fetch.js +17 -13
  28. package/dist/src/fetch.js.map +1 -1
  29. package/dist/src/main.d.ts +5 -1
  30. package/dist/src/main.d.ts.map +1 -1
  31. package/dist/src/main.js +31 -5
  32. package/dist/src/main.js.map +1 -1
  33. package/dist/src/providers/anthropic/messages.d.ts.map +1 -1
  34. package/dist/src/providers/anthropic/messages.js +1 -0
  35. package/dist/src/providers/anthropic/messages.js.map +1 -1
  36. package/dist/src/providers/azure/chat.d.ts.map +1 -1
  37. package/dist/src/providers/azure/chat.js +1 -0
  38. package/dist/src/providers/azure/chat.js.map +1 -1
  39. package/dist/src/providers/bedrock.d.ts +4 -3
  40. package/dist/src/providers/bedrock.d.ts.map +1 -1
  41. package/dist/src/providers/bedrock.js +110 -70
  42. package/dist/src/providers/bedrock.js.map +1 -1
  43. package/dist/src/providers/google/ai.studio.d.ts +10 -1
  44. package/dist/src/providers/google/ai.studio.d.ts.map +1 -1
  45. package/dist/src/providers/google/ai.studio.js +64 -1
  46. package/dist/src/providers/google/ai.studio.js.map +1 -1
  47. package/dist/src/providers/google/shared.d.ts.map +1 -1
  48. package/dist/src/providers/google/shared.js +14 -3
  49. package/dist/src/providers/google/shared.js.map +1 -1
  50. package/dist/src/providers/google/vertex.d.ts.map +1 -1
  51. package/dist/src/providers/google/vertex.js +1 -0
  52. package/dist/src/providers/google/vertex.js.map +1 -1
  53. package/dist/src/providers/http.js +2 -2
  54. package/dist/src/providers/http.js.map +1 -1
  55. package/dist/src/providers/mcp/client.d.ts.map +1 -1
  56. package/dist/src/providers/mcp/client.js +8 -4
  57. package/dist/src/providers/mcp/client.js.map +1 -1
  58. package/dist/src/providers/openai/chat.d.ts.map +1 -1
  59. package/dist/src/providers/openai/chat.js +1 -0
  60. package/dist/src/providers/openai/chat.js.map +1 -1
  61. package/dist/src/providers/perplexity.d.ts +75 -0
  62. package/dist/src/providers/perplexity.d.ts.map +1 -0
  63. package/dist/src/providers/perplexity.js +154 -0
  64. package/dist/src/providers/perplexity.js.map +1 -0
  65. package/dist/src/providers/registry.d.ts.map +1 -1
  66. package/dist/src/providers/registry.js +4 -9
  67. package/dist/src/providers/registry.js.map +1 -1
  68. package/dist/src/providers/sagemaker.d.ts +1 -5
  69. package/dist/src/providers/sagemaker.d.ts.map +1 -1
  70. package/dist/src/providers/scriptBasedProvider.js +1 -1
  71. package/dist/src/providers/scriptBasedProvider.js.map +1 -1
  72. package/dist/src/redteam/commands/generate.d.ts.map +1 -1
  73. package/dist/src/redteam/commands/generate.js +14 -6
  74. package/dist/src/redteam/commands/generate.js.map +1 -1
  75. package/dist/src/redteam/commands/poison.d.ts.map +1 -1
  76. package/dist/src/redteam/commands/poison.js +1 -5
  77. package/dist/src/redteam/commands/poison.js.map +1 -1
  78. package/dist/src/redteam/commands/run.d.ts.map +1 -1
  79. package/dist/src/redteam/commands/run.js +1 -39
  80. package/dist/src/redteam/commands/run.js.map +1 -1
  81. package/dist/src/redteam/constants.d.ts +23 -8
  82. package/dist/src/redteam/constants.d.ts.map +1 -1
  83. package/dist/src/redteam/constants.js +101 -16
  84. package/dist/src/redteam/constants.js.map +1 -1
  85. package/dist/src/redteam/graders.d.ts.map +1 -1
  86. package/dist/src/redteam/graders.js +1 -0
  87. package/dist/src/redteam/graders.js.map +1 -1
  88. package/dist/src/redteam/plugins/harmful/graders.d.ts +4 -0
  89. package/dist/src/redteam/plugins/harmful/graders.d.ts.map +1 -1
  90. package/dist/src/redteam/plugins/harmful/graders.js +49 -1
  91. package/dist/src/redteam/plugins/harmful/graders.js.map +1 -1
  92. package/dist/src/redteam/plugins/index.d.ts.map +1 -1
  93. package/dist/src/redteam/plugins/index.js +6 -1
  94. package/dist/src/redteam/plugins/index.js.map +1 -1
  95. package/dist/src/redteam/plugins/pii.d.ts.map +1 -1
  96. package/dist/src/redteam/plugins/pii.js +5 -0
  97. package/dist/src/redteam/plugins/pii.js.map +1 -1
  98. package/dist/src/redteam/plugins/shellInjection.d.ts.map +1 -1
  99. package/dist/src/redteam/plugins/shellInjection.js +4 -2
  100. package/dist/src/redteam/plugins/shellInjection.js.map +1 -1
  101. package/dist/src/redteam/providers/crescendo/index.d.ts +1 -1
  102. package/dist/src/redteam/providers/crescendo/index.d.ts.map +1 -1
  103. package/dist/src/redteam/providers/crescendo/index.js +1 -1
  104. package/dist/src/redteam/providers/crescendo/index.js.map +1 -1
  105. package/dist/src/redteam/providers/iterative.d.ts.map +1 -1
  106. package/dist/src/redteam/providers/iterative.js +5 -2
  107. package/dist/src/redteam/providers/iterative.js.map +1 -1
  108. package/dist/src/redteam/providers/iterativeTree.d.ts +13 -7
  109. package/dist/src/redteam/providers/iterativeTree.d.ts.map +1 -1
  110. package/dist/src/redteam/providers/iterativeTree.js +49 -27
  111. package/dist/src/redteam/providers/iterativeTree.js.map +1 -1
  112. package/dist/src/redteam/providers/prompts.d.ts.map +1 -1
  113. package/dist/src/redteam/providers/prompts.js +20 -2
  114. package/dist/src/redteam/providers/prompts.js.map +1 -1
  115. package/dist/src/redteam/types.d.ts +1 -0
  116. package/dist/src/redteam/types.d.ts.map +1 -1
  117. package/dist/src/types/index.d.ts +976 -2061
  118. package/dist/src/types/index.d.ts.map +1 -1
  119. package/dist/src/types/shared.d.ts +3 -3
  120. package/dist/src/util/config/load.d.ts.map +1 -1
  121. package/dist/src/util/config/load.js +2 -1
  122. package/dist/src/util/config/load.js.map +1 -1
  123. package/dist/src/util/file.d.ts +1 -2
  124. package/dist/src/util/file.d.ts.map +1 -1
  125. package/dist/src/util/file.js +4 -7
  126. package/dist/src/util/file.js.map +1 -1
  127. package/dist/src/util/formatDuration.d.ts +7 -0
  128. package/dist/src/util/formatDuration.d.ts.map +1 -0
  129. package/dist/src/util/formatDuration.js +27 -0
  130. package/dist/src/util/formatDuration.js.map +1 -0
  131. package/dist/src/validators/providers.d.ts +3 -3
  132. package/dist/src/validators/redteam.d.ts +5 -5
  133. package/dist/tsconfig.tsbuildinfo +1 -1
  134. package/package.json +40 -35
package/dist/src/types/index.d.ts CHANGED
@@ -444,7 +444,7 @@ declare const PromptMetricsSchema: z.ZodObject<{
444
444
  assertPassCount: z.ZodNumber;
445
445
  assertFailCount: z.ZodNumber;
446
446
  totalLatencyMs: z.ZodNumber;
447
- tokenUsage: z.ZodObject<z.objectUtil.extendShape<{
447
+ tokenUsage: z.ZodObject<{
448
448
  prompt: z.ZodOptional<z.ZodNumber>;
449
449
  completion: z.ZodOptional<z.ZodNumber>;
450
450
  cached: z.ZodOptional<z.ZodNumber>;
@@ -463,7 +463,7 @@ declare const PromptMetricsSchema: z.ZodObject<{
463
463
  acceptedPrediction?: number | undefined;
464
464
  rejectedPrediction?: number | undefined;
465
465
  }>>;
466
- }, {
466
+ } & {
467
467
  assertions: z.ZodOptional<z.ZodObject<{
468
468
  prompt: z.ZodOptional<z.ZodNumber>;
469
469
  completion: z.ZodOptional<z.ZodNumber>;
@@ -506,7 +506,7 @@ declare const PromptMetricsSchema: z.ZodObject<{
506
506
  rejectedPrediction?: number | undefined;
507
507
  } | undefined;
508
508
  }>>;
509
- }>, "strip", z.ZodTypeAny, {
509
+ }, "strip", z.ZodTypeAny, {
510
510
  prompt?: number | undefined;
511
511
  completion?: number | undefined;
512
512
  cached?: number | undefined;
@@ -656,7 +656,7 @@ declare const PromptMetricsSchema: z.ZodObject<{
656
656
  } | undefined;
657
657
  }>;
658
658
  export type PromptMetrics = z.infer<typeof PromptMetricsSchema>;
659
- export declare const CompletedPromptSchema: z.ZodObject<z.objectUtil.extendShape<{
659
+ export declare const CompletedPromptSchema: z.ZodObject<{
660
660
  id: z.ZodOptional<z.ZodString>;
661
661
  raw: z.ZodString;
662
662
  display: z.ZodOptional<z.ZodString>;
@@ -672,7 +672,7 @@ export declare const CompletedPromptSchema: z.ZodObject<z.objectUtil.extendShape
672
672
  provider?: ApiProvider | undefined;
673
673
  }>], z.ZodUnknown>, z.ZodPromise<z.ZodUnion<[z.ZodString, z.ZodAny]>>>>;
674
674
  config: z.ZodOptional<z.ZodAny>;
675
- }, {
675
+ } & {
676
676
  provider: z.ZodString;
677
677
  metrics: z.ZodOptional<z.ZodObject<{
678
678
  score: z.ZodNumber;
@@ -682,7 +682,7 @@ export declare const CompletedPromptSchema: z.ZodObject<z.objectUtil.extendShape
682
682
  assertPassCount: z.ZodNumber;
683
683
  assertFailCount: z.ZodNumber;
684
684
  totalLatencyMs: z.ZodNumber;
685
- tokenUsage: z.ZodObject<z.objectUtil.extendShape<{
685
+ tokenUsage: z.ZodObject<{
686
686
  prompt: z.ZodOptional<z.ZodNumber>;
687
687
  completion: z.ZodOptional<z.ZodNumber>;
688
688
  cached: z.ZodOptional<z.ZodNumber>;
@@ -701,7 +701,7 @@ export declare const CompletedPromptSchema: z.ZodObject<z.objectUtil.extendShape
701
701
  acceptedPrediction?: number | undefined;
702
702
  rejectedPrediction?: number | undefined;
703
703
  }>>;
704
- }, {
704
+ } & {
705
705
  assertions: z.ZodOptional<z.ZodObject<{
706
706
  prompt: z.ZodOptional<z.ZodNumber>;
707
707
  completion: z.ZodOptional<z.ZodNumber>;
@@ -744,7 +744,7 @@ export declare const CompletedPromptSchema: z.ZodObject<z.objectUtil.extendShape
744
744
  rejectedPrediction?: number | undefined;
745
745
  } | undefined;
746
746
  }>>;
747
- }>, "strip", z.ZodTypeAny, {
747
+ }, "strip", z.ZodTypeAny, {
748
748
  prompt?: number | undefined;
749
749
  completion?: number | undefined;
750
750
  cached?: number | undefined;
@@ -893,7 +893,7 @@ export declare const CompletedPromptSchema: z.ZodObject<z.objectUtil.extendShape
893
893
  strategyFailCount: Record<string, number>;
894
894
  } | undefined;
895
895
  }>>;
896
- }>, "strip", z.ZodTypeAny, {
896
+ }, "strip", z.ZodTypeAny, {
897
897
  provider: string;
898
898
  raw: string;
899
899
  label: string;
@@ -1295,7 +1295,7 @@ export interface AssertionParams {
1295
1295
  valueFromScript?: string | boolean | number | GradingResult | object;
1296
1296
  }
1297
1297
  export declare const TestCasesWithMetadataPromptSchema: z.ZodObject<{
1298
- prompt: z.ZodObject<z.objectUtil.extendShape<{
1298
+ prompt: z.ZodObject<{
1299
1299
  id: z.ZodOptional<z.ZodString>;
1300
1300
  raw: z.ZodString;
1301
1301
  display: z.ZodOptional<z.ZodString>;
@@ -1311,7 +1311,7 @@ export declare const TestCasesWithMetadataPromptSchema: z.ZodObject<{
1311
1311
  provider?: ApiProvider | undefined;
1312
1312
  }>], z.ZodUnknown>, z.ZodPromise<z.ZodUnion<[z.ZodString, z.ZodAny]>>>>;
1313
1313
  config: z.ZodOptional<z.ZodAny>;
1314
- }, {
1314
+ } & {
1315
1315
  provider: z.ZodString;
1316
1316
  metrics: z.ZodOptional<z.ZodObject<{
1317
1317
  score: z.ZodNumber;
@@ -1321,7 +1321,7 @@ export declare const TestCasesWithMetadataPromptSchema: z.ZodObject<{
1321
1321
  assertPassCount: z.ZodNumber;
1322
1322
  assertFailCount: z.ZodNumber;
1323
1323
  totalLatencyMs: z.ZodNumber;
1324
- tokenUsage: z.ZodObject<z.objectUtil.extendShape<{
1324
+ tokenUsage: z.ZodObject<{
1325
1325
  prompt: z.ZodOptional<z.ZodNumber>;
1326
1326
  completion: z.ZodOptional<z.ZodNumber>;
1327
1327
  cached: z.ZodOptional<z.ZodNumber>;
@@ -1340,7 +1340,7 @@ export declare const TestCasesWithMetadataPromptSchema: z.ZodObject<{
1340
1340
  acceptedPrediction?: number | undefined;
1341
1341
  rejectedPrediction?: number | undefined;
1342
1342
  }>>;
1343
- }, {
1343
+ } & {
1344
1344
  assertions: z.ZodOptional<z.ZodObject<{
1345
1345
  prompt: z.ZodOptional<z.ZodNumber>;
1346
1346
  completion: z.ZodOptional<z.ZodNumber>;
@@ -1383,7 +1383,7 @@ export declare const TestCasesWithMetadataPromptSchema: z.ZodObject<{
1383
1383
  rejectedPrediction?: number | undefined;
1384
1384
  } | undefined;
1385
1385
  }>>;
1386
- }>, "strip", z.ZodTypeAny, {
1386
+ }, "strip", z.ZodTypeAny, {
1387
1387
  prompt?: number | undefined;
1388
1388
  completion?: number | undefined;
1389
1389
  cached?: number | undefined;
@@ -1532,7 +1532,7 @@ export declare const TestCasesWithMetadataPromptSchema: z.ZodObject<{
1532
1532
  strategyFailCount: Record<string, number>;
1533
1533
  } | undefined;
1534
1534
  }>>;
1535
- }>, "strip", z.ZodTypeAny, {
1535
+ }, "strip", z.ZodTypeAny, {
1536
1536
  provider: string;
1537
1537
  raw: string;
1538
1538
  label: string;
@@ -2847,9 +2847,8 @@ export type TestCaseWithPlugin = TestCase & {
2847
2847
  pluginId: string;
2848
2848
  };
2849
2849
  };
2850
- export declare const TestCaseWithVarsFileSchema: z.ZodObject<z.objectUtil.extendShape<{
2850
+ export declare const TestCaseWithVarsFileSchema: z.ZodObject<{
2851
2851
  description: z.ZodOptional<z.ZodString>;
2852
- vars: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnion<[z.ZodString, z.ZodNumber, z.ZodBoolean, z.ZodArray<z.ZodUnion<[z.ZodString, z.ZodNumber, z.ZodBoolean]>, "many">, z.ZodRecord<z.ZodString, z.ZodAny>, z.ZodArray<z.ZodAny, "many">]>>>;
2853
2852
  provider: z.ZodOptional<z.ZodUnion<[z.ZodString, z.ZodObject<{
2854
2853
  id: z.ZodOptional<z.ZodType<string, z.ZodTypeDef, string>>;
2855
2854
  label: z.ZodOptional<z.ZodType<string, z.ZodTypeDef, string>>;
@@ -3569,9 +3568,9 @@ export declare const TestCaseWithVarsFileSchema: z.ZodObject<z.objectUtil.extend
3569
3568
  pluginConfig?: PluginConfig | undefined;
3570
3569
  strategyConfig?: import("../redteam/types").RedteamObjectConfig | undefined;
3571
3570
  }>>>;
3572
- }, {
3571
+ } & {
3573
3572
  vars: z.ZodOptional<z.ZodUnion<[z.ZodRecord<z.ZodString, z.ZodUnion<[z.ZodString, z.ZodNumber, z.ZodBoolean, z.ZodArray<z.ZodUnion<[z.ZodString, z.ZodNumber, z.ZodBoolean]>, "many">, z.ZodRecord<z.ZodString, z.ZodAny>, z.ZodArray<z.ZodAny, "many">]>>, z.ZodString, z.ZodArray<z.ZodString, "many">]>>;
3574
- }>, "strip", z.ZodTypeAny, {
3573
+ }, "strip", z.ZodTypeAny, {
3575
3574
  options?: ({
3576
3575
  prefix?: string | undefined;
3577
3576
  suffix?: string | undefined;
@@ -4994,7 +4993,7 @@ export declare const TestCasesWithMetadataSchema: z.ZodObject<{
4994
4993
  recentEvalId: z.ZodString;
4995
4994
  count: z.ZodNumber;
4996
4995
  prompts: z.ZodArray<z.ZodObject<{
4997
- prompt: z.ZodObject<z.objectUtil.extendShape<{
4996
+ prompt: z.ZodObject<{
4998
4997
  id: z.ZodOptional<z.ZodString>;
4999
4998
  raw: z.ZodString;
5000
4999
  display: z.ZodOptional<z.ZodString>;
@@ -5010,7 +5009,7 @@ export declare const TestCasesWithMetadataSchema: z.ZodObject<{
5010
5009
  provider?: ApiProvider | undefined;
5011
5010
  }>], z.ZodUnknown>, z.ZodPromise<z.ZodUnion<[z.ZodString, z.ZodAny]>>>>;
5012
5011
  config: z.ZodOptional<z.ZodAny>;
5013
- }, {
5012
+ } & {
5014
5013
  provider: z.ZodString;
5015
5014
  metrics: z.ZodOptional<z.ZodObject<{
5016
5015
  score: z.ZodNumber;
@@ -5020,7 +5019,7 @@ export declare const TestCasesWithMetadataSchema: z.ZodObject<{
5020
5019
  assertPassCount: z.ZodNumber;
5021
5020
  assertFailCount: z.ZodNumber;
5022
5021
  totalLatencyMs: z.ZodNumber;
5023
- tokenUsage: z.ZodObject<z.objectUtil.extendShape<{
5022
+ tokenUsage: z.ZodObject<{
5024
5023
  prompt: z.ZodOptional<z.ZodNumber>;
5025
5024
  completion: z.ZodOptional<z.ZodNumber>;
5026
5025
  cached: z.ZodOptional<z.ZodNumber>;
@@ -5039,7 +5038,7 @@ export declare const TestCasesWithMetadataSchema: z.ZodObject<{
5039
5038
  acceptedPrediction?: number | undefined;
5040
5039
  rejectedPrediction?: number | undefined;
5041
5040
  }>>;
5042
- }, {
5041
+ } & {
5043
5042
  assertions: z.ZodOptional<z.ZodObject<{
5044
5043
  prompt: z.ZodOptional<z.ZodNumber>;
5045
5044
  completion: z.ZodOptional<z.ZodNumber>;
@@ -5082,7 +5081,7 @@ export declare const TestCasesWithMetadataSchema: z.ZodObject<{
5082
5081
  rejectedPrediction?: number | undefined;
5083
5082
  } | undefined;
5084
5083
  }>>;
5085
- }>, "strip", z.ZodTypeAny, {
5084
+ }, "strip", z.ZodTypeAny, {
5086
5085
  prompt?: number | undefined;
5087
5086
  completion?: number | undefined;
5088
5087
  cached?: number | undefined;
@@ -5231,7 +5230,7 @@ export declare const TestCasesWithMetadataSchema: z.ZodObject<{
5231
5230
  strategyFailCount: Record<string, number>;
5232
5231
  } | undefined;
5233
5232
  }>>;
5234
- }>, "strip", z.ZodTypeAny, {
5233
+ }, "strip", z.ZodTypeAny, {
5235
5234
  provider: string;
5236
5235
  raw: string;
5237
5236
  label: string;
@@ -8773,9 +8772,8 @@ export declare const ScenarioSchema: z.ZodObject<{
8773
8772
  description?: string | undefined;
8774
8773
  }>;
8775
8774
  export type Scenario = z.infer<typeof ScenarioSchema>;
8776
- export declare const AtomicTestCaseSchema: z.ZodObject<z.objectUtil.extendShape<{
8775
+ export declare const AtomicTestCaseSchema: z.ZodObject<{
8777
8776
  description: z.ZodOptional<z.ZodString>;
8778
- vars: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnion<[z.ZodString, z.ZodNumber, z.ZodBoolean, z.ZodArray<z.ZodUnion<[z.ZodString, z.ZodNumber, z.ZodBoolean]>, "many">, z.ZodRecord<z.ZodString, z.ZodAny>, z.ZodArray<z.ZodAny, "many">]>>>;
8779
8777
  provider: z.ZodOptional<z.ZodUnion<[z.ZodString, z.ZodObject<{
8780
8778
  id: z.ZodOptional<z.ZodType<string, z.ZodTypeDef, string>>;
8781
8779
  label: z.ZodOptional<z.ZodType<string, z.ZodTypeDef, string>>;
@@ -9495,9 +9493,9 @@ export declare const AtomicTestCaseSchema: z.ZodObject<z.objectUtil.extendShape<
9495
9493
  pluginConfig?: PluginConfig | undefined;
9496
9494
  strategyConfig?: import("../redteam/types").RedteamObjectConfig | undefined;
9497
9495
  }>>>;
9498
- }, {
9496
+ } & {
9499
9497
  vars: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnion<[z.ZodString, z.ZodObject<{}, "strip", z.ZodTypeAny, {}, {}>]>>>;
9500
- }>, "strict", z.ZodTypeAny, {
9498
+ }, "strict", z.ZodTypeAny, {
9501
9499
  options?: ({
9502
9500
  prefix?: string | undefined;
9503
9501
  suffix?: string | undefined;
@@ -23722,7 +23720,6 @@ export declare const TestSuiteConfigSchema: z.ZodObject<{
23722
23720
  })[];
23723
23721
  maxConcurrency?: number | undefined;
23724
23722
  entities?: string[] | undefined;
23725
- numTests?: number | undefined;
23726
23723
  provider?: string | {
23727
23724
  id?: string | undefined;
23728
23725
  config?: any;
@@ -23827,6 +23824,7 @@ export declare const TestSuiteConfigSchema: z.ZodObject<{
23827
23824
  callClassificationApi?: ((args_0: string, ...args: unknown[]) => Promise<import("./providers").ProviderClassificationResponse>) | undefined;
23828
23825
  } | undefined;
23829
23826
  delay?: number | undefined;
23827
+ numTests?: number | undefined;
23830
23828
  injectVar?: string | undefined;
23831
23829
  language?: string | undefined;
23832
23830
  purpose?: string | undefined;
@@ -23839,7 +23837,6 @@ export declare const TestSuiteConfigSchema: z.ZodObject<{
23839
23837
  })[] | undefined;
23840
23838
  maxConcurrency?: number | undefined;
23841
23839
  entities?: string[] | undefined;
23842
- numTests?: number | undefined;
23843
23840
  provider?: string | {
23844
23841
  id?: string | undefined;
23845
23842
  config?: any;
@@ -23944,6 +23941,7 @@ export declare const TestSuiteConfigSchema: z.ZodObject<{
23944
23941
  callClassificationApi?: ((args_0: string, ...args: unknown[]) => Promise<import("./providers").ProviderClassificationResponse>) | undefined;
23945
23942
  } | undefined;
23946
23943
  delay?: number | undefined;
23944
+ numTests?: number | undefined;
23947
23945
  injectVar?: string | undefined;
23948
23946
  language?: string | undefined;
23949
23947
  purpose?: string | undefined;
@@ -23960,484 +23958,484 @@ export declare const TestSuiteConfigSchema: z.ZodObject<{
23960
23958
  })[] | undefined;
23961
23959
  maxConcurrency?: number | undefined;
23962
23960
  entities?: string[] | undefined;
23961
+ provider?: string | {
23962
+ id?: string | undefined;
23963
+ config?: any;
23964
+ label?: string | undefined;
23965
+ prompts?: string[] | undefined;
23966
+ transform?: string | undefined;
23967
+ delay?: number | undefined;
23968
+ env?: {
23969
+ AI21_API_BASE_URL?: string | undefined;
23970
+ AI21_API_KEY?: string | undefined;
23971
+ ANTHROPIC_API_KEY?: string | undefined;
23972
+ ANTHROPIC_BASE_URL?: string | undefined;
23973
+ AWS_BEDROCK_REGION?: string | undefined;
23974
+ AZURE_API_BASE_URL?: string | undefined;
23975
+ AZURE_API_HOST?: string | undefined;
23976
+ AZURE_API_KEY?: string | undefined;
23977
+ AZURE_AUTHORITY_HOST?: string | undefined;
23978
+ AZURE_CLIENT_ID?: string | undefined;
23979
+ AZURE_CLIENT_SECRET?: string | undefined;
23980
+ AZURE_DEPLOYMENT_NAME?: string | undefined;
23981
+ AZURE_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
23982
+ AZURE_OPENAI_API_BASE_URL?: string | undefined;
23983
+ AZURE_OPENAI_API_HOST?: string | undefined;
23984
+ AZURE_OPENAI_API_KEY?: string | undefined;
23985
+ AZURE_OPENAI_BASE_URL?: string | undefined;
23986
+ AZURE_OPENAI_DEPLOYMENT_NAME?: string | undefined;
23987
+ AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
23988
+ AZURE_TENANT_ID?: string | undefined;
23989
+ AZURE_TOKEN_SCOPE?: string | undefined;
23990
+ BAM_API_HOST?: string | undefined;
23991
+ BAM_API_KEY?: string | undefined;
23992
+ CLOUDFLARE_ACCOUNT_ID?: string | undefined;
23993
+ CLOUDFLARE_API_KEY?: string | undefined;
23994
+ COHERE_API_KEY?: string | undefined;
23995
+ COHERE_CLIENT_NAME?: string | undefined;
23996
+ DATABRICKS_TOKEN?: string | undefined;
23997
+ DATABRICKS_WORKSPACE_URL?: string | undefined;
23998
+ FAL_KEY?: string | undefined;
23999
+ GOOGLE_API_HOST?: string | undefined;
24000
+ GOOGLE_API_KEY?: string | undefined;
24001
+ GROQ_API_KEY?: string | undefined;
24002
+ HELICONE_API_KEY?: string | undefined;
24003
+ HF_API_TOKEN?: string | undefined;
24004
+ HF_TOKEN?: string | undefined;
24005
+ HUGGING_FACE_HUB_TOKEN?: string | undefined;
24006
+ JFROG_API_KEY?: string | undefined;
24007
+ LAMBDA_API_KEY?: string | undefined;
24008
+ LANGFUSE_HOST?: string | undefined;
24009
+ LANGFUSE_PUBLIC_KEY?: string | undefined;
24010
+ LANGFUSE_SECRET_KEY?: string | undefined;
24011
+ LITELLM_API_BASE?: string | undefined;
24012
+ LLAMA_BASE_URL?: string | undefined;
24013
+ LOCALAI_BASE_URL?: string | undefined;
24014
+ MISTRAL_API_BASE_URL?: string | undefined;
24015
+ MISTRAL_API_HOST?: string | undefined;
24016
+ MISTRAL_API_KEY?: string | undefined;
24017
+ OLLAMA_API_KEY?: string | undefined;
24018
+ OLLAMA_BASE_URL?: string | undefined;
24019
+ OPENAI_API_BASE_URL?: string | undefined;
24020
+ OPENAI_API_HOST?: string | undefined;
24021
+ OPENAI_API_KEY?: string | undefined;
24022
+ OPENAI_BASE_URL?: string | undefined;
24023
+ OPENAI_ORGANIZATION?: string | undefined;
24024
+ PALM_API_HOST?: string | undefined;
24025
+ PALM_API_KEY?: string | undefined;
24026
+ PORTKEY_API_KEY?: string | undefined;
24027
+ PROMPTFOO_CA_CERT_PATH?: string | undefined;
24028
+ PROMPTFOO_INSECURE_SSL?: string | undefined;
24029
+ REPLICATE_API_KEY?: string | undefined;
24030
+ REPLICATE_API_TOKEN?: string | undefined;
24031
+ VERTEX_API_HOST?: string | undefined;
24032
+ VERTEX_API_KEY?: string | undefined;
24033
+ VERTEX_API_VERSION?: string | undefined;
24034
+ VERTEX_PROJECT_ID?: string | undefined;
24035
+ VERTEX_PUBLISHER?: string | undefined;
24036
+ VERTEX_REGION?: string | undefined;
24037
+ VOYAGE_API_BASE_URL?: string | undefined;
24038
+ VOYAGE_API_KEY?: string | undefined;
24039
+ WATSONX_AI_APIKEY?: string | undefined;
24040
+ WATSONX_AI_AUTH_TYPE?: string | undefined;
24041
+ WATSONX_AI_BEARER_TOKEN?: string | undefined;
24042
+ WATSONX_AI_PROJECT_ID?: string | undefined;
24043
+ AZURE_CONTENT_SAFETY_ENDPOINT?: string | undefined;
24044
+ AZURE_CONTENT_SAFETY_API_KEY?: string | undefined;
24045
+ AZURE_CONTENT_SAFETY_API_VERSION?: string | undefined;
24046
+ AWS_REGION?: string | undefined;
24047
+ AWS_DEFAULT_REGION?: string | undefined;
24048
+ AWS_SAGEMAKER_MAX_TOKENS?: string | undefined;
24049
+ AWS_SAGEMAKER_TEMPERATURE?: string | undefined;
24050
+ AWS_SAGEMAKER_TOP_P?: string | undefined;
24051
+ AWS_SAGEMAKER_MAX_RETRIES?: string | undefined;
24052
+ PROMPTFOO_EVAL_TIMEOUT_MS?: string | undefined;
24053
+ } | undefined;
24054
+ } | {
24055
+ callApi: import("./providers").CallApiFunction;
24056
+ id: (...args: unknown[]) => string;
24057
+ config?: any;
24058
+ label?: string | undefined;
24059
+ transform?: string | undefined;
24060
+ delay?: number | undefined;
24061
+ callEmbeddingApi?: ((args_0: string, ...args: unknown[]) => Promise<import("./providers").ProviderEmbeddingResponse>) | undefined;
24062
+ callClassificationApi?: ((args_0: string, ...args: unknown[]) => Promise<import("./providers").ProviderClassificationResponse>) | undefined;
24063
+ } | undefined;
24064
+ delay?: number | undefined;
23963
24065
  numTests?: number | undefined;
23964
- provider?: string | {
23965
- id?: string | undefined;
23966
- config?: any;
23967
- label?: string | undefined;
23968
- prompts?: string[] | undefined;
23969
- transform?: string | undefined;
23970
- delay?: number | undefined;
23971
- env?: {
23972
- AI21_API_BASE_URL?: string | undefined;
23973
- AI21_API_KEY?: string | undefined;
23974
- ANTHROPIC_API_KEY?: string | undefined;
23975
- ANTHROPIC_BASE_URL?: string | undefined;
23976
- AWS_BEDROCK_REGION?: string | undefined;
23977
- AZURE_API_BASE_URL?: string | undefined;
23978
- AZURE_API_HOST?: string | undefined;
23979
- AZURE_API_KEY?: string | undefined;
23980
- AZURE_AUTHORITY_HOST?: string | undefined;
23981
- AZURE_CLIENT_ID?: string | undefined;
23982
- AZURE_CLIENT_SECRET?: string | undefined;
23983
- AZURE_DEPLOYMENT_NAME?: string | undefined;
23984
- AZURE_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
23985
- AZURE_OPENAI_API_BASE_URL?: string | undefined;
23986
- AZURE_OPENAI_API_HOST?: string | undefined;
23987
- AZURE_OPENAI_API_KEY?: string | undefined;
23988
- AZURE_OPENAI_BASE_URL?: string | undefined;
23989
- AZURE_OPENAI_DEPLOYMENT_NAME?: string | undefined;
23990
- AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
23991
- AZURE_TENANT_ID?: string | undefined;
23992
- AZURE_TOKEN_SCOPE?: string | undefined;
23993
- BAM_API_HOST?: string | undefined;
23994
- BAM_API_KEY?: string | undefined;
23995
- CLOUDFLARE_ACCOUNT_ID?: string | undefined;
23996
- CLOUDFLARE_API_KEY?: string | undefined;
23997
- COHERE_API_KEY?: string | undefined;
23998
- COHERE_CLIENT_NAME?: string | undefined;
23999
- DATABRICKS_TOKEN?: string | undefined;
24000
- DATABRICKS_WORKSPACE_URL?: string | undefined;
24001
- FAL_KEY?: string | undefined;
24002
- GOOGLE_API_HOST?: string | undefined;
24003
- GOOGLE_API_KEY?: string | undefined;
24004
- GROQ_API_KEY?: string | undefined;
24005
- HELICONE_API_KEY?: string | undefined;
24006
- HF_API_TOKEN?: string | undefined;
24007
- HF_TOKEN?: string | undefined;
24008
- HUGGING_FACE_HUB_TOKEN?: string | undefined;
24009
- JFROG_API_KEY?: string | undefined;
24010
- LAMBDA_API_KEY?: string | undefined;
24011
- LANGFUSE_HOST?: string | undefined;
24012
- LANGFUSE_PUBLIC_KEY?: string | undefined;
24013
- LANGFUSE_SECRET_KEY?: string | undefined;
24014
- LITELLM_API_BASE?: string | undefined;
24015
- LLAMA_BASE_URL?: string | undefined;
24016
- LOCALAI_BASE_URL?: string | undefined;
24017
- MISTRAL_API_BASE_URL?: string | undefined;
24018
- MISTRAL_API_HOST?: string | undefined;
24019
- MISTRAL_API_KEY?: string | undefined;
24020
- OLLAMA_API_KEY?: string | undefined;
24021
- OLLAMA_BASE_URL?: string | undefined;
24022
- OPENAI_API_BASE_URL?: string | undefined;
24023
- OPENAI_API_HOST?: string | undefined;
24024
- OPENAI_API_KEY?: string | undefined;
24025
- OPENAI_BASE_URL?: string | undefined;
24026
- OPENAI_ORGANIZATION?: string | undefined;
24027
- PALM_API_HOST?: string | undefined;
24028
- PALM_API_KEY?: string | undefined;
24029
- PORTKEY_API_KEY?: string | undefined;
24030
- PROMPTFOO_CA_CERT_PATH?: string | undefined;
24031
- PROMPTFOO_INSECURE_SSL?: string | undefined;
24032
- REPLICATE_API_KEY?: string | undefined;
24033
- REPLICATE_API_TOKEN?: string | undefined;
24034
- VERTEX_API_HOST?: string | undefined;
24035
- VERTEX_API_KEY?: string | undefined;
24036
- VERTEX_API_VERSION?: string | undefined;
24037
- VERTEX_PROJECT_ID?: string | undefined;
24038
- VERTEX_PUBLISHER?: string | undefined;
24039
- VERTEX_REGION?: string | undefined;
24040
- VOYAGE_API_BASE_URL?: string | undefined;
24041
- VOYAGE_API_KEY?: string | undefined;
24042
- WATSONX_AI_APIKEY?: string | undefined;
24043
- WATSONX_AI_AUTH_TYPE?: string | undefined;
24044
- WATSONX_AI_BEARER_TOKEN?: string | undefined;
24045
- WATSONX_AI_PROJECT_ID?: string | undefined;
24046
- AZURE_CONTENT_SAFETY_ENDPOINT?: string | undefined;
24047
- AZURE_CONTENT_SAFETY_API_KEY?: string | undefined;
24048
- AZURE_CONTENT_SAFETY_API_VERSION?: string | undefined;
24049
- AWS_REGION?: string | undefined;
24050
- AWS_DEFAULT_REGION?: string | undefined;
24051
- AWS_SAGEMAKER_MAX_TOKENS?: string | undefined;
24052
- AWS_SAGEMAKER_TEMPERATURE?: string | undefined;
24053
- AWS_SAGEMAKER_TOP_P?: string | undefined;
24054
- AWS_SAGEMAKER_MAX_RETRIES?: string | undefined;
24055
- PROMPTFOO_EVAL_TIMEOUT_MS?: string | undefined;
24056
- } | undefined;
24057
- } | {
24058
- callApi: import("./providers").CallApiFunction;
24059
- id: (...args: unknown[]) => string;
24060
- config?: any;
24061
- label?: string | undefined;
24062
- transform?: string | undefined;
24063
- delay?: number | undefined;
24064
- callEmbeddingApi?: ((args_0: string, ...args: unknown[]) => Promise<import("./providers").ProviderEmbeddingResponse>) | undefined;
24065
- callClassificationApi?: ((args_0: string, ...args: unknown[]) => Promise<import("./providers").ProviderClassificationResponse>) | undefined;
24066
- } | undefined;
24067
- delay?: number | undefined;
24068
- injectVar?: string | undefined;
24069
- language?: string | undefined;
24070
- purpose?: string | undefined;
24071
- strategies?: (string | {
24072
- id: string;
24073
- config?: Record<string, unknown> | undefined;
24074
- })[] | undefined;
24075
- excludeTargetOutputFromAgenticAttackGeneration?: boolean | undefined;
24076
- }>>;
24077
- writeLatestResults: z.ZodOptional<z.ZodBoolean>;
24078
- }, "strip", z.ZodTypeAny, {
24079
- prompts: string | Record<string, string> | (string | {
24080
- raw: string;
24081
- label: string;
24082
- function?: ((args_0: {
24083
- vars: Record<string, any>;
24084
- provider?: ApiProvider | undefined;
24085
- }, ...args: unknown[]) => Promise<any>) | undefined;
24086
- id?: string | undefined;
24087
- config?: any;
24088
- display?: string | undefined;
24089
- } | {
24090
- id: string;
24091
- raw?: string | undefined;
24092
- label?: string | undefined;
24093
- })[];
24094
- providers: string | (((args_0: string, args_1: {
24095
- prompt: {
24096
- raw: string;
24097
- label: string;
24098
- function?: ((args_0: {
24099
- vars: Record<string, any>;
24100
- provider?: ApiProvider | undefined;
24101
- }, ...args: unknown[]) => Promise<any>) | undefined;
24102
- id?: string | undefined;
24103
- config?: any;
24104
- display?: string | undefined;
24105
- };
24106
- vars: Record<string, string | {}>;
24107
- fetchWithCache?: any;
24108
- filters?: Record<string, (args_0: any, ...args: any[]) => string> | undefined;
24109
- getCache?: any;
24110
- logger?: any;
24111
- originalProvider?: any;
24112
- } | undefined, args_2: {
24113
- includeLogProbs?: boolean | undefined;
24114
- } | undefined, ...args: unknown[]) => Promise<ProviderResponse>) & {
24115
- label?: string | undefined;
24116
- }) | (string | {
24117
- id?: string | undefined;
24118
- config?: any;
24119
- label?: string | undefined;
24120
- prompts?: string[] | undefined;
24121
- transform?: string | undefined;
24122
- delay?: number | undefined;
24123
- env?: {
24124
- AI21_API_BASE_URL?: string | undefined;
24125
- AI21_API_KEY?: string | undefined;
24126
- ANTHROPIC_API_KEY?: string | undefined;
24127
- ANTHROPIC_BASE_URL?: string | undefined;
24128
- AWS_BEDROCK_REGION?: string | undefined;
24129
- AZURE_API_BASE_URL?: string | undefined;
24130
- AZURE_API_HOST?: string | undefined;
24131
- AZURE_API_KEY?: string | undefined;
24132
- AZURE_AUTHORITY_HOST?: string | undefined;
24133
- AZURE_CLIENT_ID?: string | undefined;
24134
- AZURE_CLIENT_SECRET?: string | undefined;
24135
- AZURE_DEPLOYMENT_NAME?: string | undefined;
24136
- AZURE_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
24137
- AZURE_OPENAI_API_BASE_URL?: string | undefined;
24138
- AZURE_OPENAI_API_HOST?: string | undefined;
24139
- AZURE_OPENAI_API_KEY?: string | undefined;
24140
- AZURE_OPENAI_BASE_URL?: string | undefined;
24141
- AZURE_OPENAI_DEPLOYMENT_NAME?: string | undefined;
24142
- AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
24143
- AZURE_TENANT_ID?: string | undefined;
24144
- AZURE_TOKEN_SCOPE?: string | undefined;
24145
- BAM_API_HOST?: string | undefined;
24146
- BAM_API_KEY?: string | undefined;
24147
- CLOUDFLARE_ACCOUNT_ID?: string | undefined;
24148
- CLOUDFLARE_API_KEY?: string | undefined;
24149
- COHERE_API_KEY?: string | undefined;
24150
- COHERE_CLIENT_NAME?: string | undefined;
24151
- DATABRICKS_TOKEN?: string | undefined;
24152
- DATABRICKS_WORKSPACE_URL?: string | undefined;
24153
- FAL_KEY?: string | undefined;
24154
- GOOGLE_API_HOST?: string | undefined;
24155
- GOOGLE_API_KEY?: string | undefined;
24156
- GROQ_API_KEY?: string | undefined;
24157
- HELICONE_API_KEY?: string | undefined;
24158
- HF_API_TOKEN?: string | undefined;
24159
- HF_TOKEN?: string | undefined;
24160
- HUGGING_FACE_HUB_TOKEN?: string | undefined;
24161
- JFROG_API_KEY?: string | undefined;
24162
- LAMBDA_API_KEY?: string | undefined;
24163
- LANGFUSE_HOST?: string | undefined;
24164
- LANGFUSE_PUBLIC_KEY?: string | undefined;
24165
- LANGFUSE_SECRET_KEY?: string | undefined;
24166
- LITELLM_API_BASE?: string | undefined;
24167
- LLAMA_BASE_URL?: string | undefined;
24168
- LOCALAI_BASE_URL?: string | undefined;
24169
- MISTRAL_API_BASE_URL?: string | undefined;
24170
- MISTRAL_API_HOST?: string | undefined;
24171
- MISTRAL_API_KEY?: string | undefined;
24172
- OLLAMA_API_KEY?: string | undefined;
24173
- OLLAMA_BASE_URL?: string | undefined;
24174
- OPENAI_API_BASE_URL?: string | undefined;
24175
- OPENAI_API_HOST?: string | undefined;
24176
- OPENAI_API_KEY?: string | undefined;
24177
- OPENAI_BASE_URL?: string | undefined;
24178
- OPENAI_ORGANIZATION?: string | undefined;
24179
- PALM_API_HOST?: string | undefined;
24180
- PALM_API_KEY?: string | undefined;
24181
- PORTKEY_API_KEY?: string | undefined;
24182
- PROMPTFOO_CA_CERT_PATH?: string | undefined;
24183
- PROMPTFOO_INSECURE_SSL?: string | undefined;
24184
- REPLICATE_API_KEY?: string | undefined;
24185
- REPLICATE_API_TOKEN?: string | undefined;
24186
- VERTEX_API_HOST?: string | undefined;
24187
- VERTEX_API_KEY?: string | undefined;
24188
- VERTEX_API_VERSION?: string | undefined;
24189
- VERTEX_PROJECT_ID?: string | undefined;
24190
- VERTEX_PUBLISHER?: string | undefined;
24191
- VERTEX_REGION?: string | undefined;
24192
- VOYAGE_API_BASE_URL?: string | undefined;
24193
- VOYAGE_API_KEY?: string | undefined;
24194
- WATSONX_AI_APIKEY?: string | undefined;
24195
- WATSONX_AI_AUTH_TYPE?: string | undefined;
24196
- WATSONX_AI_BEARER_TOKEN?: string | undefined;
24197
- WATSONX_AI_PROJECT_ID?: string | undefined;
24198
- AZURE_CONTENT_SAFETY_ENDPOINT?: string | undefined;
24199
- AZURE_CONTENT_SAFETY_API_KEY?: string | undefined;
24200
- AZURE_CONTENT_SAFETY_API_VERSION?: string | undefined;
24201
- AWS_REGION?: string | undefined;
24202
- AWS_DEFAULT_REGION?: string | undefined;
24203
- AWS_SAGEMAKER_MAX_TOKENS?: string | undefined;
24204
- AWS_SAGEMAKER_TEMPERATURE?: string | undefined;
24205
- AWS_SAGEMAKER_TOP_P?: string | undefined;
24206
- AWS_SAGEMAKER_MAX_RETRIES?: string | undefined;
24207
- PROMPTFOO_EVAL_TIMEOUT_MS?: string | undefined;
24208
- } | undefined;
24209
- } | (((args_0: string, args_1: {
24210
- prompt: {
24211
- raw: string;
24212
- label: string;
24213
- function?: ((args_0: {
24214
- vars: Record<string, any>;
24215
- provider?: ApiProvider | undefined;
24216
- }, ...args: unknown[]) => Promise<any>) | undefined;
24217
- id?: string | undefined;
24218
- config?: any;
24219
- display?: string | undefined;
24220
- };
24221
- vars: Record<string, string | {}>;
24222
- fetchWithCache?: any;
24223
- filters?: Record<string, (args_0: any, ...args: any[]) => string> | undefined;
24224
- getCache?: any;
24225
- logger?: any;
24226
- originalProvider?: any;
24227
- } | undefined, args_2: {
24228
- includeLogProbs?: boolean | undefined;
24229
- } | undefined, ...args: unknown[]) => Promise<ProviderResponse>) & {
24230
- label?: string | undefined;
24231
- }) | Record<string, {
24232
- id?: string | undefined;
24233
- config?: any;
24234
- label?: string | undefined;
24235
- prompts?: string[] | undefined;
24236
- transform?: string | undefined;
24237
- delay?: number | undefined;
24238
- env?: {
24239
- AI21_API_BASE_URL?: string | undefined;
24240
- AI21_API_KEY?: string | undefined;
24241
- ANTHROPIC_API_KEY?: string | undefined;
24242
- ANTHROPIC_BASE_URL?: string | undefined;
24243
- AWS_BEDROCK_REGION?: string | undefined;
24244
- AZURE_API_BASE_URL?: string | undefined;
24245
- AZURE_API_HOST?: string | undefined;
24246
- AZURE_API_KEY?: string | undefined;
24247
- AZURE_AUTHORITY_HOST?: string | undefined;
24248
- AZURE_CLIENT_ID?: string | undefined;
24249
- AZURE_CLIENT_SECRET?: string | undefined;
24250
- AZURE_DEPLOYMENT_NAME?: string | undefined;
24251
- AZURE_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
24252
- AZURE_OPENAI_API_BASE_URL?: string | undefined;
24253
- AZURE_OPENAI_API_HOST?: string | undefined;
24254
- AZURE_OPENAI_API_KEY?: string | undefined;
24255
- AZURE_OPENAI_BASE_URL?: string | undefined;
24256
- AZURE_OPENAI_DEPLOYMENT_NAME?: string | undefined;
24257
- AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
24258
- AZURE_TENANT_ID?: string | undefined;
24259
- AZURE_TOKEN_SCOPE?: string | undefined;
24260
- BAM_API_HOST?: string | undefined;
24261
- BAM_API_KEY?: string | undefined;
24262
- CLOUDFLARE_ACCOUNT_ID?: string | undefined;
24263
- CLOUDFLARE_API_KEY?: string | undefined;
24264
- COHERE_API_KEY?: string | undefined;
24265
- COHERE_CLIENT_NAME?: string | undefined;
24266
- DATABRICKS_TOKEN?: string | undefined;
24267
- DATABRICKS_WORKSPACE_URL?: string | undefined;
24268
- FAL_KEY?: string | undefined;
24269
- GOOGLE_API_HOST?: string | undefined;
24270
- GOOGLE_API_KEY?: string | undefined;
24271
- GROQ_API_KEY?: string | undefined;
24272
- HELICONE_API_KEY?: string | undefined;
24273
- HF_API_TOKEN?: string | undefined;
24274
- HF_TOKEN?: string | undefined;
24275
- HUGGING_FACE_HUB_TOKEN?: string | undefined;
24276
- JFROG_API_KEY?: string | undefined;
24277
- LAMBDA_API_KEY?: string | undefined;
24278
- LANGFUSE_HOST?: string | undefined;
24279
- LANGFUSE_PUBLIC_KEY?: string | undefined;
24280
- LANGFUSE_SECRET_KEY?: string | undefined;
24281
- LITELLM_API_BASE?: string | undefined;
24282
- LLAMA_BASE_URL?: string | undefined;
24283
- LOCALAI_BASE_URL?: string | undefined;
24284
- MISTRAL_API_BASE_URL?: string | undefined;
24285
- MISTRAL_API_HOST?: string | undefined;
24286
- MISTRAL_API_KEY?: string | undefined;
24287
- OLLAMA_API_KEY?: string | undefined;
24288
- OLLAMA_BASE_URL?: string | undefined;
24289
- OPENAI_API_BASE_URL?: string | undefined;
24290
- OPENAI_API_HOST?: string | undefined;
24291
- OPENAI_API_KEY?: string | undefined;
24292
- OPENAI_BASE_URL?: string | undefined;
24293
- OPENAI_ORGANIZATION?: string | undefined;
24294
- PALM_API_HOST?: string | undefined;
24295
- PALM_API_KEY?: string | undefined;
24296
- PORTKEY_API_KEY?: string | undefined;
24297
- PROMPTFOO_CA_CERT_PATH?: string | undefined;
24298
- PROMPTFOO_INSECURE_SSL?: string | undefined;
24299
- REPLICATE_API_KEY?: string | undefined;
24300
- REPLICATE_API_TOKEN?: string | undefined;
24301
- VERTEX_API_HOST?: string | undefined;
24302
- VERTEX_API_KEY?: string | undefined;
24303
- VERTEX_API_VERSION?: string | undefined;
24304
- VERTEX_PROJECT_ID?: string | undefined;
24305
- VERTEX_PUBLISHER?: string | undefined;
24306
- VERTEX_REGION?: string | undefined;
24307
- VOYAGE_API_BASE_URL?: string | undefined;
24308
- VOYAGE_API_KEY?: string | undefined;
24309
- WATSONX_AI_APIKEY?: string | undefined;
24310
- WATSONX_AI_AUTH_TYPE?: string | undefined;
24311
- WATSONX_AI_BEARER_TOKEN?: string | undefined;
24312
- WATSONX_AI_PROJECT_ID?: string | undefined;
24313
- AZURE_CONTENT_SAFETY_ENDPOINT?: string | undefined;
24314
- AZURE_CONTENT_SAFETY_API_KEY?: string | undefined;
24315
- AZURE_CONTENT_SAFETY_API_VERSION?: string | undefined;
24316
- AWS_REGION?: string | undefined;
24317
- AWS_DEFAULT_REGION?: string | undefined;
24318
- AWS_SAGEMAKER_MAX_TOKENS?: string | undefined;
24319
- AWS_SAGEMAKER_TEMPERATURE?: string | undefined;
24320
- AWS_SAGEMAKER_TOP_P?: string | undefined;
24321
- AWS_SAGEMAKER_MAX_RETRIES?: string | undefined;
24322
- PROMPTFOO_EVAL_TIMEOUT_MS?: string | undefined;
24323
- } | undefined;
24324
- }>)[];
24325
- env?: {
24326
- AI21_API_BASE_URL?: string | undefined;
24327
- AI21_API_KEY?: string | undefined;
24328
- ANTHROPIC_API_KEY?: string | undefined;
24329
- ANTHROPIC_BASE_URL?: string | undefined;
24330
- AWS_BEDROCK_REGION?: string | undefined;
24331
- AZURE_API_BASE_URL?: string | undefined;
24332
- AZURE_API_HOST?: string | undefined;
24333
- AZURE_API_KEY?: string | undefined;
24334
- AZURE_AUTHORITY_HOST?: string | undefined;
24335
- AZURE_CLIENT_ID?: string | undefined;
24336
- AZURE_CLIENT_SECRET?: string | undefined;
24337
- AZURE_DEPLOYMENT_NAME?: string | undefined;
24338
- AZURE_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
24339
- AZURE_OPENAI_API_BASE_URL?: string | undefined;
24340
- AZURE_OPENAI_API_HOST?: string | undefined;
24341
- AZURE_OPENAI_API_KEY?: string | undefined;
24342
- AZURE_OPENAI_BASE_URL?: string | undefined;
24343
- AZURE_OPENAI_DEPLOYMENT_NAME?: string | undefined;
24344
- AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
24345
- AZURE_TENANT_ID?: string | undefined;
24346
- AZURE_TOKEN_SCOPE?: string | undefined;
24347
- BAM_API_HOST?: string | undefined;
24348
- BAM_API_KEY?: string | undefined;
24349
- CLOUDFLARE_ACCOUNT_ID?: string | undefined;
24350
- CLOUDFLARE_API_KEY?: string | undefined;
24351
- COHERE_API_KEY?: string | undefined;
24352
- COHERE_CLIENT_NAME?: string | undefined;
24353
- DATABRICKS_TOKEN?: string | undefined;
24354
- DATABRICKS_WORKSPACE_URL?: string | undefined;
24355
- FAL_KEY?: string | undefined;
24356
- GOOGLE_API_HOST?: string | undefined;
24357
- GOOGLE_API_KEY?: string | undefined;
24358
- GROQ_API_KEY?: string | undefined;
24359
- HELICONE_API_KEY?: string | undefined;
24360
- HF_API_TOKEN?: string | undefined;
24361
- HF_TOKEN?: string | undefined;
24362
- HUGGING_FACE_HUB_TOKEN?: string | undefined;
24363
- JFROG_API_KEY?: string | undefined;
24364
- LAMBDA_API_KEY?: string | undefined;
24365
- LANGFUSE_HOST?: string | undefined;
24366
- LANGFUSE_PUBLIC_KEY?: string | undefined;
24367
- LANGFUSE_SECRET_KEY?: string | undefined;
24368
- LITELLM_API_BASE?: string | undefined;
24369
- LLAMA_BASE_URL?: string | undefined;
24370
- LOCALAI_BASE_URL?: string | undefined;
24371
- MISTRAL_API_BASE_URL?: string | undefined;
24372
- MISTRAL_API_HOST?: string | undefined;
24373
- MISTRAL_API_KEY?: string | undefined;
24374
- OLLAMA_API_KEY?: string | undefined;
24375
- OLLAMA_BASE_URL?: string | undefined;
24376
- OPENAI_API_BASE_URL?: string | undefined;
24377
- OPENAI_API_HOST?: string | undefined;
24378
- OPENAI_API_KEY?: string | undefined;
24379
- OPENAI_BASE_URL?: string | undefined;
24380
- OPENAI_ORGANIZATION?: string | undefined;
24381
- PALM_API_HOST?: string | undefined;
24382
- PALM_API_KEY?: string | undefined;
24383
- PORTKEY_API_KEY?: string | undefined;
24384
- PROMPTFOO_CA_CERT_PATH?: string | undefined;
24385
- PROMPTFOO_INSECURE_SSL?: string | undefined;
24386
- REPLICATE_API_KEY?: string | undefined;
24387
- REPLICATE_API_TOKEN?: string | undefined;
24388
- VERTEX_API_HOST?: string | undefined;
24389
- VERTEX_API_KEY?: string | undefined;
24390
- VERTEX_API_VERSION?: string | undefined;
24391
- VERTEX_PROJECT_ID?: string | undefined;
24392
- VERTEX_PUBLISHER?: string | undefined;
24393
- VERTEX_REGION?: string | undefined;
24394
- VOYAGE_API_BASE_URL?: string | undefined;
24395
- VOYAGE_API_KEY?: string | undefined;
24396
- WATSONX_AI_APIKEY?: string | undefined;
24397
- WATSONX_AI_AUTH_TYPE?: string | undefined;
24398
- WATSONX_AI_BEARER_TOKEN?: string | undefined;
24399
- WATSONX_AI_PROJECT_ID?: string | undefined;
24400
- AZURE_CONTENT_SAFETY_ENDPOINT?: string | undefined;
24401
- AZURE_CONTENT_SAFETY_API_KEY?: string | undefined;
24402
- AZURE_CONTENT_SAFETY_API_VERSION?: string | undefined;
24403
- AWS_REGION?: string | undefined;
24404
- AWS_DEFAULT_REGION?: string | undefined;
24405
- AWS_SAGEMAKER_MAX_TOKENS?: string | undefined;
24406
- AWS_SAGEMAKER_TEMPERATURE?: string | undefined;
24407
- AWS_SAGEMAKER_TOP_P?: string | undefined;
24408
- AWS_SAGEMAKER_MAX_RETRIES?: string | undefined;
24409
- PROMPTFOO_EVAL_TIMEOUT_MS?: string | undefined;
24410
- } | Record<string, string> | undefined;
24411
- metadata?: Record<string, any> | undefined;
24412
- description?: string | undefined;
24413
- tests?: string | (string | {
24414
- options?: ({
24415
- prefix?: string | undefined;
24416
- suffix?: string | undefined;
24417
- } & {
24418
- transform?: string | undefined;
24419
- postprocess?: string | undefined;
24420
- transformVars?: string | undefined;
24421
- storeOutputAs?: string | undefined;
24422
- } & {
24423
- provider?: any;
24424
- rubricPrompt?: string | string[] | {
24425
- role: string;
24426
- content: string;
24427
- }[] | undefined;
24428
- factuality?: {
24429
- subset?: number | undefined;
24430
- superset?: number | undefined;
24431
- agree?: number | undefined;
24432
- disagree?: number | undefined;
24433
- differButFactual?: number | undefined;
24434
- } | undefined;
24435
- } & {
24436
- disableVarExpansion?: boolean | undefined;
24437
- disableConversationVar?: boolean | undefined;
24438
- runSerially?: boolean | undefined;
24439
- }) | undefined;
24440
- vars?: Record<string, string | number | boolean | any[] | Record<string, any> | (string | number | boolean)[]> | undefined;
24066
+ injectVar?: string | undefined;
24067
+ language?: string | undefined;
24068
+ purpose?: string | undefined;
24069
+ strategies?: (string | {
24070
+ id: string;
24071
+ config?: Record<string, unknown> | undefined;
24072
+ })[] | undefined;
24073
+ excludeTargetOutputFromAgenticAttackGeneration?: boolean | undefined;
24074
+ }>>;
24075
+ writeLatestResults: z.ZodOptional<z.ZodBoolean>;
24076
+ }, "strip", z.ZodTypeAny, {
24077
+ prompts: string | Record<string, string> | (string | {
24078
+ raw: string;
24079
+ label: string;
24080
+ function?: ((args_0: {
24081
+ vars: Record<string, any>;
24082
+ provider?: ApiProvider | undefined;
24083
+ }, ...args: unknown[]) => Promise<any>) | undefined;
24084
+ id?: string | undefined;
24085
+ config?: any;
24086
+ display?: string | undefined;
24087
+ } | {
24088
+ id: string;
24089
+ raw?: string | undefined;
24090
+ label?: string | undefined;
24091
+ })[];
24092
+ providers: string | (((args_0: string, args_1: {
24093
+ prompt: {
24094
+ raw: string;
24095
+ label: string;
24096
+ function?: ((args_0: {
24097
+ vars: Record<string, any>;
24098
+ provider?: ApiProvider | undefined;
24099
+ }, ...args: unknown[]) => Promise<any>) | undefined;
24100
+ id?: string | undefined;
24101
+ config?: any;
24102
+ display?: string | undefined;
24103
+ };
24104
+ vars: Record<string, string | {}>;
24105
+ fetchWithCache?: any;
24106
+ filters?: Record<string, (args_0: any, ...args: any[]) => string> | undefined;
24107
+ getCache?: any;
24108
+ logger?: any;
24109
+ originalProvider?: any;
24110
+ } | undefined, args_2: {
24111
+ includeLogProbs?: boolean | undefined;
24112
+ } | undefined, ...args: unknown[]) => Promise<ProviderResponse>) & {
24113
+ label?: string | undefined;
24114
+ }) | (string | {
24115
+ id?: string | undefined;
24116
+ config?: any;
24117
+ label?: string | undefined;
24118
+ prompts?: string[] | undefined;
24119
+ transform?: string | undefined;
24120
+ delay?: number | undefined;
24121
+ env?: {
24122
+ AI21_API_BASE_URL?: string | undefined;
24123
+ AI21_API_KEY?: string | undefined;
24124
+ ANTHROPIC_API_KEY?: string | undefined;
24125
+ ANTHROPIC_BASE_URL?: string | undefined;
24126
+ AWS_BEDROCK_REGION?: string | undefined;
24127
+ AZURE_API_BASE_URL?: string | undefined;
24128
+ AZURE_API_HOST?: string | undefined;
24129
+ AZURE_API_KEY?: string | undefined;
24130
+ AZURE_AUTHORITY_HOST?: string | undefined;
24131
+ AZURE_CLIENT_ID?: string | undefined;
24132
+ AZURE_CLIENT_SECRET?: string | undefined;
24133
+ AZURE_DEPLOYMENT_NAME?: string | undefined;
24134
+ AZURE_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
24135
+ AZURE_OPENAI_API_BASE_URL?: string | undefined;
24136
+ AZURE_OPENAI_API_HOST?: string | undefined;
24137
+ AZURE_OPENAI_API_KEY?: string | undefined;
24138
+ AZURE_OPENAI_BASE_URL?: string | undefined;
24139
+ AZURE_OPENAI_DEPLOYMENT_NAME?: string | undefined;
24140
+ AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
24141
+ AZURE_TENANT_ID?: string | undefined;
24142
+ AZURE_TOKEN_SCOPE?: string | undefined;
24143
+ BAM_API_HOST?: string | undefined;
24144
+ BAM_API_KEY?: string | undefined;
24145
+ CLOUDFLARE_ACCOUNT_ID?: string | undefined;
24146
+ CLOUDFLARE_API_KEY?: string | undefined;
24147
+ COHERE_API_KEY?: string | undefined;
24148
+ COHERE_CLIENT_NAME?: string | undefined;
24149
+ DATABRICKS_TOKEN?: string | undefined;
24150
+ DATABRICKS_WORKSPACE_URL?: string | undefined;
24151
+ FAL_KEY?: string | undefined;
24152
+ GOOGLE_API_HOST?: string | undefined;
24153
+ GOOGLE_API_KEY?: string | undefined;
24154
+ GROQ_API_KEY?: string | undefined;
24155
+ HELICONE_API_KEY?: string | undefined;
24156
+ HF_API_TOKEN?: string | undefined;
24157
+ HF_TOKEN?: string | undefined;
24158
+ HUGGING_FACE_HUB_TOKEN?: string | undefined;
24159
+ JFROG_API_KEY?: string | undefined;
24160
+ LAMBDA_API_KEY?: string | undefined;
24161
+ LANGFUSE_HOST?: string | undefined;
24162
+ LANGFUSE_PUBLIC_KEY?: string | undefined;
24163
+ LANGFUSE_SECRET_KEY?: string | undefined;
24164
+ LITELLM_API_BASE?: string | undefined;
24165
+ LLAMA_BASE_URL?: string | undefined;
24166
+ LOCALAI_BASE_URL?: string | undefined;
24167
+ MISTRAL_API_BASE_URL?: string | undefined;
24168
+ MISTRAL_API_HOST?: string | undefined;
24169
+ MISTRAL_API_KEY?: string | undefined;
24170
+ OLLAMA_API_KEY?: string | undefined;
24171
+ OLLAMA_BASE_URL?: string | undefined;
24172
+ OPENAI_API_BASE_URL?: string | undefined;
24173
+ OPENAI_API_HOST?: string | undefined;
24174
+ OPENAI_API_KEY?: string | undefined;
24175
+ OPENAI_BASE_URL?: string | undefined;
24176
+ OPENAI_ORGANIZATION?: string | undefined;
24177
+ PALM_API_HOST?: string | undefined;
24178
+ PALM_API_KEY?: string | undefined;
24179
+ PORTKEY_API_KEY?: string | undefined;
24180
+ PROMPTFOO_CA_CERT_PATH?: string | undefined;
24181
+ PROMPTFOO_INSECURE_SSL?: string | undefined;
24182
+ REPLICATE_API_KEY?: string | undefined;
24183
+ REPLICATE_API_TOKEN?: string | undefined;
24184
+ VERTEX_API_HOST?: string | undefined;
24185
+ VERTEX_API_KEY?: string | undefined;
24186
+ VERTEX_API_VERSION?: string | undefined;
24187
+ VERTEX_PROJECT_ID?: string | undefined;
24188
+ VERTEX_PUBLISHER?: string | undefined;
24189
+ VERTEX_REGION?: string | undefined;
24190
+ VOYAGE_API_BASE_URL?: string | undefined;
24191
+ VOYAGE_API_KEY?: string | undefined;
24192
+ WATSONX_AI_APIKEY?: string | undefined;
24193
+ WATSONX_AI_AUTH_TYPE?: string | undefined;
24194
+ WATSONX_AI_BEARER_TOKEN?: string | undefined;
24195
+ WATSONX_AI_PROJECT_ID?: string | undefined;
24196
+ AZURE_CONTENT_SAFETY_ENDPOINT?: string | undefined;
24197
+ AZURE_CONTENT_SAFETY_API_KEY?: string | undefined;
24198
+ AZURE_CONTENT_SAFETY_API_VERSION?: string | undefined;
24199
+ AWS_REGION?: string | undefined;
24200
+ AWS_DEFAULT_REGION?: string | undefined;
24201
+ AWS_SAGEMAKER_MAX_TOKENS?: string | undefined;
24202
+ AWS_SAGEMAKER_TEMPERATURE?: string | undefined;
24203
+ AWS_SAGEMAKER_TOP_P?: string | undefined;
24204
+ AWS_SAGEMAKER_MAX_RETRIES?: string | undefined;
24205
+ PROMPTFOO_EVAL_TIMEOUT_MS?: string | undefined;
24206
+ } | undefined;
24207
+ } | (((args_0: string, args_1: {
24208
+ prompt: {
24209
+ raw: string;
24210
+ label: string;
24211
+ function?: ((args_0: {
24212
+ vars: Record<string, any>;
24213
+ provider?: ApiProvider | undefined;
24214
+ }, ...args: unknown[]) => Promise<any>) | undefined;
24215
+ id?: string | undefined;
24216
+ config?: any;
24217
+ display?: string | undefined;
24218
+ };
24219
+ vars: Record<string, string | {}>;
24220
+ fetchWithCache?: any;
24221
+ filters?: Record<string, (args_0: any, ...args: any[]) => string> | undefined;
24222
+ getCache?: any;
24223
+ logger?: any;
24224
+ originalProvider?: any;
24225
+ } | undefined, args_2: {
24226
+ includeLogProbs?: boolean | undefined;
24227
+ } | undefined, ...args: unknown[]) => Promise<ProviderResponse>) & {
24228
+ label?: string | undefined;
24229
+ }) | Record<string, {
24230
+ id?: string | undefined;
24231
+ config?: any;
24232
+ label?: string | undefined;
24233
+ prompts?: string[] | undefined;
24234
+ transform?: string | undefined;
24235
+ delay?: number | undefined;
24236
+ env?: {
24237
+ AI21_API_BASE_URL?: string | undefined;
24238
+ AI21_API_KEY?: string | undefined;
24239
+ ANTHROPIC_API_KEY?: string | undefined;
24240
+ ANTHROPIC_BASE_URL?: string | undefined;
24241
+ AWS_BEDROCK_REGION?: string | undefined;
24242
+ AZURE_API_BASE_URL?: string | undefined;
24243
+ AZURE_API_HOST?: string | undefined;
24244
+ AZURE_API_KEY?: string | undefined;
24245
+ AZURE_AUTHORITY_HOST?: string | undefined;
24246
+ AZURE_CLIENT_ID?: string | undefined;
24247
+ AZURE_CLIENT_SECRET?: string | undefined;
24248
+ AZURE_DEPLOYMENT_NAME?: string | undefined;
24249
+ AZURE_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
24250
+ AZURE_OPENAI_API_BASE_URL?: string | undefined;
24251
+ AZURE_OPENAI_API_HOST?: string | undefined;
24252
+ AZURE_OPENAI_API_KEY?: string | undefined;
24253
+ AZURE_OPENAI_BASE_URL?: string | undefined;
24254
+ AZURE_OPENAI_DEPLOYMENT_NAME?: string | undefined;
24255
+ AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
24256
+ AZURE_TENANT_ID?: string | undefined;
24257
+ AZURE_TOKEN_SCOPE?: string | undefined;
24258
+ BAM_API_HOST?: string | undefined;
24259
+ BAM_API_KEY?: string | undefined;
24260
+ CLOUDFLARE_ACCOUNT_ID?: string | undefined;
24261
+ CLOUDFLARE_API_KEY?: string | undefined;
24262
+ COHERE_API_KEY?: string | undefined;
24263
+ COHERE_CLIENT_NAME?: string | undefined;
24264
+ DATABRICKS_TOKEN?: string | undefined;
24265
+ DATABRICKS_WORKSPACE_URL?: string | undefined;
24266
+ FAL_KEY?: string | undefined;
24267
+ GOOGLE_API_HOST?: string | undefined;
24268
+ GOOGLE_API_KEY?: string | undefined;
24269
+ GROQ_API_KEY?: string | undefined;
24270
+ HELICONE_API_KEY?: string | undefined;
24271
+ HF_API_TOKEN?: string | undefined;
24272
+ HF_TOKEN?: string | undefined;
24273
+ HUGGING_FACE_HUB_TOKEN?: string | undefined;
24274
+ JFROG_API_KEY?: string | undefined;
24275
+ LAMBDA_API_KEY?: string | undefined;
24276
+ LANGFUSE_HOST?: string | undefined;
24277
+ LANGFUSE_PUBLIC_KEY?: string | undefined;
24278
+ LANGFUSE_SECRET_KEY?: string | undefined;
24279
+ LITELLM_API_BASE?: string | undefined;
24280
+ LLAMA_BASE_URL?: string | undefined;
24281
+ LOCALAI_BASE_URL?: string | undefined;
24282
+ MISTRAL_API_BASE_URL?: string | undefined;
24283
+ MISTRAL_API_HOST?: string | undefined;
24284
+ MISTRAL_API_KEY?: string | undefined;
24285
+ OLLAMA_API_KEY?: string | undefined;
24286
+ OLLAMA_BASE_URL?: string | undefined;
24287
+ OPENAI_API_BASE_URL?: string | undefined;
24288
+ OPENAI_API_HOST?: string | undefined;
24289
+ OPENAI_API_KEY?: string | undefined;
24290
+ OPENAI_BASE_URL?: string | undefined;
24291
+ OPENAI_ORGANIZATION?: string | undefined;
24292
+ PALM_API_HOST?: string | undefined;
24293
+ PALM_API_KEY?: string | undefined;
24294
+ PORTKEY_API_KEY?: string | undefined;
24295
+ PROMPTFOO_CA_CERT_PATH?: string | undefined;
24296
+ PROMPTFOO_INSECURE_SSL?: string | undefined;
24297
+ REPLICATE_API_KEY?: string | undefined;
24298
+ REPLICATE_API_TOKEN?: string | undefined;
24299
+ VERTEX_API_HOST?: string | undefined;
24300
+ VERTEX_API_KEY?: string | undefined;
24301
+ VERTEX_API_VERSION?: string | undefined;
24302
+ VERTEX_PROJECT_ID?: string | undefined;
24303
+ VERTEX_PUBLISHER?: string | undefined;
24304
+ VERTEX_REGION?: string | undefined;
24305
+ VOYAGE_API_BASE_URL?: string | undefined;
24306
+ VOYAGE_API_KEY?: string | undefined;
24307
+ WATSONX_AI_APIKEY?: string | undefined;
24308
+ WATSONX_AI_AUTH_TYPE?: string | undefined;
24309
+ WATSONX_AI_BEARER_TOKEN?: string | undefined;
24310
+ WATSONX_AI_PROJECT_ID?: string | undefined;
24311
+ AZURE_CONTENT_SAFETY_ENDPOINT?: string | undefined;
24312
+ AZURE_CONTENT_SAFETY_API_KEY?: string | undefined;
24313
+ AZURE_CONTENT_SAFETY_API_VERSION?: string | undefined;
24314
+ AWS_REGION?: string | undefined;
24315
+ AWS_DEFAULT_REGION?: string | undefined;
24316
+ AWS_SAGEMAKER_MAX_TOKENS?: string | undefined;
24317
+ AWS_SAGEMAKER_TEMPERATURE?: string | undefined;
24318
+ AWS_SAGEMAKER_TOP_P?: string | undefined;
24319
+ AWS_SAGEMAKER_MAX_RETRIES?: string | undefined;
24320
+ PROMPTFOO_EVAL_TIMEOUT_MS?: string | undefined;
24321
+ } | undefined;
24322
+ }>)[];
24323
+ env?: {
24324
+ AI21_API_BASE_URL?: string | undefined;
24325
+ AI21_API_KEY?: string | undefined;
24326
+ ANTHROPIC_API_KEY?: string | undefined;
24327
+ ANTHROPIC_BASE_URL?: string | undefined;
24328
+ AWS_BEDROCK_REGION?: string | undefined;
24329
+ AZURE_API_BASE_URL?: string | undefined;
24330
+ AZURE_API_HOST?: string | undefined;
24331
+ AZURE_API_KEY?: string | undefined;
24332
+ AZURE_AUTHORITY_HOST?: string | undefined;
24333
+ AZURE_CLIENT_ID?: string | undefined;
24334
+ AZURE_CLIENT_SECRET?: string | undefined;
24335
+ AZURE_DEPLOYMENT_NAME?: string | undefined;
24336
+ AZURE_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
24337
+ AZURE_OPENAI_API_BASE_URL?: string | undefined;
24338
+ AZURE_OPENAI_API_HOST?: string | undefined;
24339
+ AZURE_OPENAI_API_KEY?: string | undefined;
24340
+ AZURE_OPENAI_BASE_URL?: string | undefined;
24341
+ AZURE_OPENAI_DEPLOYMENT_NAME?: string | undefined;
24342
+ AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
24343
+ AZURE_TENANT_ID?: string | undefined;
24344
+ AZURE_TOKEN_SCOPE?: string | undefined;
24345
+ BAM_API_HOST?: string | undefined;
24346
+ BAM_API_KEY?: string | undefined;
24347
+ CLOUDFLARE_ACCOUNT_ID?: string | undefined;
24348
+ CLOUDFLARE_API_KEY?: string | undefined;
24349
+ COHERE_API_KEY?: string | undefined;
24350
+ COHERE_CLIENT_NAME?: string | undefined;
24351
+ DATABRICKS_TOKEN?: string | undefined;
24352
+ DATABRICKS_WORKSPACE_URL?: string | undefined;
24353
+ FAL_KEY?: string | undefined;
24354
+ GOOGLE_API_HOST?: string | undefined;
24355
+ GOOGLE_API_KEY?: string | undefined;
24356
+ GROQ_API_KEY?: string | undefined;
24357
+ HELICONE_API_KEY?: string | undefined;
24358
+ HF_API_TOKEN?: string | undefined;
24359
+ HF_TOKEN?: string | undefined;
24360
+ HUGGING_FACE_HUB_TOKEN?: string | undefined;
24361
+ JFROG_API_KEY?: string | undefined;
24362
+ LAMBDA_API_KEY?: string | undefined;
24363
+ LANGFUSE_HOST?: string | undefined;
24364
+ LANGFUSE_PUBLIC_KEY?: string | undefined;
24365
+ LANGFUSE_SECRET_KEY?: string | undefined;
24366
+ LITELLM_API_BASE?: string | undefined;
24367
+ LLAMA_BASE_URL?: string | undefined;
24368
+ LOCALAI_BASE_URL?: string | undefined;
24369
+ MISTRAL_API_BASE_URL?: string | undefined;
24370
+ MISTRAL_API_HOST?: string | undefined;
24371
+ MISTRAL_API_KEY?: string | undefined;
24372
+ OLLAMA_API_KEY?: string | undefined;
24373
+ OLLAMA_BASE_URL?: string | undefined;
24374
+ OPENAI_API_BASE_URL?: string | undefined;
24375
+ OPENAI_API_HOST?: string | undefined;
24376
+ OPENAI_API_KEY?: string | undefined;
24377
+ OPENAI_BASE_URL?: string | undefined;
24378
+ OPENAI_ORGANIZATION?: string | undefined;
24379
+ PALM_API_HOST?: string | undefined;
24380
+ PALM_API_KEY?: string | undefined;
24381
+ PORTKEY_API_KEY?: string | undefined;
24382
+ PROMPTFOO_CA_CERT_PATH?: string | undefined;
24383
+ PROMPTFOO_INSECURE_SSL?: string | undefined;
24384
+ REPLICATE_API_KEY?: string | undefined;
24385
+ REPLICATE_API_TOKEN?: string | undefined;
24386
+ VERTEX_API_HOST?: string | undefined;
24387
+ VERTEX_API_KEY?: string | undefined;
24388
+ VERTEX_API_VERSION?: string | undefined;
24389
+ VERTEX_PROJECT_ID?: string | undefined;
24390
+ VERTEX_PUBLISHER?: string | undefined;
24391
+ VERTEX_REGION?: string | undefined;
24392
+ VOYAGE_API_BASE_URL?: string | undefined;
24393
+ VOYAGE_API_KEY?: string | undefined;
24394
+ WATSONX_AI_APIKEY?: string | undefined;
24395
+ WATSONX_AI_AUTH_TYPE?: string | undefined;
24396
+ WATSONX_AI_BEARER_TOKEN?: string | undefined;
24397
+ WATSONX_AI_PROJECT_ID?: string | undefined;
24398
+ AZURE_CONTENT_SAFETY_ENDPOINT?: string | undefined;
24399
+ AZURE_CONTENT_SAFETY_API_KEY?: string | undefined;
24400
+ AZURE_CONTENT_SAFETY_API_VERSION?: string | undefined;
24401
+ AWS_REGION?: string | undefined;
24402
+ AWS_DEFAULT_REGION?: string | undefined;
24403
+ AWS_SAGEMAKER_MAX_TOKENS?: string | undefined;
24404
+ AWS_SAGEMAKER_TEMPERATURE?: string | undefined;
24405
+ AWS_SAGEMAKER_TOP_P?: string | undefined;
24406
+ AWS_SAGEMAKER_MAX_RETRIES?: string | undefined;
24407
+ PROMPTFOO_EVAL_TIMEOUT_MS?: string | undefined;
24408
+ } | Record<string, string> | undefined;
24409
+ metadata?: Record<string, any> | undefined;
24410
+ description?: string | undefined;
24411
+ tests?: string | (string | {
24412
+ options?: ({
24413
+ prefix?: string | undefined;
24414
+ suffix?: string | undefined;
24415
+ } & {
24416
+ transform?: string | undefined;
24417
+ postprocess?: string | undefined;
24418
+ transformVars?: string | undefined;
24419
+ storeOutputAs?: string | undefined;
24420
+ } & {
24421
+ provider?: any;
24422
+ rubricPrompt?: string | string[] | {
24423
+ role: string;
24424
+ content: string;
24425
+ }[] | undefined;
24426
+ factuality?: {
24427
+ subset?: number | undefined;
24428
+ superset?: number | undefined;
24429
+ agree?: number | undefined;
24430
+ disagree?: number | undefined;
24431
+ differButFactual?: number | undefined;
24432
+ } | undefined;
24433
+ } & {
24434
+ disableVarExpansion?: boolean | undefined;
24435
+ disableConversationVar?: boolean | undefined;
24436
+ runSerially?: boolean | undefined;
24437
+ }) | undefined;
24438
+ vars?: Record<string, string | number | boolean | any[] | Record<string, any> | (string | number | boolean)[]> | undefined;
24441
24439
  provider?: string | {
24442
24440
  id?: string | undefined;
24443
24441
  config?: any;
@@ -25639,7 +25637,6 @@ export declare const TestSuiteConfigSchema: z.ZodObject<{
25639
25637
  })[] | undefined;
25640
25638
  maxConcurrency?: number | undefined;
25641
25639
  entities?: string[] | undefined;
25642
- numTests?: number | undefined;
25643
25640
  provider?: string | {
25644
25641
  id?: string | undefined;
25645
25642
  config?: any;
@@ -25744,6 +25741,7 @@ export declare const TestSuiteConfigSchema: z.ZodObject<{
25744
25741
  callClassificationApi?: ((args_0: string, ...args: unknown[]) => Promise<import("./providers").ProviderClassificationResponse>) | undefined;
25745
25742
  } | undefined;
25746
25743
  delay?: number | undefined;
25744
+ numTests?: number | undefined;
25747
25745
  injectVar?: string | undefined;
25748
25746
  language?: string | undefined;
25749
25747
  purpose?: string | undefined;
@@ -25831,1553 +25829,470 @@ export declare const TestSuiteConfigSchema: z.ZodObject<{
25831
25829
  JFROG_API_KEY?: string | undefined;
25832
25830
  LAMBDA_API_KEY?: string | undefined;
25833
25831
  LANGFUSE_HOST?: string | undefined;
25834
- LANGFUSE_PUBLIC_KEY?: string | undefined;
25835
- LANGFUSE_SECRET_KEY?: string | undefined;
25836
- LITELLM_API_BASE?: string | undefined;
25837
- LLAMA_BASE_URL?: string | undefined;
25838
- LOCALAI_BASE_URL?: string | undefined;
25839
- MISTRAL_API_BASE_URL?: string | undefined;
25840
- MISTRAL_API_HOST?: string | undefined;
25841
- MISTRAL_API_KEY?: string | undefined;
25842
- OLLAMA_API_KEY?: string | undefined;
25843
- OLLAMA_BASE_URL?: string | undefined;
25844
- OPENAI_API_BASE_URL?: string | undefined;
25845
- OPENAI_API_HOST?: string | undefined;
25846
- OPENAI_API_KEY?: string | undefined;
25847
- OPENAI_BASE_URL?: string | undefined;
25848
- OPENAI_ORGANIZATION?: string | undefined;
25849
- PALM_API_HOST?: string | undefined;
25850
- PALM_API_KEY?: string | undefined;
25851
- PORTKEY_API_KEY?: string | undefined;
25852
- PROMPTFOO_CA_CERT_PATH?: string | undefined;
25853
- PROMPTFOO_INSECURE_SSL?: string | undefined;
25854
- REPLICATE_API_KEY?: string | undefined;
25855
- REPLICATE_API_TOKEN?: string | undefined;
25856
- VERTEX_API_HOST?: string | undefined;
25857
- VERTEX_API_KEY?: string | undefined;
25858
- VERTEX_API_VERSION?: string | undefined;
25859
- VERTEX_PROJECT_ID?: string | undefined;
25860
- VERTEX_PUBLISHER?: string | undefined;
25861
- VERTEX_REGION?: string | undefined;
25862
- VOYAGE_API_BASE_URL?: string | undefined;
25863
- VOYAGE_API_KEY?: string | undefined;
25864
- WATSONX_AI_APIKEY?: string | undefined;
25865
- WATSONX_AI_AUTH_TYPE?: string | undefined;
25866
- WATSONX_AI_BEARER_TOKEN?: string | undefined;
25867
- WATSONX_AI_PROJECT_ID?: string | undefined;
25868
- AZURE_CONTENT_SAFETY_ENDPOINT?: string | undefined;
25869
- AZURE_CONTENT_SAFETY_API_KEY?: string | undefined;
25870
- AZURE_CONTENT_SAFETY_API_VERSION?: string | undefined;
25871
- AWS_REGION?: string | undefined;
25872
- AWS_DEFAULT_REGION?: string | undefined;
25873
- AWS_SAGEMAKER_MAX_TOKENS?: string | undefined;
25874
- AWS_SAGEMAKER_TEMPERATURE?: string | undefined;
25875
- AWS_SAGEMAKER_TOP_P?: string | undefined;
25876
- AWS_SAGEMAKER_MAX_RETRIES?: string | undefined;
25877
- PROMPTFOO_EVAL_TIMEOUT_MS?: string | undefined;
25878
- } | undefined;
25879
- } | {
25880
- callApi: import("./providers").CallApiFunction;
25881
- id: (...args: unknown[]) => string;
25882
- config?: any;
25883
- label?: string | undefined;
25884
- transform?: string | undefined;
25885
- delay?: number | undefined;
25886
- callEmbeddingApi?: ((args_0: string, ...args: unknown[]) => Promise<import("./providers").ProviderEmbeddingResponse>) | undefined;
25887
- callClassificationApi?: ((args_0: string, ...args: unknown[]) => Promise<import("./providers").ProviderClassificationResponse>) | undefined;
25888
- } | undefined;
25889
- metadata?: (Record<string, any> & {
25890
- pluginConfig?: PluginConfig | undefined;
25891
- strategyConfig?: import("../redteam/types").RedteamObjectConfig | undefined;
25892
- }) | undefined;
25893
- description?: string | undefined;
25894
- providerOutput?: string | {} | undefined;
25895
- assert?: ({
25896
- type: "moderation" | `promptfoo:redteam:${string}` | "cost" | "factuality" | "answer-relevance" | "bleu" | "classifier" | "contains" | "contains-all" | "contains-any" | "contains-json" | "contains-sql" | "contains-xml" | "context-faithfulness" | "context-recall" | "context-relevance" | "equals" | "g-eval" | "gleu" | "guardrails" | "icontains" | "icontains-all" | "icontains-any" | "is-json" | "is-refusal" | "is-sql" | "is-valid-function-call" | "is-valid-openai-function-call" | "is-valid-openai-tools-call" | "is-xml" | "javascript" | "latency" | "levenshtein" | "llm-rubric" | "pi" | "meteor" | "model-graded-closedqa" | "model-graded-factuality" | "perplexity" | "perplexity-score" | "python" | "regex" | "rouge-n" | "similar" | "starts-with" | "webhook" | "select-best" | "human";
25897
- value?: AssertionValue | undefined;
25898
- config?: Record<string, any> | undefined;
25899
- provider?: any;
25900
- transform?: string | undefined;
25901
- rubricPrompt?: string | string[] | {
25902
- role: string;
25903
- content: string;
25904
- }[] | undefined;
25905
- threshold?: number | undefined;
25906
- weight?: number | undefined;
25907
- metric?: string | undefined;
25908
- } | {
25909
- type: "assert-set";
25910
- assert: {
25911
- type: "moderation" | `promptfoo:redteam:${string}` | "cost" | "factuality" | "answer-relevance" | "bleu" | "classifier" | "contains" | "contains-all" | "contains-any" | "contains-json" | "contains-sql" | "contains-xml" | "context-faithfulness" | "context-recall" | "context-relevance" | "equals" | "g-eval" | "gleu" | "guardrails" | "icontains" | "icontains-all" | "icontains-any" | "is-json" | "is-refusal" | "is-sql" | "is-valid-function-call" | "is-valid-openai-function-call" | "is-valid-openai-tools-call" | "is-xml" | "javascript" | "latency" | "levenshtein" | "llm-rubric" | "pi" | "meteor" | "model-graded-closedqa" | "model-graded-factuality" | "perplexity" | "perplexity-score" | "python" | "regex" | "rouge-n" | "similar" | "starts-with" | "webhook" | "select-best" | "human";
25912
- value?: AssertionValue | undefined;
25913
- config?: Record<string, any> | undefined;
25914
- provider?: any;
25915
- transform?: string | undefined;
25916
- rubricPrompt?: string | string[] | {
25917
- role: string;
25918
- content: string;
25919
- }[] | undefined;
25920
- threshold?: number | undefined;
25921
- weight?: number | undefined;
25922
- metric?: string | undefined;
25923
- }[];
25924
- config?: Record<string, any> | undefined;
25925
- threshold?: number | undefined;
25926
- weight?: number | undefined;
25927
- metric?: string | undefined;
25928
- })[] | undefined;
25929
- threshold?: number | undefined;
25930
- assertScoringFunction?: string | ScoringFunction | undefined;
25931
- }[];
25932
- tests: {
25933
- options?: ({
25934
- prefix?: string | undefined;
25935
- suffix?: string | undefined;
25936
- } & {
25937
- transform?: string | undefined;
25938
- postprocess?: string | undefined;
25939
- transformVars?: string | undefined;
25940
- storeOutputAs?: string | undefined;
25941
- } & {
25942
- provider?: any;
25943
- rubricPrompt?: string | string[] | {
25944
- role: string;
25945
- content: string;
25946
- }[] | undefined;
25947
- factuality?: {
25948
- subset?: number | undefined;
25949
- superset?: number | undefined;
25950
- agree?: number | undefined;
25951
- disagree?: number | undefined;
25952
- differButFactual?: number | undefined;
25953
- } | undefined;
25954
- } & {
25955
- disableVarExpansion?: boolean | undefined;
25956
- disableConversationVar?: boolean | undefined;
25957
- runSerially?: boolean | undefined;
25958
- }) | undefined;
25959
- vars?: Record<string, string | number | boolean | any[] | Record<string, any> | (string | number | boolean)[]> | undefined;
25960
- provider?: string | {
25961
- id?: string | undefined;
25962
- config?: any;
25963
- label?: string | undefined;
25964
- prompts?: string[] | undefined;
25965
- transform?: string | undefined;
25966
- delay?: number | undefined;
25967
- env?: {
25968
- AI21_API_BASE_URL?: string | undefined;
25969
- AI21_API_KEY?: string | undefined;
25970
- ANTHROPIC_API_KEY?: string | undefined;
25971
- ANTHROPIC_BASE_URL?: string | undefined;
25972
- AWS_BEDROCK_REGION?: string | undefined;
25973
- AZURE_API_BASE_URL?: string | undefined;
25974
- AZURE_API_HOST?: string | undefined;
25975
- AZURE_API_KEY?: string | undefined;
25976
- AZURE_AUTHORITY_HOST?: string | undefined;
25977
- AZURE_CLIENT_ID?: string | undefined;
25978
- AZURE_CLIENT_SECRET?: string | undefined;
25979
- AZURE_DEPLOYMENT_NAME?: string | undefined;
25980
- AZURE_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
25981
- AZURE_OPENAI_API_BASE_URL?: string | undefined;
25982
- AZURE_OPENAI_API_HOST?: string | undefined;
25983
- AZURE_OPENAI_API_KEY?: string | undefined;
25984
- AZURE_OPENAI_BASE_URL?: string | undefined;
25985
- AZURE_OPENAI_DEPLOYMENT_NAME?: string | undefined;
25986
- AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
25987
- AZURE_TENANT_ID?: string | undefined;
25988
- AZURE_TOKEN_SCOPE?: string | undefined;
25989
- BAM_API_HOST?: string | undefined;
25990
- BAM_API_KEY?: string | undefined;
25991
- CLOUDFLARE_ACCOUNT_ID?: string | undefined;
25992
- CLOUDFLARE_API_KEY?: string | undefined;
25993
- COHERE_API_KEY?: string | undefined;
25994
- COHERE_CLIENT_NAME?: string | undefined;
25995
- DATABRICKS_TOKEN?: string | undefined;
25996
- DATABRICKS_WORKSPACE_URL?: string | undefined;
25997
- FAL_KEY?: string | undefined;
25998
- GOOGLE_API_HOST?: string | undefined;
25999
- GOOGLE_API_KEY?: string | undefined;
26000
- GROQ_API_KEY?: string | undefined;
26001
- HELICONE_API_KEY?: string | undefined;
26002
- HF_API_TOKEN?: string | undefined;
26003
- HF_TOKEN?: string | undefined;
26004
- HUGGING_FACE_HUB_TOKEN?: string | undefined;
26005
- JFROG_API_KEY?: string | undefined;
26006
- LAMBDA_API_KEY?: string | undefined;
26007
- LANGFUSE_HOST?: string | undefined;
26008
- LANGFUSE_PUBLIC_KEY?: string | undefined;
26009
- LANGFUSE_SECRET_KEY?: string | undefined;
26010
- LITELLM_API_BASE?: string | undefined;
26011
- LLAMA_BASE_URL?: string | undefined;
26012
- LOCALAI_BASE_URL?: string | undefined;
26013
- MISTRAL_API_BASE_URL?: string | undefined;
26014
- MISTRAL_API_HOST?: string | undefined;
26015
- MISTRAL_API_KEY?: string | undefined;
26016
- OLLAMA_API_KEY?: string | undefined;
26017
- OLLAMA_BASE_URL?: string | undefined;
26018
- OPENAI_API_BASE_URL?: string | undefined;
26019
- OPENAI_API_HOST?: string | undefined;
26020
- OPENAI_API_KEY?: string | undefined;
26021
- OPENAI_BASE_URL?: string | undefined;
26022
- OPENAI_ORGANIZATION?: string | undefined;
26023
- PALM_API_HOST?: string | undefined;
26024
- PALM_API_KEY?: string | undefined;
26025
- PORTKEY_API_KEY?: string | undefined;
26026
- PROMPTFOO_CA_CERT_PATH?: string | undefined;
26027
- PROMPTFOO_INSECURE_SSL?: string | undefined;
26028
- REPLICATE_API_KEY?: string | undefined;
26029
- REPLICATE_API_TOKEN?: string | undefined;
26030
- VERTEX_API_HOST?: string | undefined;
26031
- VERTEX_API_KEY?: string | undefined;
26032
- VERTEX_API_VERSION?: string | undefined;
26033
- VERTEX_PROJECT_ID?: string | undefined;
26034
- VERTEX_PUBLISHER?: string | undefined;
26035
- VERTEX_REGION?: string | undefined;
26036
- VOYAGE_API_BASE_URL?: string | undefined;
26037
- VOYAGE_API_KEY?: string | undefined;
26038
- WATSONX_AI_APIKEY?: string | undefined;
26039
- WATSONX_AI_AUTH_TYPE?: string | undefined;
26040
- WATSONX_AI_BEARER_TOKEN?: string | undefined;
26041
- WATSONX_AI_PROJECT_ID?: string | undefined;
26042
- AZURE_CONTENT_SAFETY_ENDPOINT?: string | undefined;
26043
- AZURE_CONTENT_SAFETY_API_KEY?: string | undefined;
26044
- AZURE_CONTENT_SAFETY_API_VERSION?: string | undefined;
26045
- AWS_REGION?: string | undefined;
26046
- AWS_DEFAULT_REGION?: string | undefined;
26047
- AWS_SAGEMAKER_MAX_TOKENS?: string | undefined;
26048
- AWS_SAGEMAKER_TEMPERATURE?: string | undefined;
26049
- AWS_SAGEMAKER_TOP_P?: string | undefined;
26050
- AWS_SAGEMAKER_MAX_RETRIES?: string | undefined;
26051
- PROMPTFOO_EVAL_TIMEOUT_MS?: string | undefined;
26052
- } | undefined;
26053
- } | {
26054
- callApi: import("./providers").CallApiFunction;
26055
- id: (...args: unknown[]) => string;
26056
- config?: any;
26057
- label?: string | undefined;
26058
- transform?: string | undefined;
26059
- delay?: number | undefined;
26060
- callEmbeddingApi?: ((args_0: string, ...args: unknown[]) => Promise<import("./providers").ProviderEmbeddingResponse>) | undefined;
26061
- callClassificationApi?: ((args_0: string, ...args: unknown[]) => Promise<import("./providers").ProviderClassificationResponse>) | undefined;
26062
- } | undefined;
26063
- metadata?: (Record<string, any> & {
26064
- pluginConfig?: PluginConfig | undefined;
26065
- strategyConfig?: import("../redteam/types").RedteamObjectConfig | undefined;
26066
- }) | undefined;
26067
- description?: string | undefined;
26068
- providerOutput?: string | {} | undefined;
26069
- assert?: ({
26070
- type: "moderation" | `promptfoo:redteam:${string}` | "cost" | "factuality" | "answer-relevance" | "bleu" | "classifier" | "contains" | "contains-all" | "contains-any" | "contains-json" | "contains-sql" | "contains-xml" | "context-faithfulness" | "context-recall" | "context-relevance" | "equals" | "g-eval" | "gleu" | "guardrails" | "icontains" | "icontains-all" | "icontains-any" | "is-json" | "is-refusal" | "is-sql" | "is-valid-function-call" | "is-valid-openai-function-call" | "is-valid-openai-tools-call" | "is-xml" | "javascript" | "latency" | "levenshtein" | "llm-rubric" | "pi" | "meteor" | "model-graded-closedqa" | "model-graded-factuality" | "perplexity" | "perplexity-score" | "python" | "regex" | "rouge-n" | "similar" | "starts-with" | "webhook" | "select-best" | "human";
26071
- value?: AssertionValue | undefined;
26072
- config?: Record<string, any> | undefined;
26073
- provider?: any;
26074
- transform?: string | undefined;
26075
- rubricPrompt?: string | string[] | {
26076
- role: string;
26077
- content: string;
26078
- }[] | undefined;
26079
- threshold?: number | undefined;
26080
- weight?: number | undefined;
26081
- metric?: string | undefined;
26082
- } | {
26083
- type: "assert-set";
26084
- assert: {
26085
- type: "moderation" | `promptfoo:redteam:${string}` | "cost" | "factuality" | "answer-relevance" | "bleu" | "classifier" | "contains" | "contains-all" | "contains-any" | "contains-json" | "contains-sql" | "contains-xml" | "context-faithfulness" | "context-recall" | "context-relevance" | "equals" | "g-eval" | "gleu" | "guardrails" | "icontains" | "icontains-all" | "icontains-any" | "is-json" | "is-refusal" | "is-sql" | "is-valid-function-call" | "is-valid-openai-function-call" | "is-valid-openai-tools-call" | "is-xml" | "javascript" | "latency" | "levenshtein" | "llm-rubric" | "pi" | "meteor" | "model-graded-closedqa" | "model-graded-factuality" | "perplexity" | "perplexity-score" | "python" | "regex" | "rouge-n" | "similar" | "starts-with" | "webhook" | "select-best" | "human";
26086
- value?: AssertionValue | undefined;
26087
- config?: Record<string, any> | undefined;
26088
- provider?: any;
26089
- transform?: string | undefined;
26090
- rubricPrompt?: string | string[] | {
26091
- role: string;
26092
- content: string;
26093
- }[] | undefined;
26094
- threshold?: number | undefined;
26095
- weight?: number | undefined;
26096
- metric?: string | undefined;
26097
- }[];
26098
- config?: Record<string, any> | undefined;
26099
- threshold?: number | undefined;
26100
- weight?: number | undefined;
26101
- metric?: string | undefined;
26102
- })[] | undefined;
26103
- threshold?: number | undefined;
26104
- assertScoringFunction?: string | ScoringFunction | undefined;
26105
- }[];
26106
- description?: string | undefined;
26107
- })[] | undefined;
26108
- defaultTest?: {
26109
- options?: ({
26110
- prefix?: string | undefined;
26111
- suffix?: string | undefined;
26112
- } & {
26113
- transform?: string | undefined;
26114
- postprocess?: string | undefined;
26115
- transformVars?: string | undefined;
26116
- storeOutputAs?: string | undefined;
26117
- } & {
26118
- provider?: any;
26119
- rubricPrompt?: string | string[] | {
26120
- role: string;
26121
- content: string;
26122
- }[] | undefined;
26123
- factuality?: {
26124
- subset?: number | undefined;
26125
- superset?: number | undefined;
26126
- agree?: number | undefined;
26127
- disagree?: number | undefined;
26128
- differButFactual?: number | undefined;
26129
- } | undefined;
26130
- } & {
26131
- disableVarExpansion?: boolean | undefined;
26132
- disableConversationVar?: boolean | undefined;
26133
- runSerially?: boolean | undefined;
26134
- }) | undefined;
26135
- vars?: Record<string, string | number | boolean | any[] | Record<string, any> | (string | number | boolean)[]> | undefined;
26136
- provider?: string | {
26137
- id?: string | undefined;
26138
- config?: any;
26139
- label?: string | undefined;
26140
- prompts?: string[] | undefined;
26141
- transform?: string | undefined;
26142
- delay?: number | undefined;
26143
- env?: {
26144
- AI21_API_BASE_URL?: string | undefined;
26145
- AI21_API_KEY?: string | undefined;
26146
- ANTHROPIC_API_KEY?: string | undefined;
26147
- ANTHROPIC_BASE_URL?: string | undefined;
26148
- AWS_BEDROCK_REGION?: string | undefined;
26149
- AZURE_API_BASE_URL?: string | undefined;
26150
- AZURE_API_HOST?: string | undefined;
26151
- AZURE_API_KEY?: string | undefined;
26152
- AZURE_AUTHORITY_HOST?: string | undefined;
26153
- AZURE_CLIENT_ID?: string | undefined;
26154
- AZURE_CLIENT_SECRET?: string | undefined;
26155
- AZURE_DEPLOYMENT_NAME?: string | undefined;
26156
- AZURE_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
26157
- AZURE_OPENAI_API_BASE_URL?: string | undefined;
26158
- AZURE_OPENAI_API_HOST?: string | undefined;
26159
- AZURE_OPENAI_API_KEY?: string | undefined;
26160
- AZURE_OPENAI_BASE_URL?: string | undefined;
26161
- AZURE_OPENAI_DEPLOYMENT_NAME?: string | undefined;
26162
- AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
26163
- AZURE_TENANT_ID?: string | undefined;
26164
- AZURE_TOKEN_SCOPE?: string | undefined;
26165
- BAM_API_HOST?: string | undefined;
26166
- BAM_API_KEY?: string | undefined;
26167
- CLOUDFLARE_ACCOUNT_ID?: string | undefined;
26168
- CLOUDFLARE_API_KEY?: string | undefined;
26169
- COHERE_API_KEY?: string | undefined;
26170
- COHERE_CLIENT_NAME?: string | undefined;
26171
- DATABRICKS_TOKEN?: string | undefined;
26172
- DATABRICKS_WORKSPACE_URL?: string | undefined;
26173
- FAL_KEY?: string | undefined;
26174
- GOOGLE_API_HOST?: string | undefined;
26175
- GOOGLE_API_KEY?: string | undefined;
26176
- GROQ_API_KEY?: string | undefined;
26177
- HELICONE_API_KEY?: string | undefined;
26178
- HF_API_TOKEN?: string | undefined;
26179
- HF_TOKEN?: string | undefined;
26180
- HUGGING_FACE_HUB_TOKEN?: string | undefined;
26181
- JFROG_API_KEY?: string | undefined;
26182
- LAMBDA_API_KEY?: string | undefined;
26183
- LANGFUSE_HOST?: string | undefined;
26184
- LANGFUSE_PUBLIC_KEY?: string | undefined;
26185
- LANGFUSE_SECRET_KEY?: string | undefined;
26186
- LITELLM_API_BASE?: string | undefined;
26187
- LLAMA_BASE_URL?: string | undefined;
26188
- LOCALAI_BASE_URL?: string | undefined;
26189
- MISTRAL_API_BASE_URL?: string | undefined;
26190
- MISTRAL_API_HOST?: string | undefined;
26191
- MISTRAL_API_KEY?: string | undefined;
26192
- OLLAMA_API_KEY?: string | undefined;
26193
- OLLAMA_BASE_URL?: string | undefined;
26194
- OPENAI_API_BASE_URL?: string | undefined;
26195
- OPENAI_API_HOST?: string | undefined;
26196
- OPENAI_API_KEY?: string | undefined;
26197
- OPENAI_BASE_URL?: string | undefined;
26198
- OPENAI_ORGANIZATION?: string | undefined;
26199
- PALM_API_HOST?: string | undefined;
26200
- PALM_API_KEY?: string | undefined;
26201
- PORTKEY_API_KEY?: string | undefined;
26202
- PROMPTFOO_CA_CERT_PATH?: string | undefined;
26203
- PROMPTFOO_INSECURE_SSL?: string | undefined;
26204
- REPLICATE_API_KEY?: string | undefined;
26205
- REPLICATE_API_TOKEN?: string | undefined;
26206
- VERTEX_API_HOST?: string | undefined;
26207
- VERTEX_API_KEY?: string | undefined;
26208
- VERTEX_API_VERSION?: string | undefined;
26209
- VERTEX_PROJECT_ID?: string | undefined;
26210
- VERTEX_PUBLISHER?: string | undefined;
26211
- VERTEX_REGION?: string | undefined;
26212
- VOYAGE_API_BASE_URL?: string | undefined;
26213
- VOYAGE_API_KEY?: string | undefined;
26214
- WATSONX_AI_APIKEY?: string | undefined;
26215
- WATSONX_AI_AUTH_TYPE?: string | undefined;
26216
- WATSONX_AI_BEARER_TOKEN?: string | undefined;
26217
- WATSONX_AI_PROJECT_ID?: string | undefined;
26218
- AZURE_CONTENT_SAFETY_ENDPOINT?: string | undefined;
26219
- AZURE_CONTENT_SAFETY_API_KEY?: string | undefined;
26220
- AZURE_CONTENT_SAFETY_API_VERSION?: string | undefined;
26221
- AWS_REGION?: string | undefined;
26222
- AWS_DEFAULT_REGION?: string | undefined;
26223
- AWS_SAGEMAKER_MAX_TOKENS?: string | undefined;
26224
- AWS_SAGEMAKER_TEMPERATURE?: string | undefined;
26225
- AWS_SAGEMAKER_TOP_P?: string | undefined;
26226
- AWS_SAGEMAKER_MAX_RETRIES?: string | undefined;
26227
- PROMPTFOO_EVAL_TIMEOUT_MS?: string | undefined;
26228
- } | undefined;
26229
- } | {
26230
- callApi: import("./providers").CallApiFunction;
26231
- id: (...args: unknown[]) => string;
26232
- config?: any;
26233
- label?: string | undefined;
26234
- transform?: string | undefined;
26235
- delay?: number | undefined;
26236
- callEmbeddingApi?: ((args_0: string, ...args: unknown[]) => Promise<import("./providers").ProviderEmbeddingResponse>) | undefined;
26237
- callClassificationApi?: ((args_0: string, ...args: unknown[]) => Promise<import("./providers").ProviderClassificationResponse>) | undefined;
26238
- } | undefined;
26239
- metadata?: (Record<string, any> & {
26240
- pluginConfig?: PluginConfig | undefined;
26241
- strategyConfig?: import("../redteam/types").RedteamObjectConfig | undefined;
26242
- }) | undefined;
26243
- providerOutput?: string | {} | undefined;
26244
- assert?: ({
26245
- type: "moderation" | `promptfoo:redteam:${string}` | "cost" | "factuality" | "answer-relevance" | "bleu" | "classifier" | "contains" | "contains-all" | "contains-any" | "contains-json" | "contains-sql" | "contains-xml" | "context-faithfulness" | "context-recall" | "context-relevance" | "equals" | "g-eval" | "gleu" | "guardrails" | "icontains" | "icontains-all" | "icontains-any" | "is-json" | "is-refusal" | "is-sql" | "is-valid-function-call" | "is-valid-openai-function-call" | "is-valid-openai-tools-call" | "is-xml" | "javascript" | "latency" | "levenshtein" | "llm-rubric" | "pi" | "meteor" | "model-graded-closedqa" | "model-graded-factuality" | "perplexity" | "perplexity-score" | "python" | "regex" | "rouge-n" | "similar" | "starts-with" | "webhook" | "select-best" | "human";
26246
- value?: AssertionValue | undefined;
26247
- config?: Record<string, any> | undefined;
26248
- provider?: any;
26249
- transform?: string | undefined;
26250
- rubricPrompt?: string | string[] | {
26251
- role: string;
26252
- content: string;
26253
- }[] | undefined;
26254
- threshold?: number | undefined;
26255
- weight?: number | undefined;
26256
- metric?: string | undefined;
26257
- } | {
26258
- type: "assert-set";
26259
- assert: {
26260
- type: "moderation" | `promptfoo:redteam:${string}` | "cost" | "factuality" | "answer-relevance" | "bleu" | "classifier" | "contains" | "contains-all" | "contains-any" | "contains-json" | "contains-sql" | "contains-xml" | "context-faithfulness" | "context-recall" | "context-relevance" | "equals" | "g-eval" | "gleu" | "guardrails" | "icontains" | "icontains-all" | "icontains-any" | "is-json" | "is-refusal" | "is-sql" | "is-valid-function-call" | "is-valid-openai-function-call" | "is-valid-openai-tools-call" | "is-xml" | "javascript" | "latency" | "levenshtein" | "llm-rubric" | "pi" | "meteor" | "model-graded-closedqa" | "model-graded-factuality" | "perplexity" | "perplexity-score" | "python" | "regex" | "rouge-n" | "similar" | "starts-with" | "webhook" | "select-best" | "human";
26261
- value?: AssertionValue | undefined;
26262
- config?: Record<string, any> | undefined;
26263
- provider?: any;
26264
- transform?: string | undefined;
26265
- rubricPrompt?: string | string[] | {
26266
- role: string;
26267
- content: string;
26268
- }[] | undefined;
26269
- threshold?: number | undefined;
26270
- weight?: number | undefined;
26271
- metric?: string | undefined;
26272
- }[];
26273
- config?: Record<string, any> | undefined;
26274
- threshold?: number | undefined;
26275
- weight?: number | undefined;
26276
- metric?: string | undefined;
26277
- })[] | undefined;
26278
- threshold?: number | undefined;
26279
- assertScoringFunction?: string | ScoringFunction | undefined;
26280
- } | undefined;
26281
- nunjucksFilters?: Record<string, string> | undefined;
26282
- derivedMetrics?: {
26283
- value: string | ((args_0: Record<string, number>, args_1: RunEvalOptions, ...args: unknown[]) => number);
26284
- name: string;
26285
- }[] | undefined;
26286
- extensions?: string[] | null | undefined;
26287
- outputPath?: string | string[] | undefined;
26288
- sharing?: boolean | {
26289
- apiBaseUrl?: string | undefined;
26290
- appBaseUrl?: string | undefined;
26291
- } | undefined;
26292
- writeLatestResults?: boolean | undefined;
26293
- }>;
26294
- export type TestSuiteConfig = z.infer<typeof TestSuiteConfigSchema>;
26295
- export declare const UnifiedConfigSchema: z.ZodEffects<z.ZodEffects<z.ZodObject<z.objectUtil.extendShape<{
26296
- tags: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
26297
- description: z.ZodOptional<z.ZodString>;
26298
- providers: z.ZodUnion<[z.ZodString, z.ZodIntersection<z.ZodFunction<z.ZodTuple<[z.ZodString, z.ZodOptional<z.ZodObject<{
26299
- fetchWithCache: z.ZodOptional<z.ZodAny>;
26300
- filters: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodFunction<z.ZodTuple<[z.ZodAny], z.ZodAny>, z.ZodString>>>;
26301
- getCache: z.ZodOptional<z.ZodAny>;
26302
- logger: z.ZodOptional<z.ZodAny>;
26303
- originalProvider: z.ZodOptional<z.ZodAny>;
26304
- prompt: z.ZodObject<{
26305
- id: z.ZodOptional<z.ZodString>;
26306
- raw: z.ZodString;
26307
- display: z.ZodOptional<z.ZodString>;
26308
- label: z.ZodString;
26309
- function: z.ZodOptional<z.ZodFunction<z.ZodTuple<[z.ZodObject<{
26310
- vars: z.ZodRecord<z.ZodString, z.ZodUnion<[z.ZodString, z.ZodAny]>>;
26311
- provider: z.ZodOptional<z.ZodType<ApiProvider, z.ZodTypeDef, ApiProvider>>;
26312
- }, "strip", z.ZodTypeAny, {
26313
- vars: Record<string, any>;
26314
- provider?: ApiProvider | undefined;
26315
- }, {
26316
- vars: Record<string, any>;
26317
- provider?: ApiProvider | undefined;
26318
- }>], z.ZodUnknown>, z.ZodPromise<z.ZodUnion<[z.ZodString, z.ZodAny]>>>>;
26319
- config: z.ZodOptional<z.ZodAny>;
26320
- }, "strip", z.ZodTypeAny, {
26321
- raw: string;
26322
- label: string;
26323
- function?: ((args_0: {
26324
- vars: Record<string, any>;
26325
- provider?: ApiProvider | undefined;
26326
- }, ...args: unknown[]) => Promise<any>) | undefined;
26327
- id?: string | undefined;
26328
- config?: any;
26329
- display?: string | undefined;
26330
- }, {
26331
- raw: string;
26332
- label: string;
26333
- function?: ((args_0: {
26334
- vars: Record<string, any>;
26335
- provider?: ApiProvider | undefined;
26336
- }, ...args: unknown[]) => Promise<any>) | undefined;
26337
- id?: string | undefined;
26338
- config?: any;
26339
- display?: string | undefined;
26340
- }>;
26341
- vars: z.ZodRecord<z.ZodString, z.ZodUnion<[z.ZodString, z.ZodObject<{}, "strip", z.ZodTypeAny, {}, {}>]>>;
26342
- }, "strip", z.ZodTypeAny, {
26343
- prompt: {
26344
- raw: string;
26345
- label: string;
26346
- function?: ((args_0: {
26347
- vars: Record<string, any>;
26348
- provider?: ApiProvider | undefined;
26349
- }, ...args: unknown[]) => Promise<any>) | undefined;
26350
- id?: string | undefined;
26351
- config?: any;
26352
- display?: string | undefined;
26353
- };
26354
- vars: Record<string, string | {}>;
26355
- fetchWithCache?: any;
26356
- filters?: Record<string, (args_0: any, ...args: any[]) => string> | undefined;
26357
- getCache?: any;
26358
- logger?: any;
26359
- originalProvider?: any;
26360
- }, {
26361
- prompt: {
26362
- raw: string;
26363
- label: string;
26364
- function?: ((args_0: {
26365
- vars: Record<string, any>;
26366
- provider?: ApiProvider | undefined;
26367
- }, ...args: unknown[]) => Promise<any>) | undefined;
26368
- id?: string | undefined;
26369
- config?: any;
26370
- display?: string | undefined;
26371
- };
26372
- vars: Record<string, string | {}>;
26373
- fetchWithCache?: any;
26374
- filters?: Record<string, (args_0: any, ...args: any[]) => string> | undefined;
26375
- getCache?: any;
26376
- logger?: any;
26377
- originalProvider?: any;
26378
- }>>, z.ZodOptional<z.ZodObject<{
26379
- includeLogProbs: z.ZodOptional<z.ZodBoolean>;
26380
- }, "strip", z.ZodTypeAny, {
26381
- includeLogProbs?: boolean | undefined;
26382
- }, {
26383
- includeLogProbs?: boolean | undefined;
26384
- }>>], z.ZodUnknown>, z.ZodPromise<z.ZodType<ProviderResponse, z.ZodTypeDef, ProviderResponse>>>, z.ZodObject<{
26385
- label: z.ZodOptional<z.ZodString>;
26386
- }, "strip", z.ZodTypeAny, {
26387
- label?: string | undefined;
26388
- }, {
26389
- label?: string | undefined;
26390
- }>>, z.ZodArray<z.ZodUnion<[z.ZodString, z.ZodRecord<z.ZodString, z.ZodObject<{
26391
- id: z.ZodOptional<z.ZodType<string, z.ZodTypeDef, string>>;
26392
- label: z.ZodOptional<z.ZodType<string, z.ZodTypeDef, string>>;
26393
- config: z.ZodOptional<z.ZodAny>;
26394
- prompts: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
26395
- transform: z.ZodOptional<z.ZodString>;
26396
- delay: z.ZodOptional<z.ZodNumber>;
26397
- env: z.ZodOptional<z.ZodObject<{
26398
- AI21_API_BASE_URL: z.ZodOptional<z.ZodString>;
26399
- AI21_API_KEY: z.ZodOptional<z.ZodString>;
26400
- ANTHROPIC_API_KEY: z.ZodOptional<z.ZodString>;
26401
- ANTHROPIC_BASE_URL: z.ZodOptional<z.ZodString>;
26402
- AWS_BEDROCK_REGION: z.ZodOptional<z.ZodString>;
26403
- AZURE_API_BASE_URL: z.ZodOptional<z.ZodString>;
26404
- AZURE_API_HOST: z.ZodOptional<z.ZodString>;
26405
- AZURE_API_KEY: z.ZodOptional<z.ZodString>;
26406
- AZURE_AUTHORITY_HOST: z.ZodOptional<z.ZodString>;
26407
- AZURE_CLIENT_ID: z.ZodOptional<z.ZodString>;
26408
- AZURE_CLIENT_SECRET: z.ZodOptional<z.ZodString>;
26409
- AZURE_DEPLOYMENT_NAME: z.ZodOptional<z.ZodString>;
26410
- AZURE_EMBEDDING_DEPLOYMENT_NAME: z.ZodOptional<z.ZodString>;
26411
- AZURE_OPENAI_API_BASE_URL: z.ZodOptional<z.ZodString>;
26412
- AZURE_OPENAI_API_HOST: z.ZodOptional<z.ZodString>;
26413
- AZURE_OPENAI_API_KEY: z.ZodOptional<z.ZodString>;
26414
- AZURE_OPENAI_BASE_URL: z.ZodOptional<z.ZodString>;
26415
- AZURE_OPENAI_DEPLOYMENT_NAME: z.ZodOptional<z.ZodString>;
26416
- AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME: z.ZodOptional<z.ZodString>;
26417
- AZURE_TENANT_ID: z.ZodOptional<z.ZodString>;
26418
- AZURE_TOKEN_SCOPE: z.ZodOptional<z.ZodString>;
26419
- BAM_API_HOST: z.ZodOptional<z.ZodString>;
26420
- BAM_API_KEY: z.ZodOptional<z.ZodString>;
26421
- CLOUDFLARE_ACCOUNT_ID: z.ZodOptional<z.ZodString>;
26422
- CLOUDFLARE_API_KEY: z.ZodOptional<z.ZodString>;
26423
- COHERE_API_KEY: z.ZodOptional<z.ZodString>;
26424
- COHERE_CLIENT_NAME: z.ZodOptional<z.ZodString>;
26425
- DATABRICKS_TOKEN: z.ZodOptional<z.ZodString>;
26426
- DATABRICKS_WORKSPACE_URL: z.ZodOptional<z.ZodString>;
26427
- FAL_KEY: z.ZodOptional<z.ZodString>;
26428
- GOOGLE_API_HOST: z.ZodOptional<z.ZodString>;
26429
- GOOGLE_API_KEY: z.ZodOptional<z.ZodString>;
26430
- GROQ_API_KEY: z.ZodOptional<z.ZodString>;
26431
- HELICONE_API_KEY: z.ZodOptional<z.ZodString>;
26432
- HF_API_TOKEN: z.ZodOptional<z.ZodString>;
26433
- HF_TOKEN: z.ZodOptional<z.ZodString>;
26434
- HUGGING_FACE_HUB_TOKEN: z.ZodOptional<z.ZodString>;
26435
- JFROG_API_KEY: z.ZodOptional<z.ZodString>;
26436
- LAMBDA_API_KEY: z.ZodOptional<z.ZodString>;
26437
- LANGFUSE_HOST: z.ZodOptional<z.ZodString>;
26438
- LANGFUSE_PUBLIC_KEY: z.ZodOptional<z.ZodString>;
26439
- LANGFUSE_SECRET_KEY: z.ZodOptional<z.ZodString>;
26440
- LITELLM_API_BASE: z.ZodOptional<z.ZodString>;
26441
- LLAMA_BASE_URL: z.ZodOptional<z.ZodString>;
26442
- LOCALAI_BASE_URL: z.ZodOptional<z.ZodString>;
26443
- MISTRAL_API_BASE_URL: z.ZodOptional<z.ZodString>;
26444
- MISTRAL_API_HOST: z.ZodOptional<z.ZodString>;
26445
- MISTRAL_API_KEY: z.ZodOptional<z.ZodString>;
26446
- OLLAMA_API_KEY: z.ZodOptional<z.ZodString>;
26447
- OLLAMA_BASE_URL: z.ZodOptional<z.ZodString>;
26448
- OPENAI_API_BASE_URL: z.ZodOptional<z.ZodString>;
26449
- OPENAI_API_HOST: z.ZodOptional<z.ZodString>;
26450
- OPENAI_API_KEY: z.ZodOptional<z.ZodString>;
26451
- OPENAI_BASE_URL: z.ZodOptional<z.ZodString>;
26452
- OPENAI_ORGANIZATION: z.ZodOptional<z.ZodString>;
26453
- PALM_API_HOST: z.ZodOptional<z.ZodString>;
26454
- PALM_API_KEY: z.ZodOptional<z.ZodString>;
26455
- PORTKEY_API_KEY: z.ZodOptional<z.ZodString>;
26456
- PROMPTFOO_CA_CERT_PATH: z.ZodOptional<z.ZodString>;
26457
- PROMPTFOO_INSECURE_SSL: z.ZodOptional<z.ZodString>;
26458
- REPLICATE_API_KEY: z.ZodOptional<z.ZodString>;
26459
- REPLICATE_API_TOKEN: z.ZodOptional<z.ZodString>;
26460
- VERTEX_API_HOST: z.ZodOptional<z.ZodString>;
26461
- VERTEX_API_KEY: z.ZodOptional<z.ZodString>;
26462
- VERTEX_API_VERSION: z.ZodOptional<z.ZodString>;
26463
- VERTEX_PROJECT_ID: z.ZodOptional<z.ZodString>;
26464
- VERTEX_PUBLISHER: z.ZodOptional<z.ZodString>;
26465
- VERTEX_REGION: z.ZodOptional<z.ZodString>;
26466
- VOYAGE_API_BASE_URL: z.ZodOptional<z.ZodString>;
26467
- VOYAGE_API_KEY: z.ZodOptional<z.ZodString>;
26468
- WATSONX_AI_APIKEY: z.ZodOptional<z.ZodString>;
26469
- WATSONX_AI_AUTH_TYPE: z.ZodOptional<z.ZodString>;
26470
- WATSONX_AI_BEARER_TOKEN: z.ZodOptional<z.ZodString>;
26471
- WATSONX_AI_PROJECT_ID: z.ZodOptional<z.ZodString>;
26472
- AZURE_CONTENT_SAFETY_ENDPOINT: z.ZodOptional<z.ZodString>;
26473
- AZURE_CONTENT_SAFETY_API_KEY: z.ZodOptional<z.ZodString>;
26474
- AZURE_CONTENT_SAFETY_API_VERSION: z.ZodOptional<z.ZodString>;
26475
- AWS_REGION: z.ZodOptional<z.ZodString>;
26476
- AWS_DEFAULT_REGION: z.ZodOptional<z.ZodString>;
26477
- AWS_SAGEMAKER_MAX_TOKENS: z.ZodOptional<z.ZodString>;
26478
- AWS_SAGEMAKER_TEMPERATURE: z.ZodOptional<z.ZodString>;
26479
- AWS_SAGEMAKER_TOP_P: z.ZodOptional<z.ZodString>;
26480
- AWS_SAGEMAKER_MAX_RETRIES: z.ZodOptional<z.ZodString>;
26481
- PROMPTFOO_EVAL_TIMEOUT_MS: z.ZodOptional<z.ZodString>;
26482
- }, "strip", z.ZodTypeAny, {
26483
- AI21_API_BASE_URL?: string | undefined;
26484
- AI21_API_KEY?: string | undefined;
26485
- ANTHROPIC_API_KEY?: string | undefined;
26486
- ANTHROPIC_BASE_URL?: string | undefined;
26487
- AWS_BEDROCK_REGION?: string | undefined;
26488
- AZURE_API_BASE_URL?: string | undefined;
26489
- AZURE_API_HOST?: string | undefined;
26490
- AZURE_API_KEY?: string | undefined;
26491
- AZURE_AUTHORITY_HOST?: string | undefined;
26492
- AZURE_CLIENT_ID?: string | undefined;
26493
- AZURE_CLIENT_SECRET?: string | undefined;
26494
- AZURE_DEPLOYMENT_NAME?: string | undefined;
26495
- AZURE_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
26496
- AZURE_OPENAI_API_BASE_URL?: string | undefined;
26497
- AZURE_OPENAI_API_HOST?: string | undefined;
26498
- AZURE_OPENAI_API_KEY?: string | undefined;
26499
- AZURE_OPENAI_BASE_URL?: string | undefined;
26500
- AZURE_OPENAI_DEPLOYMENT_NAME?: string | undefined;
26501
- AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
26502
- AZURE_TENANT_ID?: string | undefined;
26503
- AZURE_TOKEN_SCOPE?: string | undefined;
26504
- BAM_API_HOST?: string | undefined;
26505
- BAM_API_KEY?: string | undefined;
26506
- CLOUDFLARE_ACCOUNT_ID?: string | undefined;
26507
- CLOUDFLARE_API_KEY?: string | undefined;
26508
- COHERE_API_KEY?: string | undefined;
26509
- COHERE_CLIENT_NAME?: string | undefined;
26510
- DATABRICKS_TOKEN?: string | undefined;
26511
- DATABRICKS_WORKSPACE_URL?: string | undefined;
26512
- FAL_KEY?: string | undefined;
26513
- GOOGLE_API_HOST?: string | undefined;
26514
- GOOGLE_API_KEY?: string | undefined;
26515
- GROQ_API_KEY?: string | undefined;
26516
- HELICONE_API_KEY?: string | undefined;
26517
- HF_API_TOKEN?: string | undefined;
26518
- HF_TOKEN?: string | undefined;
26519
- HUGGING_FACE_HUB_TOKEN?: string | undefined;
26520
- JFROG_API_KEY?: string | undefined;
26521
- LAMBDA_API_KEY?: string | undefined;
26522
- LANGFUSE_HOST?: string | undefined;
26523
- LANGFUSE_PUBLIC_KEY?: string | undefined;
26524
- LANGFUSE_SECRET_KEY?: string | undefined;
26525
- LITELLM_API_BASE?: string | undefined;
26526
- LLAMA_BASE_URL?: string | undefined;
26527
- LOCALAI_BASE_URL?: string | undefined;
26528
- MISTRAL_API_BASE_URL?: string | undefined;
26529
- MISTRAL_API_HOST?: string | undefined;
26530
- MISTRAL_API_KEY?: string | undefined;
26531
- OLLAMA_API_KEY?: string | undefined;
26532
- OLLAMA_BASE_URL?: string | undefined;
26533
- OPENAI_API_BASE_URL?: string | undefined;
26534
- OPENAI_API_HOST?: string | undefined;
26535
- OPENAI_API_KEY?: string | undefined;
26536
- OPENAI_BASE_URL?: string | undefined;
26537
- OPENAI_ORGANIZATION?: string | undefined;
26538
- PALM_API_HOST?: string | undefined;
26539
- PALM_API_KEY?: string | undefined;
26540
- PORTKEY_API_KEY?: string | undefined;
26541
- PROMPTFOO_CA_CERT_PATH?: string | undefined;
26542
- PROMPTFOO_INSECURE_SSL?: string | undefined;
26543
- REPLICATE_API_KEY?: string | undefined;
26544
- REPLICATE_API_TOKEN?: string | undefined;
26545
- VERTEX_API_HOST?: string | undefined;
26546
- VERTEX_API_KEY?: string | undefined;
26547
- VERTEX_API_VERSION?: string | undefined;
26548
- VERTEX_PROJECT_ID?: string | undefined;
26549
- VERTEX_PUBLISHER?: string | undefined;
26550
- VERTEX_REGION?: string | undefined;
26551
- VOYAGE_API_BASE_URL?: string | undefined;
26552
- VOYAGE_API_KEY?: string | undefined;
26553
- WATSONX_AI_APIKEY?: string | undefined;
26554
- WATSONX_AI_AUTH_TYPE?: string | undefined;
26555
- WATSONX_AI_BEARER_TOKEN?: string | undefined;
26556
- WATSONX_AI_PROJECT_ID?: string | undefined;
26557
- AZURE_CONTENT_SAFETY_ENDPOINT?: string | undefined;
26558
- AZURE_CONTENT_SAFETY_API_KEY?: string | undefined;
26559
- AZURE_CONTENT_SAFETY_API_VERSION?: string | undefined;
26560
- AWS_REGION?: string | undefined;
26561
- AWS_DEFAULT_REGION?: string | undefined;
26562
- AWS_SAGEMAKER_MAX_TOKENS?: string | undefined;
26563
- AWS_SAGEMAKER_TEMPERATURE?: string | undefined;
26564
- AWS_SAGEMAKER_TOP_P?: string | undefined;
26565
- AWS_SAGEMAKER_MAX_RETRIES?: string | undefined;
26566
- PROMPTFOO_EVAL_TIMEOUT_MS?: string | undefined;
26567
- }, {
26568
- AI21_API_BASE_URL?: string | undefined;
26569
- AI21_API_KEY?: string | undefined;
26570
- ANTHROPIC_API_KEY?: string | undefined;
26571
- ANTHROPIC_BASE_URL?: string | undefined;
26572
- AWS_BEDROCK_REGION?: string | undefined;
26573
- AZURE_API_BASE_URL?: string | undefined;
26574
- AZURE_API_HOST?: string | undefined;
26575
- AZURE_API_KEY?: string | undefined;
26576
- AZURE_AUTHORITY_HOST?: string | undefined;
26577
- AZURE_CLIENT_ID?: string | undefined;
26578
- AZURE_CLIENT_SECRET?: string | undefined;
26579
- AZURE_DEPLOYMENT_NAME?: string | undefined;
26580
- AZURE_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
26581
- AZURE_OPENAI_API_BASE_URL?: string | undefined;
26582
- AZURE_OPENAI_API_HOST?: string | undefined;
26583
- AZURE_OPENAI_API_KEY?: string | undefined;
26584
- AZURE_OPENAI_BASE_URL?: string | undefined;
26585
- AZURE_OPENAI_DEPLOYMENT_NAME?: string | undefined;
26586
- AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
26587
- AZURE_TENANT_ID?: string | undefined;
26588
- AZURE_TOKEN_SCOPE?: string | undefined;
26589
- BAM_API_HOST?: string | undefined;
26590
- BAM_API_KEY?: string | undefined;
26591
- CLOUDFLARE_ACCOUNT_ID?: string | undefined;
26592
- CLOUDFLARE_API_KEY?: string | undefined;
26593
- COHERE_API_KEY?: string | undefined;
26594
- COHERE_CLIENT_NAME?: string | undefined;
26595
- DATABRICKS_TOKEN?: string | undefined;
26596
- DATABRICKS_WORKSPACE_URL?: string | undefined;
26597
- FAL_KEY?: string | undefined;
26598
- GOOGLE_API_HOST?: string | undefined;
26599
- GOOGLE_API_KEY?: string | undefined;
26600
- GROQ_API_KEY?: string | undefined;
26601
- HELICONE_API_KEY?: string | undefined;
26602
- HF_API_TOKEN?: string | undefined;
26603
- HF_TOKEN?: string | undefined;
26604
- HUGGING_FACE_HUB_TOKEN?: string | undefined;
26605
- JFROG_API_KEY?: string | undefined;
26606
- LAMBDA_API_KEY?: string | undefined;
26607
- LANGFUSE_HOST?: string | undefined;
26608
- LANGFUSE_PUBLIC_KEY?: string | undefined;
26609
- LANGFUSE_SECRET_KEY?: string | undefined;
26610
- LITELLM_API_BASE?: string | undefined;
26611
- LLAMA_BASE_URL?: string | undefined;
26612
- LOCALAI_BASE_URL?: string | undefined;
26613
- MISTRAL_API_BASE_URL?: string | undefined;
26614
- MISTRAL_API_HOST?: string | undefined;
26615
- MISTRAL_API_KEY?: string | undefined;
26616
- OLLAMA_API_KEY?: string | undefined;
26617
- OLLAMA_BASE_URL?: string | undefined;
26618
- OPENAI_API_BASE_URL?: string | undefined;
26619
- OPENAI_API_HOST?: string | undefined;
26620
- OPENAI_API_KEY?: string | undefined;
26621
- OPENAI_BASE_URL?: string | undefined;
26622
- OPENAI_ORGANIZATION?: string | undefined;
26623
- PALM_API_HOST?: string | undefined;
26624
- PALM_API_KEY?: string | undefined;
26625
- PORTKEY_API_KEY?: string | undefined;
26626
- PROMPTFOO_CA_CERT_PATH?: string | undefined;
26627
- PROMPTFOO_INSECURE_SSL?: string | undefined;
26628
- REPLICATE_API_KEY?: string | undefined;
26629
- REPLICATE_API_TOKEN?: string | undefined;
26630
- VERTEX_API_HOST?: string | undefined;
26631
- VERTEX_API_KEY?: string | undefined;
26632
- VERTEX_API_VERSION?: string | undefined;
26633
- VERTEX_PROJECT_ID?: string | undefined;
26634
- VERTEX_PUBLISHER?: string | undefined;
26635
- VERTEX_REGION?: string | undefined;
26636
- VOYAGE_API_BASE_URL?: string | undefined;
26637
- VOYAGE_API_KEY?: string | undefined;
26638
- WATSONX_AI_APIKEY?: string | undefined;
26639
- WATSONX_AI_AUTH_TYPE?: string | undefined;
26640
- WATSONX_AI_BEARER_TOKEN?: string | undefined;
26641
- WATSONX_AI_PROJECT_ID?: string | undefined;
26642
- AZURE_CONTENT_SAFETY_ENDPOINT?: string | undefined;
26643
- AZURE_CONTENT_SAFETY_API_KEY?: string | undefined;
26644
- AZURE_CONTENT_SAFETY_API_VERSION?: string | undefined;
26645
- AWS_REGION?: string | undefined;
26646
- AWS_DEFAULT_REGION?: string | undefined;
26647
- AWS_SAGEMAKER_MAX_TOKENS?: string | undefined;
26648
- AWS_SAGEMAKER_TEMPERATURE?: string | undefined;
26649
- AWS_SAGEMAKER_TOP_P?: string | undefined;
26650
- AWS_SAGEMAKER_MAX_RETRIES?: string | undefined;
26651
- PROMPTFOO_EVAL_TIMEOUT_MS?: string | undefined;
26652
- }>>;
26653
- }, "strict", z.ZodTypeAny, {
26654
- id?: string | undefined;
26655
- config?: any;
26656
- label?: string | undefined;
26657
- prompts?: string[] | undefined;
26658
- transform?: string | undefined;
26659
- delay?: number | undefined;
26660
- env?: {
26661
- AI21_API_BASE_URL?: string | undefined;
26662
- AI21_API_KEY?: string | undefined;
26663
- ANTHROPIC_API_KEY?: string | undefined;
26664
- ANTHROPIC_BASE_URL?: string | undefined;
26665
- AWS_BEDROCK_REGION?: string | undefined;
26666
- AZURE_API_BASE_URL?: string | undefined;
26667
- AZURE_API_HOST?: string | undefined;
26668
- AZURE_API_KEY?: string | undefined;
26669
- AZURE_AUTHORITY_HOST?: string | undefined;
26670
- AZURE_CLIENT_ID?: string | undefined;
26671
- AZURE_CLIENT_SECRET?: string | undefined;
26672
- AZURE_DEPLOYMENT_NAME?: string | undefined;
26673
- AZURE_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
26674
- AZURE_OPENAI_API_BASE_URL?: string | undefined;
26675
- AZURE_OPENAI_API_HOST?: string | undefined;
26676
- AZURE_OPENAI_API_KEY?: string | undefined;
26677
- AZURE_OPENAI_BASE_URL?: string | undefined;
26678
- AZURE_OPENAI_DEPLOYMENT_NAME?: string | undefined;
26679
- AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
26680
- AZURE_TENANT_ID?: string | undefined;
26681
- AZURE_TOKEN_SCOPE?: string | undefined;
26682
- BAM_API_HOST?: string | undefined;
26683
- BAM_API_KEY?: string | undefined;
26684
- CLOUDFLARE_ACCOUNT_ID?: string | undefined;
26685
- CLOUDFLARE_API_KEY?: string | undefined;
26686
- COHERE_API_KEY?: string | undefined;
26687
- COHERE_CLIENT_NAME?: string | undefined;
26688
- DATABRICKS_TOKEN?: string | undefined;
26689
- DATABRICKS_WORKSPACE_URL?: string | undefined;
26690
- FAL_KEY?: string | undefined;
26691
- GOOGLE_API_HOST?: string | undefined;
26692
- GOOGLE_API_KEY?: string | undefined;
26693
- GROQ_API_KEY?: string | undefined;
26694
- HELICONE_API_KEY?: string | undefined;
26695
- HF_API_TOKEN?: string | undefined;
26696
- HF_TOKEN?: string | undefined;
26697
- HUGGING_FACE_HUB_TOKEN?: string | undefined;
26698
- JFROG_API_KEY?: string | undefined;
26699
- LAMBDA_API_KEY?: string | undefined;
26700
- LANGFUSE_HOST?: string | undefined;
26701
- LANGFUSE_PUBLIC_KEY?: string | undefined;
26702
- LANGFUSE_SECRET_KEY?: string | undefined;
26703
- LITELLM_API_BASE?: string | undefined;
26704
- LLAMA_BASE_URL?: string | undefined;
26705
- LOCALAI_BASE_URL?: string | undefined;
26706
- MISTRAL_API_BASE_URL?: string | undefined;
26707
- MISTRAL_API_HOST?: string | undefined;
26708
- MISTRAL_API_KEY?: string | undefined;
26709
- OLLAMA_API_KEY?: string | undefined;
26710
- OLLAMA_BASE_URL?: string | undefined;
26711
- OPENAI_API_BASE_URL?: string | undefined;
26712
- OPENAI_API_HOST?: string | undefined;
26713
- OPENAI_API_KEY?: string | undefined;
26714
- OPENAI_BASE_URL?: string | undefined;
26715
- OPENAI_ORGANIZATION?: string | undefined;
26716
- PALM_API_HOST?: string | undefined;
26717
- PALM_API_KEY?: string | undefined;
26718
- PORTKEY_API_KEY?: string | undefined;
26719
- PROMPTFOO_CA_CERT_PATH?: string | undefined;
26720
- PROMPTFOO_INSECURE_SSL?: string | undefined;
26721
- REPLICATE_API_KEY?: string | undefined;
26722
- REPLICATE_API_TOKEN?: string | undefined;
26723
- VERTEX_API_HOST?: string | undefined;
26724
- VERTEX_API_KEY?: string | undefined;
26725
- VERTEX_API_VERSION?: string | undefined;
26726
- VERTEX_PROJECT_ID?: string | undefined;
26727
- VERTEX_PUBLISHER?: string | undefined;
26728
- VERTEX_REGION?: string | undefined;
26729
- VOYAGE_API_BASE_URL?: string | undefined;
26730
- VOYAGE_API_KEY?: string | undefined;
26731
- WATSONX_AI_APIKEY?: string | undefined;
26732
- WATSONX_AI_AUTH_TYPE?: string | undefined;
26733
- WATSONX_AI_BEARER_TOKEN?: string | undefined;
26734
- WATSONX_AI_PROJECT_ID?: string | undefined;
26735
- AZURE_CONTENT_SAFETY_ENDPOINT?: string | undefined;
26736
- AZURE_CONTENT_SAFETY_API_KEY?: string | undefined;
26737
- AZURE_CONTENT_SAFETY_API_VERSION?: string | undefined;
26738
- AWS_REGION?: string | undefined;
26739
- AWS_DEFAULT_REGION?: string | undefined;
26740
- AWS_SAGEMAKER_MAX_TOKENS?: string | undefined;
26741
- AWS_SAGEMAKER_TEMPERATURE?: string | undefined;
26742
- AWS_SAGEMAKER_TOP_P?: string | undefined;
26743
- AWS_SAGEMAKER_MAX_RETRIES?: string | undefined;
26744
- PROMPTFOO_EVAL_TIMEOUT_MS?: string | undefined;
26745
- } | undefined;
26746
- }, {
26747
- id?: string | undefined;
26748
- config?: any;
26749
- label?: string | undefined;
26750
- prompts?: string[] | undefined;
26751
- transform?: string | undefined;
26752
- delay?: number | undefined;
26753
- env?: {
26754
- AI21_API_BASE_URL?: string | undefined;
26755
- AI21_API_KEY?: string | undefined;
26756
- ANTHROPIC_API_KEY?: string | undefined;
26757
- ANTHROPIC_BASE_URL?: string | undefined;
26758
- AWS_BEDROCK_REGION?: string | undefined;
26759
- AZURE_API_BASE_URL?: string | undefined;
26760
- AZURE_API_HOST?: string | undefined;
26761
- AZURE_API_KEY?: string | undefined;
26762
- AZURE_AUTHORITY_HOST?: string | undefined;
26763
- AZURE_CLIENT_ID?: string | undefined;
26764
- AZURE_CLIENT_SECRET?: string | undefined;
26765
- AZURE_DEPLOYMENT_NAME?: string | undefined;
26766
- AZURE_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
26767
- AZURE_OPENAI_API_BASE_URL?: string | undefined;
26768
- AZURE_OPENAI_API_HOST?: string | undefined;
26769
- AZURE_OPENAI_API_KEY?: string | undefined;
26770
- AZURE_OPENAI_BASE_URL?: string | undefined;
26771
- AZURE_OPENAI_DEPLOYMENT_NAME?: string | undefined;
26772
- AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
26773
- AZURE_TENANT_ID?: string | undefined;
26774
- AZURE_TOKEN_SCOPE?: string | undefined;
26775
- BAM_API_HOST?: string | undefined;
26776
- BAM_API_KEY?: string | undefined;
26777
- CLOUDFLARE_ACCOUNT_ID?: string | undefined;
26778
- CLOUDFLARE_API_KEY?: string | undefined;
26779
- COHERE_API_KEY?: string | undefined;
26780
- COHERE_CLIENT_NAME?: string | undefined;
26781
- DATABRICKS_TOKEN?: string | undefined;
26782
- DATABRICKS_WORKSPACE_URL?: string | undefined;
26783
- FAL_KEY?: string | undefined;
26784
- GOOGLE_API_HOST?: string | undefined;
26785
- GOOGLE_API_KEY?: string | undefined;
26786
- GROQ_API_KEY?: string | undefined;
26787
- HELICONE_API_KEY?: string | undefined;
26788
- HF_API_TOKEN?: string | undefined;
26789
- HF_TOKEN?: string | undefined;
26790
- HUGGING_FACE_HUB_TOKEN?: string | undefined;
26791
- JFROG_API_KEY?: string | undefined;
26792
- LAMBDA_API_KEY?: string | undefined;
26793
- LANGFUSE_HOST?: string | undefined;
26794
- LANGFUSE_PUBLIC_KEY?: string | undefined;
26795
- LANGFUSE_SECRET_KEY?: string | undefined;
26796
- LITELLM_API_BASE?: string | undefined;
26797
- LLAMA_BASE_URL?: string | undefined;
26798
- LOCALAI_BASE_URL?: string | undefined;
26799
- MISTRAL_API_BASE_URL?: string | undefined;
26800
- MISTRAL_API_HOST?: string | undefined;
26801
- MISTRAL_API_KEY?: string | undefined;
26802
- OLLAMA_API_KEY?: string | undefined;
26803
- OLLAMA_BASE_URL?: string | undefined;
26804
- OPENAI_API_BASE_URL?: string | undefined;
26805
- OPENAI_API_HOST?: string | undefined;
26806
- OPENAI_API_KEY?: string | undefined;
26807
- OPENAI_BASE_URL?: string | undefined;
26808
- OPENAI_ORGANIZATION?: string | undefined;
26809
- PALM_API_HOST?: string | undefined;
26810
- PALM_API_KEY?: string | undefined;
26811
- PORTKEY_API_KEY?: string | undefined;
26812
- PROMPTFOO_CA_CERT_PATH?: string | undefined;
26813
- PROMPTFOO_INSECURE_SSL?: string | undefined;
26814
- REPLICATE_API_KEY?: string | undefined;
26815
- REPLICATE_API_TOKEN?: string | undefined;
26816
- VERTEX_API_HOST?: string | undefined;
26817
- VERTEX_API_KEY?: string | undefined;
26818
- VERTEX_API_VERSION?: string | undefined;
26819
- VERTEX_PROJECT_ID?: string | undefined;
26820
- VERTEX_PUBLISHER?: string | undefined;
26821
- VERTEX_REGION?: string | undefined;
26822
- VOYAGE_API_BASE_URL?: string | undefined;
26823
- VOYAGE_API_KEY?: string | undefined;
26824
- WATSONX_AI_APIKEY?: string | undefined;
26825
- WATSONX_AI_AUTH_TYPE?: string | undefined;
26826
- WATSONX_AI_BEARER_TOKEN?: string | undefined;
26827
- WATSONX_AI_PROJECT_ID?: string | undefined;
26828
- AZURE_CONTENT_SAFETY_ENDPOINT?: string | undefined;
26829
- AZURE_CONTENT_SAFETY_API_KEY?: string | undefined;
26830
- AZURE_CONTENT_SAFETY_API_VERSION?: string | undefined;
26831
- AWS_REGION?: string | undefined;
26832
- AWS_DEFAULT_REGION?: string | undefined;
26833
- AWS_SAGEMAKER_MAX_TOKENS?: string | undefined;
26834
- AWS_SAGEMAKER_TEMPERATURE?: string | undefined;
26835
- AWS_SAGEMAKER_TOP_P?: string | undefined;
26836
- AWS_SAGEMAKER_MAX_RETRIES?: string | undefined;
26837
- PROMPTFOO_EVAL_TIMEOUT_MS?: string | undefined;
26838
- } | undefined;
26839
- }>>, z.ZodObject<{
26840
- id: z.ZodOptional<z.ZodType<string, z.ZodTypeDef, string>>;
26841
- label: z.ZodOptional<z.ZodType<string, z.ZodTypeDef, string>>;
26842
- config: z.ZodOptional<z.ZodAny>;
26843
- prompts: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
26844
- transform: z.ZodOptional<z.ZodString>;
26845
- delay: z.ZodOptional<z.ZodNumber>;
26846
- env: z.ZodOptional<z.ZodObject<{
26847
- AI21_API_BASE_URL: z.ZodOptional<z.ZodString>;
26848
- AI21_API_KEY: z.ZodOptional<z.ZodString>;
26849
- ANTHROPIC_API_KEY: z.ZodOptional<z.ZodString>;
26850
- ANTHROPIC_BASE_URL: z.ZodOptional<z.ZodString>;
26851
- AWS_BEDROCK_REGION: z.ZodOptional<z.ZodString>;
26852
- AZURE_API_BASE_URL: z.ZodOptional<z.ZodString>;
26853
- AZURE_API_HOST: z.ZodOptional<z.ZodString>;
26854
- AZURE_API_KEY: z.ZodOptional<z.ZodString>;
26855
- AZURE_AUTHORITY_HOST: z.ZodOptional<z.ZodString>;
26856
- AZURE_CLIENT_ID: z.ZodOptional<z.ZodString>;
26857
- AZURE_CLIENT_SECRET: z.ZodOptional<z.ZodString>;
26858
- AZURE_DEPLOYMENT_NAME: z.ZodOptional<z.ZodString>;
26859
- AZURE_EMBEDDING_DEPLOYMENT_NAME: z.ZodOptional<z.ZodString>;
26860
- AZURE_OPENAI_API_BASE_URL: z.ZodOptional<z.ZodString>;
26861
- AZURE_OPENAI_API_HOST: z.ZodOptional<z.ZodString>;
26862
- AZURE_OPENAI_API_KEY: z.ZodOptional<z.ZodString>;
26863
- AZURE_OPENAI_BASE_URL: z.ZodOptional<z.ZodString>;
26864
- AZURE_OPENAI_DEPLOYMENT_NAME: z.ZodOptional<z.ZodString>;
26865
- AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME: z.ZodOptional<z.ZodString>;
26866
- AZURE_TENANT_ID: z.ZodOptional<z.ZodString>;
26867
- AZURE_TOKEN_SCOPE: z.ZodOptional<z.ZodString>;
26868
- BAM_API_HOST: z.ZodOptional<z.ZodString>;
26869
- BAM_API_KEY: z.ZodOptional<z.ZodString>;
26870
- CLOUDFLARE_ACCOUNT_ID: z.ZodOptional<z.ZodString>;
26871
- CLOUDFLARE_API_KEY: z.ZodOptional<z.ZodString>;
26872
- COHERE_API_KEY: z.ZodOptional<z.ZodString>;
26873
- COHERE_CLIENT_NAME: z.ZodOptional<z.ZodString>;
26874
- DATABRICKS_TOKEN: z.ZodOptional<z.ZodString>;
26875
- DATABRICKS_WORKSPACE_URL: z.ZodOptional<z.ZodString>;
26876
- FAL_KEY: z.ZodOptional<z.ZodString>;
26877
- GOOGLE_API_HOST: z.ZodOptional<z.ZodString>;
26878
- GOOGLE_API_KEY: z.ZodOptional<z.ZodString>;
26879
- GROQ_API_KEY: z.ZodOptional<z.ZodString>;
26880
- HELICONE_API_KEY: z.ZodOptional<z.ZodString>;
26881
- HF_API_TOKEN: z.ZodOptional<z.ZodString>;
26882
- HF_TOKEN: z.ZodOptional<z.ZodString>;
26883
- HUGGING_FACE_HUB_TOKEN: z.ZodOptional<z.ZodString>;
26884
- JFROG_API_KEY: z.ZodOptional<z.ZodString>;
26885
- LAMBDA_API_KEY: z.ZodOptional<z.ZodString>;
26886
- LANGFUSE_HOST: z.ZodOptional<z.ZodString>;
26887
- LANGFUSE_PUBLIC_KEY: z.ZodOptional<z.ZodString>;
26888
- LANGFUSE_SECRET_KEY: z.ZodOptional<z.ZodString>;
26889
- LITELLM_API_BASE: z.ZodOptional<z.ZodString>;
26890
- LLAMA_BASE_URL: z.ZodOptional<z.ZodString>;
26891
- LOCALAI_BASE_URL: z.ZodOptional<z.ZodString>;
26892
- MISTRAL_API_BASE_URL: z.ZodOptional<z.ZodString>;
26893
- MISTRAL_API_HOST: z.ZodOptional<z.ZodString>;
26894
- MISTRAL_API_KEY: z.ZodOptional<z.ZodString>;
26895
- OLLAMA_API_KEY: z.ZodOptional<z.ZodString>;
26896
- OLLAMA_BASE_URL: z.ZodOptional<z.ZodString>;
26897
- OPENAI_API_BASE_URL: z.ZodOptional<z.ZodString>;
26898
- OPENAI_API_HOST: z.ZodOptional<z.ZodString>;
26899
- OPENAI_API_KEY: z.ZodOptional<z.ZodString>;
26900
- OPENAI_BASE_URL: z.ZodOptional<z.ZodString>;
26901
- OPENAI_ORGANIZATION: z.ZodOptional<z.ZodString>;
26902
- PALM_API_HOST: z.ZodOptional<z.ZodString>;
26903
- PALM_API_KEY: z.ZodOptional<z.ZodString>;
26904
- PORTKEY_API_KEY: z.ZodOptional<z.ZodString>;
26905
- PROMPTFOO_CA_CERT_PATH: z.ZodOptional<z.ZodString>;
26906
- PROMPTFOO_INSECURE_SSL: z.ZodOptional<z.ZodString>;
26907
- REPLICATE_API_KEY: z.ZodOptional<z.ZodString>;
26908
- REPLICATE_API_TOKEN: z.ZodOptional<z.ZodString>;
26909
- VERTEX_API_HOST: z.ZodOptional<z.ZodString>;
26910
- VERTEX_API_KEY: z.ZodOptional<z.ZodString>;
26911
- VERTEX_API_VERSION: z.ZodOptional<z.ZodString>;
26912
- VERTEX_PROJECT_ID: z.ZodOptional<z.ZodString>;
26913
- VERTEX_PUBLISHER: z.ZodOptional<z.ZodString>;
26914
- VERTEX_REGION: z.ZodOptional<z.ZodString>;
26915
- VOYAGE_API_BASE_URL: z.ZodOptional<z.ZodString>;
26916
- VOYAGE_API_KEY: z.ZodOptional<z.ZodString>;
26917
- WATSONX_AI_APIKEY: z.ZodOptional<z.ZodString>;
26918
- WATSONX_AI_AUTH_TYPE: z.ZodOptional<z.ZodString>;
26919
- WATSONX_AI_BEARER_TOKEN: z.ZodOptional<z.ZodString>;
26920
- WATSONX_AI_PROJECT_ID: z.ZodOptional<z.ZodString>;
26921
- AZURE_CONTENT_SAFETY_ENDPOINT: z.ZodOptional<z.ZodString>;
26922
- AZURE_CONTENT_SAFETY_API_KEY: z.ZodOptional<z.ZodString>;
26923
- AZURE_CONTENT_SAFETY_API_VERSION: z.ZodOptional<z.ZodString>;
26924
- AWS_REGION: z.ZodOptional<z.ZodString>;
26925
- AWS_DEFAULT_REGION: z.ZodOptional<z.ZodString>;
26926
- AWS_SAGEMAKER_MAX_TOKENS: z.ZodOptional<z.ZodString>;
26927
- AWS_SAGEMAKER_TEMPERATURE: z.ZodOptional<z.ZodString>;
26928
- AWS_SAGEMAKER_TOP_P: z.ZodOptional<z.ZodString>;
26929
- AWS_SAGEMAKER_MAX_RETRIES: z.ZodOptional<z.ZodString>;
26930
- PROMPTFOO_EVAL_TIMEOUT_MS: z.ZodOptional<z.ZodString>;
26931
- }, "strip", z.ZodTypeAny, {
26932
- AI21_API_BASE_URL?: string | undefined;
26933
- AI21_API_KEY?: string | undefined;
26934
- ANTHROPIC_API_KEY?: string | undefined;
26935
- ANTHROPIC_BASE_URL?: string | undefined;
26936
- AWS_BEDROCK_REGION?: string | undefined;
26937
- AZURE_API_BASE_URL?: string | undefined;
26938
- AZURE_API_HOST?: string | undefined;
26939
- AZURE_API_KEY?: string | undefined;
26940
- AZURE_AUTHORITY_HOST?: string | undefined;
26941
- AZURE_CLIENT_ID?: string | undefined;
26942
- AZURE_CLIENT_SECRET?: string | undefined;
26943
- AZURE_DEPLOYMENT_NAME?: string | undefined;
26944
- AZURE_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
26945
- AZURE_OPENAI_API_BASE_URL?: string | undefined;
26946
- AZURE_OPENAI_API_HOST?: string | undefined;
26947
- AZURE_OPENAI_API_KEY?: string | undefined;
26948
- AZURE_OPENAI_BASE_URL?: string | undefined;
26949
- AZURE_OPENAI_DEPLOYMENT_NAME?: string | undefined;
26950
- AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
26951
- AZURE_TENANT_ID?: string | undefined;
26952
- AZURE_TOKEN_SCOPE?: string | undefined;
26953
- BAM_API_HOST?: string | undefined;
26954
- BAM_API_KEY?: string | undefined;
26955
- CLOUDFLARE_ACCOUNT_ID?: string | undefined;
26956
- CLOUDFLARE_API_KEY?: string | undefined;
26957
- COHERE_API_KEY?: string | undefined;
26958
- COHERE_CLIENT_NAME?: string | undefined;
26959
- DATABRICKS_TOKEN?: string | undefined;
26960
- DATABRICKS_WORKSPACE_URL?: string | undefined;
26961
- FAL_KEY?: string | undefined;
26962
- GOOGLE_API_HOST?: string | undefined;
26963
- GOOGLE_API_KEY?: string | undefined;
26964
- GROQ_API_KEY?: string | undefined;
26965
- HELICONE_API_KEY?: string | undefined;
26966
- HF_API_TOKEN?: string | undefined;
26967
- HF_TOKEN?: string | undefined;
26968
- HUGGING_FACE_HUB_TOKEN?: string | undefined;
26969
- JFROG_API_KEY?: string | undefined;
26970
- LAMBDA_API_KEY?: string | undefined;
26971
- LANGFUSE_HOST?: string | undefined;
26972
- LANGFUSE_PUBLIC_KEY?: string | undefined;
26973
- LANGFUSE_SECRET_KEY?: string | undefined;
26974
- LITELLM_API_BASE?: string | undefined;
26975
- LLAMA_BASE_URL?: string | undefined;
26976
- LOCALAI_BASE_URL?: string | undefined;
26977
- MISTRAL_API_BASE_URL?: string | undefined;
26978
- MISTRAL_API_HOST?: string | undefined;
26979
- MISTRAL_API_KEY?: string | undefined;
26980
- OLLAMA_API_KEY?: string | undefined;
26981
- OLLAMA_BASE_URL?: string | undefined;
26982
- OPENAI_API_BASE_URL?: string | undefined;
26983
- OPENAI_API_HOST?: string | undefined;
26984
- OPENAI_API_KEY?: string | undefined;
26985
- OPENAI_BASE_URL?: string | undefined;
26986
- OPENAI_ORGANIZATION?: string | undefined;
26987
- PALM_API_HOST?: string | undefined;
26988
- PALM_API_KEY?: string | undefined;
26989
- PORTKEY_API_KEY?: string | undefined;
26990
- PROMPTFOO_CA_CERT_PATH?: string | undefined;
26991
- PROMPTFOO_INSECURE_SSL?: string | undefined;
26992
- REPLICATE_API_KEY?: string | undefined;
26993
- REPLICATE_API_TOKEN?: string | undefined;
26994
- VERTEX_API_HOST?: string | undefined;
26995
- VERTEX_API_KEY?: string | undefined;
26996
- VERTEX_API_VERSION?: string | undefined;
26997
- VERTEX_PROJECT_ID?: string | undefined;
26998
- VERTEX_PUBLISHER?: string | undefined;
26999
- VERTEX_REGION?: string | undefined;
27000
- VOYAGE_API_BASE_URL?: string | undefined;
27001
- VOYAGE_API_KEY?: string | undefined;
27002
- WATSONX_AI_APIKEY?: string | undefined;
27003
- WATSONX_AI_AUTH_TYPE?: string | undefined;
27004
- WATSONX_AI_BEARER_TOKEN?: string | undefined;
27005
- WATSONX_AI_PROJECT_ID?: string | undefined;
27006
- AZURE_CONTENT_SAFETY_ENDPOINT?: string | undefined;
27007
- AZURE_CONTENT_SAFETY_API_KEY?: string | undefined;
27008
- AZURE_CONTENT_SAFETY_API_VERSION?: string | undefined;
27009
- AWS_REGION?: string | undefined;
27010
- AWS_DEFAULT_REGION?: string | undefined;
27011
- AWS_SAGEMAKER_MAX_TOKENS?: string | undefined;
27012
- AWS_SAGEMAKER_TEMPERATURE?: string | undefined;
27013
- AWS_SAGEMAKER_TOP_P?: string | undefined;
27014
- AWS_SAGEMAKER_MAX_RETRIES?: string | undefined;
27015
- PROMPTFOO_EVAL_TIMEOUT_MS?: string | undefined;
27016
- }, {
27017
- AI21_API_BASE_URL?: string | undefined;
27018
- AI21_API_KEY?: string | undefined;
27019
- ANTHROPIC_API_KEY?: string | undefined;
27020
- ANTHROPIC_BASE_URL?: string | undefined;
27021
- AWS_BEDROCK_REGION?: string | undefined;
27022
- AZURE_API_BASE_URL?: string | undefined;
27023
- AZURE_API_HOST?: string | undefined;
27024
- AZURE_API_KEY?: string | undefined;
27025
- AZURE_AUTHORITY_HOST?: string | undefined;
27026
- AZURE_CLIENT_ID?: string | undefined;
27027
- AZURE_CLIENT_SECRET?: string | undefined;
27028
- AZURE_DEPLOYMENT_NAME?: string | undefined;
27029
- AZURE_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
27030
- AZURE_OPENAI_API_BASE_URL?: string | undefined;
27031
- AZURE_OPENAI_API_HOST?: string | undefined;
27032
- AZURE_OPENAI_API_KEY?: string | undefined;
27033
- AZURE_OPENAI_BASE_URL?: string | undefined;
27034
- AZURE_OPENAI_DEPLOYMENT_NAME?: string | undefined;
27035
- AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
27036
- AZURE_TENANT_ID?: string | undefined;
27037
- AZURE_TOKEN_SCOPE?: string | undefined;
27038
- BAM_API_HOST?: string | undefined;
27039
- BAM_API_KEY?: string | undefined;
27040
- CLOUDFLARE_ACCOUNT_ID?: string | undefined;
27041
- CLOUDFLARE_API_KEY?: string | undefined;
27042
- COHERE_API_KEY?: string | undefined;
27043
- COHERE_CLIENT_NAME?: string | undefined;
27044
- DATABRICKS_TOKEN?: string | undefined;
27045
- DATABRICKS_WORKSPACE_URL?: string | undefined;
27046
- FAL_KEY?: string | undefined;
27047
- GOOGLE_API_HOST?: string | undefined;
27048
- GOOGLE_API_KEY?: string | undefined;
27049
- GROQ_API_KEY?: string | undefined;
27050
- HELICONE_API_KEY?: string | undefined;
27051
- HF_API_TOKEN?: string | undefined;
27052
- HF_TOKEN?: string | undefined;
27053
- HUGGING_FACE_HUB_TOKEN?: string | undefined;
27054
- JFROG_API_KEY?: string | undefined;
27055
- LAMBDA_API_KEY?: string | undefined;
27056
- LANGFUSE_HOST?: string | undefined;
27057
- LANGFUSE_PUBLIC_KEY?: string | undefined;
27058
- LANGFUSE_SECRET_KEY?: string | undefined;
27059
- LITELLM_API_BASE?: string | undefined;
27060
- LLAMA_BASE_URL?: string | undefined;
27061
- LOCALAI_BASE_URL?: string | undefined;
27062
- MISTRAL_API_BASE_URL?: string | undefined;
27063
- MISTRAL_API_HOST?: string | undefined;
27064
- MISTRAL_API_KEY?: string | undefined;
27065
- OLLAMA_API_KEY?: string | undefined;
27066
- OLLAMA_BASE_URL?: string | undefined;
27067
- OPENAI_API_BASE_URL?: string | undefined;
27068
- OPENAI_API_HOST?: string | undefined;
27069
- OPENAI_API_KEY?: string | undefined;
27070
- OPENAI_BASE_URL?: string | undefined;
27071
- OPENAI_ORGANIZATION?: string | undefined;
27072
- PALM_API_HOST?: string | undefined;
27073
- PALM_API_KEY?: string | undefined;
27074
- PORTKEY_API_KEY?: string | undefined;
27075
- PROMPTFOO_CA_CERT_PATH?: string | undefined;
27076
- PROMPTFOO_INSECURE_SSL?: string | undefined;
27077
- REPLICATE_API_KEY?: string | undefined;
27078
- REPLICATE_API_TOKEN?: string | undefined;
27079
- VERTEX_API_HOST?: string | undefined;
27080
- VERTEX_API_KEY?: string | undefined;
27081
- VERTEX_API_VERSION?: string | undefined;
27082
- VERTEX_PROJECT_ID?: string | undefined;
27083
- VERTEX_PUBLISHER?: string | undefined;
27084
- VERTEX_REGION?: string | undefined;
27085
- VOYAGE_API_BASE_URL?: string | undefined;
27086
- VOYAGE_API_KEY?: string | undefined;
27087
- WATSONX_AI_APIKEY?: string | undefined;
27088
- WATSONX_AI_AUTH_TYPE?: string | undefined;
27089
- WATSONX_AI_BEARER_TOKEN?: string | undefined;
27090
- WATSONX_AI_PROJECT_ID?: string | undefined;
27091
- AZURE_CONTENT_SAFETY_ENDPOINT?: string | undefined;
27092
- AZURE_CONTENT_SAFETY_API_KEY?: string | undefined;
27093
- AZURE_CONTENT_SAFETY_API_VERSION?: string | undefined;
27094
- AWS_REGION?: string | undefined;
27095
- AWS_DEFAULT_REGION?: string | undefined;
27096
- AWS_SAGEMAKER_MAX_TOKENS?: string | undefined;
27097
- AWS_SAGEMAKER_TEMPERATURE?: string | undefined;
27098
- AWS_SAGEMAKER_TOP_P?: string | undefined;
27099
- AWS_SAGEMAKER_MAX_RETRIES?: string | undefined;
27100
- PROMPTFOO_EVAL_TIMEOUT_MS?: string | undefined;
27101
- }>>;
27102
- }, "strict", z.ZodTypeAny, {
27103
- id?: string | undefined;
27104
- config?: any;
27105
- label?: string | undefined;
27106
- prompts?: string[] | undefined;
27107
- transform?: string | undefined;
27108
- delay?: number | undefined;
27109
- env?: {
27110
- AI21_API_BASE_URL?: string | undefined;
27111
- AI21_API_KEY?: string | undefined;
27112
- ANTHROPIC_API_KEY?: string | undefined;
27113
- ANTHROPIC_BASE_URL?: string | undefined;
27114
- AWS_BEDROCK_REGION?: string | undefined;
27115
- AZURE_API_BASE_URL?: string | undefined;
27116
- AZURE_API_HOST?: string | undefined;
27117
- AZURE_API_KEY?: string | undefined;
27118
- AZURE_AUTHORITY_HOST?: string | undefined;
27119
- AZURE_CLIENT_ID?: string | undefined;
27120
- AZURE_CLIENT_SECRET?: string | undefined;
27121
- AZURE_DEPLOYMENT_NAME?: string | undefined;
27122
- AZURE_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
27123
- AZURE_OPENAI_API_BASE_URL?: string | undefined;
27124
- AZURE_OPENAI_API_HOST?: string | undefined;
27125
- AZURE_OPENAI_API_KEY?: string | undefined;
27126
- AZURE_OPENAI_BASE_URL?: string | undefined;
27127
- AZURE_OPENAI_DEPLOYMENT_NAME?: string | undefined;
27128
- AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
27129
- AZURE_TENANT_ID?: string | undefined;
27130
- AZURE_TOKEN_SCOPE?: string | undefined;
27131
- BAM_API_HOST?: string | undefined;
27132
- BAM_API_KEY?: string | undefined;
27133
- CLOUDFLARE_ACCOUNT_ID?: string | undefined;
27134
- CLOUDFLARE_API_KEY?: string | undefined;
27135
- COHERE_API_KEY?: string | undefined;
27136
- COHERE_CLIENT_NAME?: string | undefined;
27137
- DATABRICKS_TOKEN?: string | undefined;
27138
- DATABRICKS_WORKSPACE_URL?: string | undefined;
27139
- FAL_KEY?: string | undefined;
27140
- GOOGLE_API_HOST?: string | undefined;
27141
- GOOGLE_API_KEY?: string | undefined;
27142
- GROQ_API_KEY?: string | undefined;
27143
- HELICONE_API_KEY?: string | undefined;
27144
- HF_API_TOKEN?: string | undefined;
27145
- HF_TOKEN?: string | undefined;
27146
- HUGGING_FACE_HUB_TOKEN?: string | undefined;
27147
- JFROG_API_KEY?: string | undefined;
27148
- LAMBDA_API_KEY?: string | undefined;
27149
- LANGFUSE_HOST?: string | undefined;
27150
- LANGFUSE_PUBLIC_KEY?: string | undefined;
27151
- LANGFUSE_SECRET_KEY?: string | undefined;
27152
- LITELLM_API_BASE?: string | undefined;
27153
- LLAMA_BASE_URL?: string | undefined;
27154
- LOCALAI_BASE_URL?: string | undefined;
27155
- MISTRAL_API_BASE_URL?: string | undefined;
27156
- MISTRAL_API_HOST?: string | undefined;
27157
- MISTRAL_API_KEY?: string | undefined;
27158
- OLLAMA_API_KEY?: string | undefined;
27159
- OLLAMA_BASE_URL?: string | undefined;
27160
- OPENAI_API_BASE_URL?: string | undefined;
27161
- OPENAI_API_HOST?: string | undefined;
27162
- OPENAI_API_KEY?: string | undefined;
27163
- OPENAI_BASE_URL?: string | undefined;
27164
- OPENAI_ORGANIZATION?: string | undefined;
27165
- PALM_API_HOST?: string | undefined;
27166
- PALM_API_KEY?: string | undefined;
27167
- PORTKEY_API_KEY?: string | undefined;
27168
- PROMPTFOO_CA_CERT_PATH?: string | undefined;
27169
- PROMPTFOO_INSECURE_SSL?: string | undefined;
27170
- REPLICATE_API_KEY?: string | undefined;
27171
- REPLICATE_API_TOKEN?: string | undefined;
27172
- VERTEX_API_HOST?: string | undefined;
27173
- VERTEX_API_KEY?: string | undefined;
27174
- VERTEX_API_VERSION?: string | undefined;
27175
- VERTEX_PROJECT_ID?: string | undefined;
27176
- VERTEX_PUBLISHER?: string | undefined;
27177
- VERTEX_REGION?: string | undefined;
27178
- VOYAGE_API_BASE_URL?: string | undefined;
27179
- VOYAGE_API_KEY?: string | undefined;
27180
- WATSONX_AI_APIKEY?: string | undefined;
27181
- WATSONX_AI_AUTH_TYPE?: string | undefined;
27182
- WATSONX_AI_BEARER_TOKEN?: string | undefined;
27183
- WATSONX_AI_PROJECT_ID?: string | undefined;
27184
- AZURE_CONTENT_SAFETY_ENDPOINT?: string | undefined;
27185
- AZURE_CONTENT_SAFETY_API_KEY?: string | undefined;
27186
- AZURE_CONTENT_SAFETY_API_VERSION?: string | undefined;
27187
- AWS_REGION?: string | undefined;
27188
- AWS_DEFAULT_REGION?: string | undefined;
27189
- AWS_SAGEMAKER_MAX_TOKENS?: string | undefined;
27190
- AWS_SAGEMAKER_TEMPERATURE?: string | undefined;
27191
- AWS_SAGEMAKER_TOP_P?: string | undefined;
27192
- AWS_SAGEMAKER_MAX_RETRIES?: string | undefined;
27193
- PROMPTFOO_EVAL_TIMEOUT_MS?: string | undefined;
27194
- } | undefined;
27195
- }, {
27196
- id?: string | undefined;
27197
- config?: any;
27198
- label?: string | undefined;
27199
- prompts?: string[] | undefined;
27200
- transform?: string | undefined;
27201
- delay?: number | undefined;
27202
- env?: {
27203
- AI21_API_BASE_URL?: string | undefined;
27204
- AI21_API_KEY?: string | undefined;
27205
- ANTHROPIC_API_KEY?: string | undefined;
27206
- ANTHROPIC_BASE_URL?: string | undefined;
27207
- AWS_BEDROCK_REGION?: string | undefined;
27208
- AZURE_API_BASE_URL?: string | undefined;
27209
- AZURE_API_HOST?: string | undefined;
27210
- AZURE_API_KEY?: string | undefined;
27211
- AZURE_AUTHORITY_HOST?: string | undefined;
27212
- AZURE_CLIENT_ID?: string | undefined;
27213
- AZURE_CLIENT_SECRET?: string | undefined;
27214
- AZURE_DEPLOYMENT_NAME?: string | undefined;
27215
- AZURE_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
27216
- AZURE_OPENAI_API_BASE_URL?: string | undefined;
27217
- AZURE_OPENAI_API_HOST?: string | undefined;
27218
- AZURE_OPENAI_API_KEY?: string | undefined;
27219
- AZURE_OPENAI_BASE_URL?: string | undefined;
27220
- AZURE_OPENAI_DEPLOYMENT_NAME?: string | undefined;
27221
- AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
27222
- AZURE_TENANT_ID?: string | undefined;
27223
- AZURE_TOKEN_SCOPE?: string | undefined;
27224
- BAM_API_HOST?: string | undefined;
27225
- BAM_API_KEY?: string | undefined;
27226
- CLOUDFLARE_ACCOUNT_ID?: string | undefined;
27227
- CLOUDFLARE_API_KEY?: string | undefined;
27228
- COHERE_API_KEY?: string | undefined;
27229
- COHERE_CLIENT_NAME?: string | undefined;
27230
- DATABRICKS_TOKEN?: string | undefined;
27231
- DATABRICKS_WORKSPACE_URL?: string | undefined;
27232
- FAL_KEY?: string | undefined;
27233
- GOOGLE_API_HOST?: string | undefined;
27234
- GOOGLE_API_KEY?: string | undefined;
27235
- GROQ_API_KEY?: string | undefined;
27236
- HELICONE_API_KEY?: string | undefined;
27237
- HF_API_TOKEN?: string | undefined;
27238
- HF_TOKEN?: string | undefined;
27239
- HUGGING_FACE_HUB_TOKEN?: string | undefined;
27240
- JFROG_API_KEY?: string | undefined;
27241
- LAMBDA_API_KEY?: string | undefined;
27242
- LANGFUSE_HOST?: string | undefined;
27243
- LANGFUSE_PUBLIC_KEY?: string | undefined;
27244
- LANGFUSE_SECRET_KEY?: string | undefined;
27245
- LITELLM_API_BASE?: string | undefined;
27246
- LLAMA_BASE_URL?: string | undefined;
27247
- LOCALAI_BASE_URL?: string | undefined;
27248
- MISTRAL_API_BASE_URL?: string | undefined;
27249
- MISTRAL_API_HOST?: string | undefined;
27250
- MISTRAL_API_KEY?: string | undefined;
27251
- OLLAMA_API_KEY?: string | undefined;
27252
- OLLAMA_BASE_URL?: string | undefined;
27253
- OPENAI_API_BASE_URL?: string | undefined;
27254
- OPENAI_API_HOST?: string | undefined;
27255
- OPENAI_API_KEY?: string | undefined;
27256
- OPENAI_BASE_URL?: string | undefined;
27257
- OPENAI_ORGANIZATION?: string | undefined;
27258
- PALM_API_HOST?: string | undefined;
27259
- PALM_API_KEY?: string | undefined;
27260
- PORTKEY_API_KEY?: string | undefined;
27261
- PROMPTFOO_CA_CERT_PATH?: string | undefined;
27262
- PROMPTFOO_INSECURE_SSL?: string | undefined;
27263
- REPLICATE_API_KEY?: string | undefined;
27264
- REPLICATE_API_TOKEN?: string | undefined;
27265
- VERTEX_API_HOST?: string | undefined;
27266
- VERTEX_API_KEY?: string | undefined;
27267
- VERTEX_API_VERSION?: string | undefined;
27268
- VERTEX_PROJECT_ID?: string | undefined;
27269
- VERTEX_PUBLISHER?: string | undefined;
27270
- VERTEX_REGION?: string | undefined;
27271
- VOYAGE_API_BASE_URL?: string | undefined;
27272
- VOYAGE_API_KEY?: string | undefined;
27273
- WATSONX_AI_APIKEY?: string | undefined;
27274
- WATSONX_AI_AUTH_TYPE?: string | undefined;
27275
- WATSONX_AI_BEARER_TOKEN?: string | undefined;
27276
- WATSONX_AI_PROJECT_ID?: string | undefined;
27277
- AZURE_CONTENT_SAFETY_ENDPOINT?: string | undefined;
27278
- AZURE_CONTENT_SAFETY_API_KEY?: string | undefined;
27279
- AZURE_CONTENT_SAFETY_API_VERSION?: string | undefined;
27280
- AWS_REGION?: string | undefined;
27281
- AWS_DEFAULT_REGION?: string | undefined;
27282
- AWS_SAGEMAKER_MAX_TOKENS?: string | undefined;
27283
- AWS_SAGEMAKER_TEMPERATURE?: string | undefined;
27284
- AWS_SAGEMAKER_TOP_P?: string | undefined;
27285
- AWS_SAGEMAKER_MAX_RETRIES?: string | undefined;
27286
- PROMPTFOO_EVAL_TIMEOUT_MS?: string | undefined;
27287
- } | undefined;
27288
- }>, z.ZodIntersection<z.ZodFunction<z.ZodTuple<[z.ZodString, z.ZodOptional<z.ZodObject<{
27289
- fetchWithCache: z.ZodOptional<z.ZodAny>;
27290
- filters: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodFunction<z.ZodTuple<[z.ZodAny], z.ZodAny>, z.ZodString>>>;
27291
- getCache: z.ZodOptional<z.ZodAny>;
27292
- logger: z.ZodOptional<z.ZodAny>;
27293
- originalProvider: z.ZodOptional<z.ZodAny>;
27294
- prompt: z.ZodObject<{
27295
- id: z.ZodOptional<z.ZodString>;
27296
- raw: z.ZodString;
27297
- display: z.ZodOptional<z.ZodString>;
27298
- label: z.ZodString;
27299
- function: z.ZodOptional<z.ZodFunction<z.ZodTuple<[z.ZodObject<{
27300
- vars: z.ZodRecord<z.ZodString, z.ZodUnion<[z.ZodString, z.ZodAny]>>;
27301
- provider: z.ZodOptional<z.ZodType<ApiProvider, z.ZodTypeDef, ApiProvider>>;
27302
- }, "strip", z.ZodTypeAny, {
27303
- vars: Record<string, any>;
27304
- provider?: ApiProvider | undefined;
27305
- }, {
27306
- vars: Record<string, any>;
27307
- provider?: ApiProvider | undefined;
27308
- }>], z.ZodUnknown>, z.ZodPromise<z.ZodUnion<[z.ZodString, z.ZodAny]>>>>;
27309
- config: z.ZodOptional<z.ZodAny>;
27310
- }, "strip", z.ZodTypeAny, {
27311
- raw: string;
27312
- label: string;
27313
- function?: ((args_0: {
27314
- vars: Record<string, any>;
27315
- provider?: ApiProvider | undefined;
27316
- }, ...args: unknown[]) => Promise<any>) | undefined;
27317
- id?: string | undefined;
27318
- config?: any;
27319
- display?: string | undefined;
27320
- }, {
27321
- raw: string;
27322
- label: string;
27323
- function?: ((args_0: {
27324
- vars: Record<string, any>;
27325
- provider?: ApiProvider | undefined;
27326
- }, ...args: unknown[]) => Promise<any>) | undefined;
27327
- id?: string | undefined;
27328
- config?: any;
27329
- display?: string | undefined;
27330
- }>;
27331
- vars: z.ZodRecord<z.ZodString, z.ZodUnion<[z.ZodString, z.ZodObject<{}, "strip", z.ZodTypeAny, {}, {}>]>>;
27332
- }, "strip", z.ZodTypeAny, {
27333
- prompt: {
27334
- raw: string;
27335
- label: string;
27336
- function?: ((args_0: {
27337
- vars: Record<string, any>;
27338
- provider?: ApiProvider | undefined;
27339
- }, ...args: unknown[]) => Promise<any>) | undefined;
25832
+ LANGFUSE_PUBLIC_KEY?: string | undefined;
25833
+ LANGFUSE_SECRET_KEY?: string | undefined;
25834
+ LITELLM_API_BASE?: string | undefined;
25835
+ LLAMA_BASE_URL?: string | undefined;
25836
+ LOCALAI_BASE_URL?: string | undefined;
25837
+ MISTRAL_API_BASE_URL?: string | undefined;
25838
+ MISTRAL_API_HOST?: string | undefined;
25839
+ MISTRAL_API_KEY?: string | undefined;
25840
+ OLLAMA_API_KEY?: string | undefined;
25841
+ OLLAMA_BASE_URL?: string | undefined;
25842
+ OPENAI_API_BASE_URL?: string | undefined;
25843
+ OPENAI_API_HOST?: string | undefined;
25844
+ OPENAI_API_KEY?: string | undefined;
25845
+ OPENAI_BASE_URL?: string | undefined;
25846
+ OPENAI_ORGANIZATION?: string | undefined;
25847
+ PALM_API_HOST?: string | undefined;
25848
+ PALM_API_KEY?: string | undefined;
25849
+ PORTKEY_API_KEY?: string | undefined;
25850
+ PROMPTFOO_CA_CERT_PATH?: string | undefined;
25851
+ PROMPTFOO_INSECURE_SSL?: string | undefined;
25852
+ REPLICATE_API_KEY?: string | undefined;
25853
+ REPLICATE_API_TOKEN?: string | undefined;
25854
+ VERTEX_API_HOST?: string | undefined;
25855
+ VERTEX_API_KEY?: string | undefined;
25856
+ VERTEX_API_VERSION?: string | undefined;
25857
+ VERTEX_PROJECT_ID?: string | undefined;
25858
+ VERTEX_PUBLISHER?: string | undefined;
25859
+ VERTEX_REGION?: string | undefined;
25860
+ VOYAGE_API_BASE_URL?: string | undefined;
25861
+ VOYAGE_API_KEY?: string | undefined;
25862
+ WATSONX_AI_APIKEY?: string | undefined;
25863
+ WATSONX_AI_AUTH_TYPE?: string | undefined;
25864
+ WATSONX_AI_BEARER_TOKEN?: string | undefined;
25865
+ WATSONX_AI_PROJECT_ID?: string | undefined;
25866
+ AZURE_CONTENT_SAFETY_ENDPOINT?: string | undefined;
25867
+ AZURE_CONTENT_SAFETY_API_KEY?: string | undefined;
25868
+ AZURE_CONTENT_SAFETY_API_VERSION?: string | undefined;
25869
+ AWS_REGION?: string | undefined;
25870
+ AWS_DEFAULT_REGION?: string | undefined;
25871
+ AWS_SAGEMAKER_MAX_TOKENS?: string | undefined;
25872
+ AWS_SAGEMAKER_TEMPERATURE?: string | undefined;
25873
+ AWS_SAGEMAKER_TOP_P?: string | undefined;
25874
+ AWS_SAGEMAKER_MAX_RETRIES?: string | undefined;
25875
+ PROMPTFOO_EVAL_TIMEOUT_MS?: string | undefined;
25876
+ } | undefined;
25877
+ } | {
25878
+ callApi: import("./providers").CallApiFunction;
25879
+ id: (...args: unknown[]) => string;
25880
+ config?: any;
25881
+ label?: string | undefined;
25882
+ transform?: string | undefined;
25883
+ delay?: number | undefined;
25884
+ callEmbeddingApi?: ((args_0: string, ...args: unknown[]) => Promise<import("./providers").ProviderEmbeddingResponse>) | undefined;
25885
+ callClassificationApi?: ((args_0: string, ...args: unknown[]) => Promise<import("./providers").ProviderClassificationResponse>) | undefined;
25886
+ } | undefined;
25887
+ metadata?: (Record<string, any> & {
25888
+ pluginConfig?: PluginConfig | undefined;
25889
+ strategyConfig?: import("../redteam/types").RedteamObjectConfig | undefined;
25890
+ }) | undefined;
25891
+ description?: string | undefined;
25892
+ providerOutput?: string | {} | undefined;
25893
+ assert?: ({
25894
+ type: "moderation" | `promptfoo:redteam:${string}` | "cost" | "factuality" | "answer-relevance" | "bleu" | "classifier" | "contains" | "contains-all" | "contains-any" | "contains-json" | "contains-sql" | "contains-xml" | "context-faithfulness" | "context-recall" | "context-relevance" | "equals" | "g-eval" | "gleu" | "guardrails" | "icontains" | "icontains-all" | "icontains-any" | "is-json" | "is-refusal" | "is-sql" | "is-valid-function-call" | "is-valid-openai-function-call" | "is-valid-openai-tools-call" | "is-xml" | "javascript" | "latency" | "levenshtein" | "llm-rubric" | "pi" | "meteor" | "model-graded-closedqa" | "model-graded-factuality" | "perplexity" | "perplexity-score" | "python" | "regex" | "rouge-n" | "similar" | "starts-with" | "webhook" | "select-best" | "human";
25895
+ value?: AssertionValue | undefined;
25896
+ config?: Record<string, any> | undefined;
25897
+ provider?: any;
25898
+ transform?: string | undefined;
25899
+ rubricPrompt?: string | string[] | {
25900
+ role: string;
25901
+ content: string;
25902
+ }[] | undefined;
25903
+ threshold?: number | undefined;
25904
+ weight?: number | undefined;
25905
+ metric?: string | undefined;
25906
+ } | {
25907
+ type: "assert-set";
25908
+ assert: {
25909
+ type: "moderation" | `promptfoo:redteam:${string}` | "cost" | "factuality" | "answer-relevance" | "bleu" | "classifier" | "contains" | "contains-all" | "contains-any" | "contains-json" | "contains-sql" | "contains-xml" | "context-faithfulness" | "context-recall" | "context-relevance" | "equals" | "g-eval" | "gleu" | "guardrails" | "icontains" | "icontains-all" | "icontains-any" | "is-json" | "is-refusal" | "is-sql" | "is-valid-function-call" | "is-valid-openai-function-call" | "is-valid-openai-tools-call" | "is-xml" | "javascript" | "latency" | "levenshtein" | "llm-rubric" | "pi" | "meteor" | "model-graded-closedqa" | "model-graded-factuality" | "perplexity" | "perplexity-score" | "python" | "regex" | "rouge-n" | "similar" | "starts-with" | "webhook" | "select-best" | "human";
25910
+ value?: AssertionValue | undefined;
25911
+ config?: Record<string, any> | undefined;
25912
+ provider?: any;
25913
+ transform?: string | undefined;
25914
+ rubricPrompt?: string | string[] | {
25915
+ role: string;
25916
+ content: string;
25917
+ }[] | undefined;
25918
+ threshold?: number | undefined;
25919
+ weight?: number | undefined;
25920
+ metric?: string | undefined;
25921
+ }[];
25922
+ config?: Record<string, any> | undefined;
25923
+ threshold?: number | undefined;
25924
+ weight?: number | undefined;
25925
+ metric?: string | undefined;
25926
+ })[] | undefined;
25927
+ threshold?: number | undefined;
25928
+ assertScoringFunction?: string | ScoringFunction | undefined;
25929
+ }[];
25930
+ tests: {
25931
+ options?: ({
25932
+ prefix?: string | undefined;
25933
+ suffix?: string | undefined;
25934
+ } & {
25935
+ transform?: string | undefined;
25936
+ postprocess?: string | undefined;
25937
+ transformVars?: string | undefined;
25938
+ storeOutputAs?: string | undefined;
25939
+ } & {
25940
+ provider?: any;
25941
+ rubricPrompt?: string | string[] | {
25942
+ role: string;
25943
+ content: string;
25944
+ }[] | undefined;
25945
+ factuality?: {
25946
+ subset?: number | undefined;
25947
+ superset?: number | undefined;
25948
+ agree?: number | undefined;
25949
+ disagree?: number | undefined;
25950
+ differButFactual?: number | undefined;
25951
+ } | undefined;
25952
+ } & {
25953
+ disableVarExpansion?: boolean | undefined;
25954
+ disableConversationVar?: boolean | undefined;
25955
+ runSerially?: boolean | undefined;
25956
+ }) | undefined;
25957
+ vars?: Record<string, string | number | boolean | any[] | Record<string, any> | (string | number | boolean)[]> | undefined;
25958
+ provider?: string | {
25959
+ id?: string | undefined;
25960
+ config?: any;
25961
+ label?: string | undefined;
25962
+ prompts?: string[] | undefined;
25963
+ transform?: string | undefined;
25964
+ delay?: number | undefined;
25965
+ env?: {
25966
+ AI21_API_BASE_URL?: string | undefined;
25967
+ AI21_API_KEY?: string | undefined;
25968
+ ANTHROPIC_API_KEY?: string | undefined;
25969
+ ANTHROPIC_BASE_URL?: string | undefined;
25970
+ AWS_BEDROCK_REGION?: string | undefined;
25971
+ AZURE_API_BASE_URL?: string | undefined;
25972
+ AZURE_API_HOST?: string | undefined;
25973
+ AZURE_API_KEY?: string | undefined;
25974
+ AZURE_AUTHORITY_HOST?: string | undefined;
25975
+ AZURE_CLIENT_ID?: string | undefined;
25976
+ AZURE_CLIENT_SECRET?: string | undefined;
25977
+ AZURE_DEPLOYMENT_NAME?: string | undefined;
25978
+ AZURE_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
25979
+ AZURE_OPENAI_API_BASE_URL?: string | undefined;
25980
+ AZURE_OPENAI_API_HOST?: string | undefined;
25981
+ AZURE_OPENAI_API_KEY?: string | undefined;
25982
+ AZURE_OPENAI_BASE_URL?: string | undefined;
25983
+ AZURE_OPENAI_DEPLOYMENT_NAME?: string | undefined;
25984
+ AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
25985
+ AZURE_TENANT_ID?: string | undefined;
25986
+ AZURE_TOKEN_SCOPE?: string | undefined;
25987
+ BAM_API_HOST?: string | undefined;
25988
+ BAM_API_KEY?: string | undefined;
25989
+ CLOUDFLARE_ACCOUNT_ID?: string | undefined;
25990
+ CLOUDFLARE_API_KEY?: string | undefined;
25991
+ COHERE_API_KEY?: string | undefined;
25992
+ COHERE_CLIENT_NAME?: string | undefined;
25993
+ DATABRICKS_TOKEN?: string | undefined;
25994
+ DATABRICKS_WORKSPACE_URL?: string | undefined;
25995
+ FAL_KEY?: string | undefined;
25996
+ GOOGLE_API_HOST?: string | undefined;
25997
+ GOOGLE_API_KEY?: string | undefined;
25998
+ GROQ_API_KEY?: string | undefined;
25999
+ HELICONE_API_KEY?: string | undefined;
26000
+ HF_API_TOKEN?: string | undefined;
26001
+ HF_TOKEN?: string | undefined;
26002
+ HUGGING_FACE_HUB_TOKEN?: string | undefined;
26003
+ JFROG_API_KEY?: string | undefined;
26004
+ LAMBDA_API_KEY?: string | undefined;
26005
+ LANGFUSE_HOST?: string | undefined;
26006
+ LANGFUSE_PUBLIC_KEY?: string | undefined;
26007
+ LANGFUSE_SECRET_KEY?: string | undefined;
26008
+ LITELLM_API_BASE?: string | undefined;
26009
+ LLAMA_BASE_URL?: string | undefined;
26010
+ LOCALAI_BASE_URL?: string | undefined;
26011
+ MISTRAL_API_BASE_URL?: string | undefined;
26012
+ MISTRAL_API_HOST?: string | undefined;
26013
+ MISTRAL_API_KEY?: string | undefined;
26014
+ OLLAMA_API_KEY?: string | undefined;
26015
+ OLLAMA_BASE_URL?: string | undefined;
26016
+ OPENAI_API_BASE_URL?: string | undefined;
26017
+ OPENAI_API_HOST?: string | undefined;
26018
+ OPENAI_API_KEY?: string | undefined;
26019
+ OPENAI_BASE_URL?: string | undefined;
26020
+ OPENAI_ORGANIZATION?: string | undefined;
26021
+ PALM_API_HOST?: string | undefined;
26022
+ PALM_API_KEY?: string | undefined;
26023
+ PORTKEY_API_KEY?: string | undefined;
26024
+ PROMPTFOO_CA_CERT_PATH?: string | undefined;
26025
+ PROMPTFOO_INSECURE_SSL?: string | undefined;
26026
+ REPLICATE_API_KEY?: string | undefined;
26027
+ REPLICATE_API_TOKEN?: string | undefined;
26028
+ VERTEX_API_HOST?: string | undefined;
26029
+ VERTEX_API_KEY?: string | undefined;
26030
+ VERTEX_API_VERSION?: string | undefined;
26031
+ VERTEX_PROJECT_ID?: string | undefined;
26032
+ VERTEX_PUBLISHER?: string | undefined;
26033
+ VERTEX_REGION?: string | undefined;
26034
+ VOYAGE_API_BASE_URL?: string | undefined;
26035
+ VOYAGE_API_KEY?: string | undefined;
26036
+ WATSONX_AI_APIKEY?: string | undefined;
26037
+ WATSONX_AI_AUTH_TYPE?: string | undefined;
26038
+ WATSONX_AI_BEARER_TOKEN?: string | undefined;
26039
+ WATSONX_AI_PROJECT_ID?: string | undefined;
26040
+ AZURE_CONTENT_SAFETY_ENDPOINT?: string | undefined;
26041
+ AZURE_CONTENT_SAFETY_API_KEY?: string | undefined;
26042
+ AZURE_CONTENT_SAFETY_API_VERSION?: string | undefined;
26043
+ AWS_REGION?: string | undefined;
26044
+ AWS_DEFAULT_REGION?: string | undefined;
26045
+ AWS_SAGEMAKER_MAX_TOKENS?: string | undefined;
26046
+ AWS_SAGEMAKER_TEMPERATURE?: string | undefined;
26047
+ AWS_SAGEMAKER_TOP_P?: string | undefined;
26048
+ AWS_SAGEMAKER_MAX_RETRIES?: string | undefined;
26049
+ PROMPTFOO_EVAL_TIMEOUT_MS?: string | undefined;
26050
+ } | undefined;
26051
+ } | {
26052
+ callApi: import("./providers").CallApiFunction;
26053
+ id: (...args: unknown[]) => string;
26054
+ config?: any;
26055
+ label?: string | undefined;
26056
+ transform?: string | undefined;
26057
+ delay?: number | undefined;
26058
+ callEmbeddingApi?: ((args_0: string, ...args: unknown[]) => Promise<import("./providers").ProviderEmbeddingResponse>) | undefined;
26059
+ callClassificationApi?: ((args_0: string, ...args: unknown[]) => Promise<import("./providers").ProviderClassificationResponse>) | undefined;
26060
+ } | undefined;
26061
+ metadata?: (Record<string, any> & {
26062
+ pluginConfig?: PluginConfig | undefined;
26063
+ strategyConfig?: import("../redteam/types").RedteamObjectConfig | undefined;
26064
+ }) | undefined;
26065
+ description?: string | undefined;
26066
+ providerOutput?: string | {} | undefined;
26067
+ assert?: ({
26068
+ type: "moderation" | `promptfoo:redteam:${string}` | "cost" | "factuality" | "answer-relevance" | "bleu" | "classifier" | "contains" | "contains-all" | "contains-any" | "contains-json" | "contains-sql" | "contains-xml" | "context-faithfulness" | "context-recall" | "context-relevance" | "equals" | "g-eval" | "gleu" | "guardrails" | "icontains" | "icontains-all" | "icontains-any" | "is-json" | "is-refusal" | "is-sql" | "is-valid-function-call" | "is-valid-openai-function-call" | "is-valid-openai-tools-call" | "is-xml" | "javascript" | "latency" | "levenshtein" | "llm-rubric" | "pi" | "meteor" | "model-graded-closedqa" | "model-graded-factuality" | "perplexity" | "perplexity-score" | "python" | "regex" | "rouge-n" | "similar" | "starts-with" | "webhook" | "select-best" | "human";
26069
+ value?: AssertionValue | undefined;
26070
+ config?: Record<string, any> | undefined;
26071
+ provider?: any;
26072
+ transform?: string | undefined;
26073
+ rubricPrompt?: string | string[] | {
26074
+ role: string;
26075
+ content: string;
26076
+ }[] | undefined;
26077
+ threshold?: number | undefined;
26078
+ weight?: number | undefined;
26079
+ metric?: string | undefined;
26080
+ } | {
26081
+ type: "assert-set";
26082
+ assert: {
26083
+ type: "moderation" | `promptfoo:redteam:${string}` | "cost" | "factuality" | "answer-relevance" | "bleu" | "classifier" | "contains" | "contains-all" | "contains-any" | "contains-json" | "contains-sql" | "contains-xml" | "context-faithfulness" | "context-recall" | "context-relevance" | "equals" | "g-eval" | "gleu" | "guardrails" | "icontains" | "icontains-all" | "icontains-any" | "is-json" | "is-refusal" | "is-sql" | "is-valid-function-call" | "is-valid-openai-function-call" | "is-valid-openai-tools-call" | "is-xml" | "javascript" | "latency" | "levenshtein" | "llm-rubric" | "pi" | "meteor" | "model-graded-closedqa" | "model-graded-factuality" | "perplexity" | "perplexity-score" | "python" | "regex" | "rouge-n" | "similar" | "starts-with" | "webhook" | "select-best" | "human";
26084
+ value?: AssertionValue | undefined;
26085
+ config?: Record<string, any> | undefined;
26086
+ provider?: any;
26087
+ transform?: string | undefined;
26088
+ rubricPrompt?: string | string[] | {
26089
+ role: string;
26090
+ content: string;
26091
+ }[] | undefined;
26092
+ threshold?: number | undefined;
26093
+ weight?: number | undefined;
26094
+ metric?: string | undefined;
26095
+ }[];
26096
+ config?: Record<string, any> | undefined;
26097
+ threshold?: number | undefined;
26098
+ weight?: number | undefined;
26099
+ metric?: string | undefined;
26100
+ })[] | undefined;
26101
+ threshold?: number | undefined;
26102
+ assertScoringFunction?: string | ScoringFunction | undefined;
26103
+ }[];
26104
+ description?: string | undefined;
26105
+ })[] | undefined;
26106
+ defaultTest?: {
26107
+ options?: ({
26108
+ prefix?: string | undefined;
26109
+ suffix?: string | undefined;
26110
+ } & {
26111
+ transform?: string | undefined;
26112
+ postprocess?: string | undefined;
26113
+ transformVars?: string | undefined;
26114
+ storeOutputAs?: string | undefined;
26115
+ } & {
26116
+ provider?: any;
26117
+ rubricPrompt?: string | string[] | {
26118
+ role: string;
26119
+ content: string;
26120
+ }[] | undefined;
26121
+ factuality?: {
26122
+ subset?: number | undefined;
26123
+ superset?: number | undefined;
26124
+ agree?: number | undefined;
26125
+ disagree?: number | undefined;
26126
+ differButFactual?: number | undefined;
26127
+ } | undefined;
26128
+ } & {
26129
+ disableVarExpansion?: boolean | undefined;
26130
+ disableConversationVar?: boolean | undefined;
26131
+ runSerially?: boolean | undefined;
26132
+ }) | undefined;
26133
+ vars?: Record<string, string | number | boolean | any[] | Record<string, any> | (string | number | boolean)[]> | undefined;
26134
+ provider?: string | {
27340
26135
  id?: string | undefined;
27341
26136
  config?: any;
27342
- display?: string | undefined;
27343
- };
27344
- vars: Record<string, string | {}>;
27345
- fetchWithCache?: any;
27346
- filters?: Record<string, (args_0: any, ...args: any[]) => string> | undefined;
27347
- getCache?: any;
27348
- logger?: any;
27349
- originalProvider?: any;
27350
- }, {
27351
- prompt: {
27352
- raw: string;
27353
- label: string;
27354
- function?: ((args_0: {
27355
- vars: Record<string, any>;
27356
- provider?: ApiProvider | undefined;
27357
- }, ...args: unknown[]) => Promise<any>) | undefined;
27358
- id?: string | undefined;
26137
+ label?: string | undefined;
26138
+ prompts?: string[] | undefined;
26139
+ transform?: string | undefined;
26140
+ delay?: number | undefined;
26141
+ env?: {
26142
+ AI21_API_BASE_URL?: string | undefined;
26143
+ AI21_API_KEY?: string | undefined;
26144
+ ANTHROPIC_API_KEY?: string | undefined;
26145
+ ANTHROPIC_BASE_URL?: string | undefined;
26146
+ AWS_BEDROCK_REGION?: string | undefined;
26147
+ AZURE_API_BASE_URL?: string | undefined;
26148
+ AZURE_API_HOST?: string | undefined;
26149
+ AZURE_API_KEY?: string | undefined;
26150
+ AZURE_AUTHORITY_HOST?: string | undefined;
26151
+ AZURE_CLIENT_ID?: string | undefined;
26152
+ AZURE_CLIENT_SECRET?: string | undefined;
26153
+ AZURE_DEPLOYMENT_NAME?: string | undefined;
26154
+ AZURE_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
26155
+ AZURE_OPENAI_API_BASE_URL?: string | undefined;
26156
+ AZURE_OPENAI_API_HOST?: string | undefined;
26157
+ AZURE_OPENAI_API_KEY?: string | undefined;
26158
+ AZURE_OPENAI_BASE_URL?: string | undefined;
26159
+ AZURE_OPENAI_DEPLOYMENT_NAME?: string | undefined;
26160
+ AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
26161
+ AZURE_TENANT_ID?: string | undefined;
26162
+ AZURE_TOKEN_SCOPE?: string | undefined;
26163
+ BAM_API_HOST?: string | undefined;
26164
+ BAM_API_KEY?: string | undefined;
26165
+ CLOUDFLARE_ACCOUNT_ID?: string | undefined;
26166
+ CLOUDFLARE_API_KEY?: string | undefined;
26167
+ COHERE_API_KEY?: string | undefined;
26168
+ COHERE_CLIENT_NAME?: string | undefined;
26169
+ DATABRICKS_TOKEN?: string | undefined;
26170
+ DATABRICKS_WORKSPACE_URL?: string | undefined;
26171
+ FAL_KEY?: string | undefined;
26172
+ GOOGLE_API_HOST?: string | undefined;
26173
+ GOOGLE_API_KEY?: string | undefined;
26174
+ GROQ_API_KEY?: string | undefined;
26175
+ HELICONE_API_KEY?: string | undefined;
26176
+ HF_API_TOKEN?: string | undefined;
26177
+ HF_TOKEN?: string | undefined;
26178
+ HUGGING_FACE_HUB_TOKEN?: string | undefined;
26179
+ JFROG_API_KEY?: string | undefined;
26180
+ LAMBDA_API_KEY?: string | undefined;
26181
+ LANGFUSE_HOST?: string | undefined;
26182
+ LANGFUSE_PUBLIC_KEY?: string | undefined;
26183
+ LANGFUSE_SECRET_KEY?: string | undefined;
26184
+ LITELLM_API_BASE?: string | undefined;
26185
+ LLAMA_BASE_URL?: string | undefined;
26186
+ LOCALAI_BASE_URL?: string | undefined;
26187
+ MISTRAL_API_BASE_URL?: string | undefined;
26188
+ MISTRAL_API_HOST?: string | undefined;
26189
+ MISTRAL_API_KEY?: string | undefined;
26190
+ OLLAMA_API_KEY?: string | undefined;
26191
+ OLLAMA_BASE_URL?: string | undefined;
26192
+ OPENAI_API_BASE_URL?: string | undefined;
26193
+ OPENAI_API_HOST?: string | undefined;
26194
+ OPENAI_API_KEY?: string | undefined;
26195
+ OPENAI_BASE_URL?: string | undefined;
26196
+ OPENAI_ORGANIZATION?: string | undefined;
26197
+ PALM_API_HOST?: string | undefined;
26198
+ PALM_API_KEY?: string | undefined;
26199
+ PORTKEY_API_KEY?: string | undefined;
26200
+ PROMPTFOO_CA_CERT_PATH?: string | undefined;
26201
+ PROMPTFOO_INSECURE_SSL?: string | undefined;
26202
+ REPLICATE_API_KEY?: string | undefined;
26203
+ REPLICATE_API_TOKEN?: string | undefined;
26204
+ VERTEX_API_HOST?: string | undefined;
26205
+ VERTEX_API_KEY?: string | undefined;
26206
+ VERTEX_API_VERSION?: string | undefined;
26207
+ VERTEX_PROJECT_ID?: string | undefined;
26208
+ VERTEX_PUBLISHER?: string | undefined;
26209
+ VERTEX_REGION?: string | undefined;
26210
+ VOYAGE_API_BASE_URL?: string | undefined;
26211
+ VOYAGE_API_KEY?: string | undefined;
26212
+ WATSONX_AI_APIKEY?: string | undefined;
26213
+ WATSONX_AI_AUTH_TYPE?: string | undefined;
26214
+ WATSONX_AI_BEARER_TOKEN?: string | undefined;
26215
+ WATSONX_AI_PROJECT_ID?: string | undefined;
26216
+ AZURE_CONTENT_SAFETY_ENDPOINT?: string | undefined;
26217
+ AZURE_CONTENT_SAFETY_API_KEY?: string | undefined;
26218
+ AZURE_CONTENT_SAFETY_API_VERSION?: string | undefined;
26219
+ AWS_REGION?: string | undefined;
26220
+ AWS_DEFAULT_REGION?: string | undefined;
26221
+ AWS_SAGEMAKER_MAX_TOKENS?: string | undefined;
26222
+ AWS_SAGEMAKER_TEMPERATURE?: string | undefined;
26223
+ AWS_SAGEMAKER_TOP_P?: string | undefined;
26224
+ AWS_SAGEMAKER_MAX_RETRIES?: string | undefined;
26225
+ PROMPTFOO_EVAL_TIMEOUT_MS?: string | undefined;
26226
+ } | undefined;
26227
+ } | {
26228
+ callApi: import("./providers").CallApiFunction;
26229
+ id: (...args: unknown[]) => string;
27359
26230
  config?: any;
27360
- display?: string | undefined;
27361
- };
27362
- vars: Record<string, string | {}>;
27363
- fetchWithCache?: any;
27364
- filters?: Record<string, (args_0: any, ...args: any[]) => string> | undefined;
27365
- getCache?: any;
27366
- logger?: any;
27367
- originalProvider?: any;
27368
- }>>, z.ZodOptional<z.ZodObject<{
27369
- includeLogProbs: z.ZodOptional<z.ZodBoolean>;
27370
- }, "strip", z.ZodTypeAny, {
27371
- includeLogProbs?: boolean | undefined;
27372
- }, {
27373
- includeLogProbs?: boolean | undefined;
27374
- }>>], z.ZodUnknown>, z.ZodPromise<z.ZodType<ProviderResponse, z.ZodTypeDef, ProviderResponse>>>, z.ZodObject<{
27375
- label: z.ZodOptional<z.ZodString>;
27376
- }, "strip", z.ZodTypeAny, {
27377
- label?: string | undefined;
27378
- }, {
27379
- label?: string | undefined;
27380
- }>>]>, "many">]>;
26231
+ label?: string | undefined;
26232
+ transform?: string | undefined;
26233
+ delay?: number | undefined;
26234
+ callEmbeddingApi?: ((args_0: string, ...args: unknown[]) => Promise<import("./providers").ProviderEmbeddingResponse>) | undefined;
26235
+ callClassificationApi?: ((args_0: string, ...args: unknown[]) => Promise<import("./providers").ProviderClassificationResponse>) | undefined;
26236
+ } | undefined;
26237
+ metadata?: (Record<string, any> & {
26238
+ pluginConfig?: PluginConfig | undefined;
26239
+ strategyConfig?: import("../redteam/types").RedteamObjectConfig | undefined;
26240
+ }) | undefined;
26241
+ providerOutput?: string | {} | undefined;
26242
+ assert?: ({
26243
+ type: "moderation" | `promptfoo:redteam:${string}` | "cost" | "factuality" | "answer-relevance" | "bleu" | "classifier" | "contains" | "contains-all" | "contains-any" | "contains-json" | "contains-sql" | "contains-xml" | "context-faithfulness" | "context-recall" | "context-relevance" | "equals" | "g-eval" | "gleu" | "guardrails" | "icontains" | "icontains-all" | "icontains-any" | "is-json" | "is-refusal" | "is-sql" | "is-valid-function-call" | "is-valid-openai-function-call" | "is-valid-openai-tools-call" | "is-xml" | "javascript" | "latency" | "levenshtein" | "llm-rubric" | "pi" | "meteor" | "model-graded-closedqa" | "model-graded-factuality" | "perplexity" | "perplexity-score" | "python" | "regex" | "rouge-n" | "similar" | "starts-with" | "webhook" | "select-best" | "human";
26244
+ value?: AssertionValue | undefined;
26245
+ config?: Record<string, any> | undefined;
26246
+ provider?: any;
26247
+ transform?: string | undefined;
26248
+ rubricPrompt?: string | string[] | {
26249
+ role: string;
26250
+ content: string;
26251
+ }[] | undefined;
26252
+ threshold?: number | undefined;
26253
+ weight?: number | undefined;
26254
+ metric?: string | undefined;
26255
+ } | {
26256
+ type: "assert-set";
26257
+ assert: {
26258
+ type: "moderation" | `promptfoo:redteam:${string}` | "cost" | "factuality" | "answer-relevance" | "bleu" | "classifier" | "contains" | "contains-all" | "contains-any" | "contains-json" | "contains-sql" | "contains-xml" | "context-faithfulness" | "context-recall" | "context-relevance" | "equals" | "g-eval" | "gleu" | "guardrails" | "icontains" | "icontains-all" | "icontains-any" | "is-json" | "is-refusal" | "is-sql" | "is-valid-function-call" | "is-valid-openai-function-call" | "is-valid-openai-tools-call" | "is-xml" | "javascript" | "latency" | "levenshtein" | "llm-rubric" | "pi" | "meteor" | "model-graded-closedqa" | "model-graded-factuality" | "perplexity" | "perplexity-score" | "python" | "regex" | "rouge-n" | "similar" | "starts-with" | "webhook" | "select-best" | "human";
26259
+ value?: AssertionValue | undefined;
26260
+ config?: Record<string, any> | undefined;
26261
+ provider?: any;
26262
+ transform?: string | undefined;
26263
+ rubricPrompt?: string | string[] | {
26264
+ role: string;
26265
+ content: string;
26266
+ }[] | undefined;
26267
+ threshold?: number | undefined;
26268
+ weight?: number | undefined;
26269
+ metric?: string | undefined;
26270
+ }[];
26271
+ config?: Record<string, any> | undefined;
26272
+ threshold?: number | undefined;
26273
+ weight?: number | undefined;
26274
+ metric?: string | undefined;
26275
+ })[] | undefined;
26276
+ threshold?: number | undefined;
26277
+ assertScoringFunction?: string | ScoringFunction | undefined;
26278
+ } | undefined;
26279
+ nunjucksFilters?: Record<string, string> | undefined;
26280
+ derivedMetrics?: {
26281
+ value: string | ((args_0: Record<string, number>, args_1: RunEvalOptions, ...args: unknown[]) => number);
26282
+ name: string;
26283
+ }[] | undefined;
26284
+ extensions?: string[] | null | undefined;
26285
+ outputPath?: string | string[] | undefined;
26286
+ sharing?: boolean | {
26287
+ apiBaseUrl?: string | undefined;
26288
+ appBaseUrl?: string | undefined;
26289
+ } | undefined;
26290
+ writeLatestResults?: boolean | undefined;
26291
+ }>;
26292
+ export type TestSuiteConfig = z.infer<typeof TestSuiteConfigSchema>;
26293
+ export declare const UnifiedConfigSchema: z.ZodEffects<z.ZodEffects<z.ZodObject<{
26294
+ tags: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
26295
+ description: z.ZodOptional<z.ZodString>;
27381
26296
  prompts: z.ZodUnion<[z.ZodString, z.ZodArray<z.ZodUnion<[z.ZodString, z.ZodObject<{
27382
26297
  id: z.ZodString;
27383
26298
  label: z.ZodOptional<z.ZodString>;
@@ -33205,7 +32120,6 @@ export declare const UnifiedConfigSchema: z.ZodEffects<z.ZodEffects<z.ZodObject<
33205
32120
  })[];
33206
32121
  maxConcurrency?: number | undefined;
33207
32122
  entities?: string[] | undefined;
33208
- numTests?: number | undefined;
33209
32123
  provider?: string | {
33210
32124
  id?: string | undefined;
33211
32125
  config?: any;
@@ -33310,6 +32224,7 @@ export declare const UnifiedConfigSchema: z.ZodEffects<z.ZodEffects<z.ZodObject<
33310
32224
  callClassificationApi?: ((args_0: string, ...args: unknown[]) => Promise<import("./providers").ProviderClassificationResponse>) | undefined;
33311
32225
  } | undefined;
33312
32226
  delay?: number | undefined;
32227
+ numTests?: number | undefined;
33313
32228
  injectVar?: string | undefined;
33314
32229
  language?: string | undefined;
33315
32230
  purpose?: string | undefined;
@@ -33322,7 +32237,6 @@ export declare const UnifiedConfigSchema: z.ZodEffects<z.ZodEffects<z.ZodObject<
33322
32237
  })[] | undefined;
33323
32238
  maxConcurrency?: number | undefined;
33324
32239
  entities?: string[] | undefined;
33325
- numTests?: number | undefined;
33326
32240
  provider?: string | {
33327
32241
  id?: string | undefined;
33328
32242
  config?: any;
@@ -33427,6 +32341,7 @@ export declare const UnifiedConfigSchema: z.ZodEffects<z.ZodEffects<z.ZodObject<
33427
32341
  callClassificationApi?: ((args_0: string, ...args: unknown[]) => Promise<import("./providers").ProviderClassificationResponse>) | undefined;
33428
32342
  } | undefined;
33429
32343
  delay?: number | undefined;
32344
+ numTests?: number | undefined;
33430
32345
  injectVar?: string | undefined;
33431
32346
  language?: string | undefined;
33432
32347
  purpose?: string | undefined;
@@ -33443,7 +32358,6 @@ export declare const UnifiedConfigSchema: z.ZodEffects<z.ZodEffects<z.ZodObject<
33443
32358
  })[] | undefined;
33444
32359
  maxConcurrency?: number | undefined;
33445
32360
  entities?: string[] | undefined;
33446
- numTests?: number | undefined;
33447
32361
  provider?: string | {
33448
32362
  id?: string | undefined;
33449
32363
  config?: any;
@@ -33548,6 +32462,7 @@ export declare const UnifiedConfigSchema: z.ZodEffects<z.ZodEffects<z.ZodObject<
33548
32462
  callClassificationApi?: ((args_0: string, ...args: unknown[]) => Promise<import("./providers").ProviderClassificationResponse>) | undefined;
33549
32463
  } | undefined;
33550
32464
  delay?: number | undefined;
32465
+ numTests?: number | undefined;
33551
32466
  injectVar?: string | undefined;
33552
32467
  language?: string | undefined;
33553
32468
  purpose?: string | undefined;
@@ -33558,7 +32473,7 @@ export declare const UnifiedConfigSchema: z.ZodEffects<z.ZodEffects<z.ZodObject<
33558
32473
  excludeTargetOutputFromAgenticAttackGeneration?: boolean | undefined;
33559
32474
  }>>;
33560
32475
  writeLatestResults: z.ZodOptional<z.ZodBoolean>;
33561
- }, {
32476
+ } & {
33562
32477
  evaluateOptions: z.ZodOptional<z.ZodObject<{
33563
32478
  cache: z.ZodOptional<z.ZodBoolean>;
33564
32479
  delay: z.ZodOptional<z.ZodNumber>;
@@ -36037,7 +34952,7 @@ export declare const UnifiedConfigSchema: z.ZodEffects<z.ZodEffects<z.ZodObject<
36037
34952
  }, {
36038
34953
  label?: string | undefined;
36039
34954
  }>>]>, "many">]>>;
36040
- }>, "strip", z.ZodTypeAny, {
34955
+ }, "strip", z.ZodTypeAny, {
36041
34956
  prompts: string | Record<string, string> | (string | {
36042
34957
  raw: string;
36043
34958
  label: string;
@@ -37921,7 +36836,6 @@ export declare const UnifiedConfigSchema: z.ZodEffects<z.ZodEffects<z.ZodObject<
37921
36836
  })[] | undefined;
37922
36837
  maxConcurrency?: number | undefined;
37923
36838
  entities?: string[] | undefined;
37924
- numTests?: number | undefined;
37925
36839
  provider?: string | {
37926
36840
  id?: string | undefined;
37927
36841
  config?: any;
@@ -38026,6 +36940,7 @@ export declare const UnifiedConfigSchema: z.ZodEffects<z.ZodEffects<z.ZodObject<
38026
36940
  callClassificationApi?: ((args_0: string, ...args: unknown[]) => Promise<import("./providers").ProviderClassificationResponse>) | undefined;
38027
36941
  } | undefined;
38028
36942
  delay?: number | undefined;
36943
+ numTests?: number | undefined;
38029
36944
  injectVar?: string | undefined;
38030
36945
  language?: string | undefined;
38031
36946
  purpose?: string | undefined;
@@ -40776,7 +39691,6 @@ export declare const UnifiedConfigSchema: z.ZodEffects<z.ZodEffects<z.ZodObject<
40776
39691
  })[] | undefined;
40777
39692
  maxConcurrency?: number | undefined;
40778
39693
  entities?: string[] | undefined;
40779
- numTests?: number | undefined;
40780
39694
  provider?: string | {
40781
39695
  id?: string | undefined;
40782
39696
  config?: any;
@@ -40881,6 +39795,7 @@ export declare const UnifiedConfigSchema: z.ZodEffects<z.ZodEffects<z.ZodObject<
40881
39795
  callClassificationApi?: ((args_0: string, ...args: unknown[]) => Promise<import("./providers").ProviderClassificationResponse>) | undefined;
40882
39796
  } | undefined;
40883
39797
  delay?: number | undefined;
39798
+ numTests?: number | undefined;
40884
39799
  injectVar?: string | undefined;
40885
39800
  language?: string | undefined;
40886
39801
  purpose?: string | undefined;
@@ -43631,7 +42546,6 @@ export declare const UnifiedConfigSchema: z.ZodEffects<z.ZodEffects<z.ZodObject<
43631
42546
  })[] | undefined;
43632
42547
  maxConcurrency?: number | undefined;
43633
42548
  entities?: string[] | undefined;
43634
- numTests?: number | undefined;
43635
42549
  provider?: string | {
43636
42550
  id?: string | undefined;
43637
42551
  config?: any;
@@ -43736,6 +42650,7 @@ export declare const UnifiedConfigSchema: z.ZodEffects<z.ZodEffects<z.ZodObject<
43736
42650
  callClassificationApi?: ((args_0: string, ...args: unknown[]) => Promise<import("./providers").ProviderClassificationResponse>) | undefined;
43737
42651
  } | undefined;
43738
42652
  delay?: number | undefined;
42653
+ numTests?: number | undefined;
43739
42654
  injectVar?: string | undefined;
43740
42655
  language?: string | undefined;
43741
42656
  purpose?: string | undefined;