promptfoo 0.96.2 → 0.98.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (289) hide show
  1. package/dist/package.json +15 -15
  2. package/dist/src/app/assets/index-D7I51A2q.js +789 -0
  3. package/dist/src/app/assets/{index-YwjS5Vq2.css → index-DatcZSwS.css} +1 -1
  4. package/dist/src/app/assets/{index.es-CrujH26S.js → index.es-COpBxjTX.js} +1 -1
  5. package/dist/src/app/assets/{sync-gFAH4kmH.js → sync-BY_V2XfM.js} +1 -1
  6. package/dist/src/app/index.html +2 -2
  7. package/dist/src/assertions/index.d.ts.map +1 -1
  8. package/dist/src/assertions/index.js +10 -3
  9. package/dist/src/assertions/index.js.map +1 -1
  10. package/dist/src/assertions/redteam.js +2 -2
  11. package/dist/src/assertions/redteam.js.map +1 -1
  12. package/dist/src/cache.d.ts +1 -0
  13. package/dist/src/cache.d.ts.map +1 -1
  14. package/dist/src/cache.js +5 -0
  15. package/dist/src/cache.js.map +1 -1
  16. package/dist/src/commands/eval.d.ts.map +1 -1
  17. package/dist/src/commands/eval.js +7 -2
  18. package/dist/src/commands/eval.js.map +1 -1
  19. package/dist/src/database/tables.d.ts +201 -182
  20. package/dist/src/database/tables.d.ts.map +1 -1
  21. package/dist/src/envars.d.ts +1 -1
  22. package/dist/src/envars.d.ts.map +1 -1
  23. package/dist/src/evaluator.d.ts.map +1 -1
  24. package/dist/src/evaluator.js +13 -3
  25. package/dist/src/evaluator.js.map +1 -1
  26. package/dist/src/index.d.ts +2 -106
  27. package/dist/src/index.d.ts.map +1 -1
  28. package/dist/src/logger.js +10 -10
  29. package/dist/src/logger.js.map +1 -1
  30. package/dist/src/main.js +15 -1
  31. package/dist/src/main.js.map +1 -1
  32. package/dist/src/models/eval.d.ts +2 -0
  33. package/dist/src/models/eval.d.ts.map +1 -1
  34. package/dist/src/models/eval.js +4 -0
  35. package/dist/src/models/eval.js.map +1 -1
  36. package/dist/src/prompts/processors/yaml.d.ts.map +1 -1
  37. package/dist/src/prompts/processors/yaml.js +7 -2
  38. package/dist/src/prompts/processors/yaml.js.map +1 -1
  39. package/dist/src/providers/adaline.gateway.d.ts +2 -1
  40. package/dist/src/providers/adaline.gateway.d.ts.map +1 -1
  41. package/dist/src/providers/adaline.gateway.js +2 -2
  42. package/dist/src/providers/adaline.gateway.js.map +1 -1
  43. package/dist/src/providers/ai21.d.ts +2 -1
  44. package/dist/src/providers/ai21.d.ts.map +1 -1
  45. package/dist/src/providers/ai21.js.map +1 -1
  46. package/dist/src/providers/anthropic.d.ts +2 -1
  47. package/dist/src/providers/anthropic.d.ts.map +1 -1
  48. package/dist/src/providers/anthropic.js.map +1 -1
  49. package/dist/src/providers/azure.d.ts +12 -4
  50. package/dist/src/providers/azure.d.ts.map +1 -1
  51. package/dist/src/providers/azure.js +106 -62
  52. package/dist/src/providers/azure.js.map +1 -1
  53. package/dist/src/providers/bam.d.ts +2 -1
  54. package/dist/src/providers/bam.d.ts.map +1 -1
  55. package/dist/src/providers/bam.js.map +1 -1
  56. package/dist/src/providers/bedrock.d.ts +6 -2
  57. package/dist/src/providers/bedrock.d.ts.map +1 -1
  58. package/dist/src/providers/bedrock.js +13 -0
  59. package/dist/src/providers/bedrock.js.map +1 -1
  60. package/dist/src/providers/cloudflare-ai.d.ts +2 -1
  61. package/dist/src/providers/cloudflare-ai.d.ts.map +1 -1
  62. package/dist/src/providers/cohere.d.ts +2 -1
  63. package/dist/src/providers/cohere.d.ts.map +1 -1
  64. package/dist/src/providers/defaults.d.ts +2 -1
  65. package/dist/src/providers/defaults.d.ts.map +1 -1
  66. package/dist/src/providers/defaults.js.map +1 -1
  67. package/dist/src/providers/fal.d.ts +2 -1
  68. package/dist/src/providers/fal.d.ts.map +1 -1
  69. package/dist/src/providers/fal.js.map +1 -1
  70. package/dist/src/providers/groq.d.ts +2 -1
  71. package/dist/src/providers/groq.d.ts.map +1 -1
  72. package/dist/src/providers/http.d.ts +5 -1
  73. package/dist/src/providers/http.d.ts.map +1 -1
  74. package/dist/src/providers/http.js +67 -10
  75. package/dist/src/providers/http.js.map +1 -1
  76. package/dist/src/providers/localai.d.ts +2 -1
  77. package/dist/src/providers/localai.d.ts.map +1 -1
  78. package/dist/src/providers/localai.js.map +1 -1
  79. package/dist/src/providers/mistral.d.ts +2 -1
  80. package/dist/src/providers/mistral.d.ts.map +1 -1
  81. package/dist/src/providers/openai.d.ts +2 -1
  82. package/dist/src/providers/openai.d.ts.map +1 -1
  83. package/dist/src/providers/openai.js +1 -1
  84. package/dist/src/providers/openai.js.map +1 -1
  85. package/dist/src/providers/palm.d.ts +2 -1
  86. package/dist/src/providers/palm.d.ts.map +1 -1
  87. package/dist/src/providers/palm.js.map +1 -1
  88. package/dist/src/providers/promptfoo.d.ts +9 -4
  89. package/dist/src/providers/promptfoo.d.ts.map +1 -1
  90. package/dist/src/providers/promptfoo.js +7 -7
  91. package/dist/src/providers/promptfoo.js.map +1 -1
  92. package/dist/src/providers/replicate.d.ts +2 -1
  93. package/dist/src/providers/replicate.d.ts.map +1 -1
  94. package/dist/src/providers/simulatedUser.d.ts.map +1 -1
  95. package/dist/src/providers/simulatedUser.js +5 -0
  96. package/dist/src/providers/simulatedUser.js.map +1 -1
  97. package/dist/src/providers/togetherai.d.ts +2 -1
  98. package/dist/src/providers/togetherai.d.ts.map +1 -1
  99. package/dist/src/providers/togetherai.js.map +1 -1
  100. package/dist/src/providers/vertex.d.ts +2 -1
  101. package/dist/src/providers/vertex.d.ts.map +1 -1
  102. package/dist/src/providers/watsonx.d.ts +2 -1
  103. package/dist/src/providers/watsonx.d.ts.map +1 -1
  104. package/dist/src/providers/watsonx.js.map +1 -1
  105. package/dist/src/providers/xai.d.ts +2 -1
  106. package/dist/src/providers/xai.d.ts.map +1 -1
  107. package/dist/src/providers/xai.js.map +1 -1
  108. package/dist/src/providers.d.ts +2 -1
  109. package/dist/src/providers.d.ts.map +1 -1
  110. package/dist/src/providers.js.map +1 -1
  111. package/dist/src/redteam/commands/generate.js +1 -1
  112. package/dist/src/redteam/commands/generate.js.map +1 -1
  113. package/dist/src/redteam/commands/init.d.ts.map +1 -1
  114. package/dist/src/redteam/commands/init.js +57 -49
  115. package/dist/src/redteam/commands/init.js.map +1 -1
  116. package/dist/src/redteam/commands/setup.d.ts +3 -0
  117. package/dist/src/redteam/commands/setup.d.ts.map +1 -0
  118. package/dist/src/redteam/commands/setup.js +31 -0
  119. package/dist/src/redteam/commands/setup.js.map +1 -0
  120. package/dist/src/redteam/constants.d.ts +2 -2
  121. package/dist/src/redteam/constants.d.ts.map +1 -1
  122. package/dist/src/redteam/constants.js +1 -1
  123. package/dist/src/redteam/constants.js.map +1 -1
  124. package/dist/src/redteam/graders.d.ts +2 -77
  125. package/dist/src/redteam/graders.d.ts.map +1 -1
  126. package/dist/src/redteam/graders.js +25 -25
  127. package/dist/src/redteam/graders.js.map +1 -1
  128. package/dist/src/redteam/index.d.ts.map +1 -1
  129. package/dist/src/redteam/index.js +46 -23
  130. package/dist/src/redteam/index.js.map +1 -1
  131. package/dist/src/redteam/plugins/base.d.ts +12 -7
  132. package/dist/src/redteam/plugins/base.d.ts.map +1 -1
  133. package/dist/src/redteam/plugins/base.js +42 -28
  134. package/dist/src/redteam/plugins/base.js.map +1 -1
  135. package/dist/src/redteam/plugins/competitors.d.ts.map +1 -1
  136. package/dist/src/redteam/plugins/competitors.js +2 -0
  137. package/dist/src/redteam/plugins/competitors.js.map +1 -1
  138. package/dist/src/redteam/plugins/debugAccess.d.ts.map +1 -1
  139. package/dist/src/redteam/plugins/debugAccess.js +18 -0
  140. package/dist/src/redteam/plugins/debugAccess.js.map +1 -1
  141. package/dist/src/redteam/plugins/harmful/aligned.d.ts +13 -0
  142. package/dist/src/redteam/plugins/harmful/aligned.d.ts.map +1 -0
  143. package/dist/src/redteam/plugins/harmful/aligned.js +29 -0
  144. package/dist/src/redteam/plugins/harmful/aligned.js.map +1 -0
  145. package/dist/src/redteam/plugins/harmful/common.d.ts +5 -0
  146. package/dist/src/redteam/plugins/harmful/common.d.ts.map +1 -0
  147. package/dist/src/redteam/plugins/harmful/common.js +39 -0
  148. package/dist/src/redteam/plugins/harmful/common.js.map +1 -0
  149. package/dist/src/redteam/plugins/harmful/constants.d.ts +10 -0
  150. package/dist/src/redteam/plugins/harmful/constants.d.ts.map +1 -0
  151. package/dist/src/redteam/plugins/harmful/constants.js +166 -0
  152. package/dist/src/redteam/plugins/harmful/constants.js.map +1 -0
  153. package/dist/src/redteam/plugins/harmful/graders.d.ts +15 -0
  154. package/dist/src/redteam/plugins/harmful/graders.d.ts.map +1 -0
  155. package/dist/src/redteam/plugins/harmful/graders.js +94 -0
  156. package/dist/src/redteam/plugins/harmful/graders.js.map +1 -0
  157. package/dist/src/redteam/plugins/harmful/unaligned.d.ts +5 -0
  158. package/dist/src/redteam/plugins/harmful/unaligned.d.ts.map +1 -0
  159. package/dist/src/redteam/plugins/harmful/unaligned.js +29 -0
  160. package/dist/src/redteam/plugins/harmful/unaligned.js.map +1 -0
  161. package/dist/src/redteam/plugins/index.d.ts.map +1 -1
  162. package/dist/src/redteam/plugins/index.js +29 -26
  163. package/dist/src/redteam/plugins/index.js.map +1 -1
  164. package/dist/src/redteam/plugins/intent.d.ts.map +1 -1
  165. package/dist/src/redteam/plugins/intent.js +6 -0
  166. package/dist/src/redteam/plugins/intent.js.map +1 -1
  167. package/dist/src/redteam/providers/crescendo/index.d.ts +9 -0
  168. package/dist/src/redteam/providers/crescendo/index.d.ts.map +1 -1
  169. package/dist/src/redteam/providers/crescendo/index.js +96 -39
  170. package/dist/src/redteam/providers/crescendo/index.js.map +1 -1
  171. package/dist/src/redteam/providers/goat.d.ts.map +1 -1
  172. package/dist/src/redteam/providers/goat.js +33 -1
  173. package/dist/src/redteam/providers/goat.js.map +1 -1
  174. package/dist/src/redteam/providers/iterative.d.ts +7 -0
  175. package/dist/src/redteam/providers/iterative.d.ts.map +1 -1
  176. package/dist/src/redteam/providers/iterative.js +55 -3
  177. package/dist/src/redteam/providers/iterative.js.map +1 -1
  178. package/dist/src/redteam/providers/iterativeImage.d.ts +9 -1
  179. package/dist/src/redteam/providers/iterativeImage.d.ts.map +1 -1
  180. package/dist/src/redteam/providers/iterativeImage.js +42 -12
  181. package/dist/src/redteam/providers/iterativeImage.js.map +1 -1
  182. package/dist/src/redteam/providers/iterativeTree.d.ts +7 -2
  183. package/dist/src/redteam/providers/iterativeTree.d.ts.map +1 -1
  184. package/dist/src/redteam/providers/iterativeTree.js +56 -11
  185. package/dist/src/redteam/providers/iterativeTree.js.map +1 -1
  186. package/dist/src/redteam/providers/shared.d.ts +9 -3
  187. package/dist/src/redteam/providers/shared.d.ts.map +1 -1
  188. package/dist/src/redteam/providers/shared.js +15 -6
  189. package/dist/src/redteam/providers/shared.js.map +1 -1
  190. package/dist/src/redteam/strategies/index.d.ts +3 -2
  191. package/dist/src/redteam/strategies/index.d.ts.map +1 -1
  192. package/dist/src/redteam/strategies/index.js +48 -13
  193. package/dist/src/redteam/strategies/index.js.map +1 -1
  194. package/dist/src/redteam/strategies/multilingual.d.ts.map +1 -1
  195. package/dist/src/redteam/strategies/multilingual.js +1 -0
  196. package/dist/src/redteam/strategies/multilingual.js.map +1 -1
  197. package/dist/src/redteam/types.d.ts +6 -0
  198. package/dist/src/redteam/types.d.ts.map +1 -1
  199. package/dist/src/server/routes/providers.js +1 -0
  200. package/dist/src/server/routes/providers.js.map +1 -1
  201. package/dist/src/server/server.d.ts +2 -1
  202. package/dist/src/server/server.d.ts.map +1 -1
  203. package/dist/src/server/server.js +13 -3
  204. package/dist/src/server/server.js.map +1 -1
  205. package/dist/src/share.d.ts.map +1 -1
  206. package/dist/src/share.js +11 -0
  207. package/dist/src/share.js.map +1 -1
  208. package/dist/src/types/env.d.ts +54 -0
  209. package/dist/src/types/env.d.ts.map +1 -0
  210. package/dist/src/types/env.js +3 -0
  211. package/dist/src/types/env.js.map +1 -0
  212. package/dist/src/types/index.d.ts +8185 -6164
  213. package/dist/src/types/index.d.ts.map +1 -1
  214. package/dist/src/types/index.js +8 -2
  215. package/dist/src/types/index.js.map +1 -1
  216. package/dist/src/types/providers.d.ts +3 -53
  217. package/dist/src/types/providers.d.ts.map +1 -1
  218. package/dist/src/types/providers.js.map +1 -1
  219. package/dist/src/types/shared.d.ts +1 -0
  220. package/dist/src/types/shared.d.ts.map +1 -1
  221. package/dist/src/util/config/load.js +1 -1
  222. package/dist/src/util/config/load.js.map +1 -1
  223. package/dist/src/util/exportToFile/index.d.ts +1 -0
  224. package/dist/src/util/exportToFile/index.d.ts.map +1 -1
  225. package/dist/src/util/index.d.ts +42 -34
  226. package/dist/src/util/index.d.ts.map +1 -1
  227. package/dist/src/util/transform.d.ts +1 -0
  228. package/dist/src/util/transform.d.ts.map +1 -1
  229. package/dist/src/util/transform.js.map +1 -1
  230. package/dist/src/validators/providers.d.ts +311 -296
  231. package/dist/src/validators/providers.d.ts.map +1 -1
  232. package/dist/src/validators/redteam.d.ts +126 -126
  233. package/dist/src/validators/redteam.d.ts.map +1 -1
  234. package/dist/src/validators/redteam.js +14 -2
  235. package/dist/src/validators/redteam.js.map +1 -1
  236. package/dist/src/validators/shared.d.ts +3 -0
  237. package/dist/src/validators/shared.d.ts.map +1 -1
  238. package/dist/src/validators/shared.js +1 -0
  239. package/dist/src/validators/shared.js.map +1 -1
  240. package/dist/test/cache.test.js +47 -7
  241. package/dist/test/cache.test.js.map +1 -1
  242. package/dist/test/commands/init.test.js +14 -0
  243. package/dist/test/commands/init.test.js.map +1 -1
  244. package/dist/test/evaluator.test.js +136 -32
  245. package/dist/test/evaluator.test.js.map +1 -1
  246. package/dist/test/factories/evalFactory.d.ts +161 -148
  247. package/dist/test/factories/evalFactory.d.ts.map +1 -1
  248. package/dist/test/prompts/index.test.js +38 -25
  249. package/dist/test/prompts/index.test.js.map +1 -1
  250. package/dist/test/prompts/processors/yaml.test.js +50 -4
  251. package/dist/test/prompts/processors/yaml.test.js.map +1 -1
  252. package/dist/test/providers/http.test.js +0 -17
  253. package/dist/test/providers/http.test.js.map +1 -1
  254. package/dist/test/providers/index.test.js +1 -0
  255. package/dist/test/providers/index.test.js.map +1 -1
  256. package/dist/test/providers/togetherai.test.js.map +1 -1
  257. package/dist/test/redteam/index.test.js +2 -2
  258. package/dist/test/redteam/index.test.js.map +1 -1
  259. package/dist/test/redteam/plugins/base.test.js +61 -23
  260. package/dist/test/redteam/plugins/base.test.js.map +1 -1
  261. package/dist/test/redteam/plugins/harmful/aligned.test.d.ts +2 -0
  262. package/dist/test/redteam/plugins/harmful/aligned.test.d.ts.map +1 -0
  263. package/dist/test/redteam/plugins/harmful/aligned.test.js +123 -0
  264. package/dist/test/redteam/plugins/harmful/aligned.test.js.map +1 -0
  265. package/dist/test/redteam/plugins/harmful/common.test.d.ts +2 -0
  266. package/dist/test/redteam/plugins/harmful/common.test.d.ts.map +1 -0
  267. package/dist/test/redteam/plugins/harmful/common.test.js +105 -0
  268. package/dist/test/redteam/plugins/harmful/common.test.js.map +1 -0
  269. package/dist/test/redteam/plugins/harmful/unaligned.test.d.ts +2 -0
  270. package/dist/test/redteam/plugins/harmful/unaligned.test.d.ts.map +1 -0
  271. package/dist/test/redteam/plugins/harmful/unaligned.test.js +150 -0
  272. package/dist/test/redteam/plugins/harmful/unaligned.test.js.map +1 -0
  273. package/dist/test/redteam/plugins/index.test.d.ts +2 -0
  274. package/dist/test/redteam/plugins/index.test.d.ts.map +1 -0
  275. package/dist/test/redteam/plugins/index.test.js +172 -0
  276. package/dist/test/redteam/plugins/index.test.js.map +1 -0
  277. package/dist/test/redteam/providers/iterativeTree.test.js +13 -6
  278. package/dist/test/redteam/providers/iterativeTree.test.js.map +1 -1
  279. package/dist/test/share.test.js +45 -1
  280. package/dist/test/share.test.js.map +1 -1
  281. package/dist/test/types.test.js +15 -1
  282. package/dist/test/types.test.js.map +1 -1
  283. package/dist/tsconfig.tsbuildinfo +1 -1
  284. package/package.json +15 -15
  285. package/dist/src/app/assets/index-CAof5cIE.js +0 -769
  286. package/dist/src/redteam/plugins/harmful.d.ts +0 -18
  287. package/dist/src/redteam/plugins/harmful.d.ts.map +0 -1
  288. package/dist/src/redteam/plugins/harmful.js +0 -384
  289. package/dist/src/redteam/plugins/harmful.js.map +0 -1
package/dist/test/factories/evalFactory.d.ts CHANGED
@@ -20,37 +20,40 @@ export default class EvalFactory {
20
20
  label?: string | undefined;
21
21
  })[];
22
22
  env?: {
23
+ PROMPTFOO_REMOTE_GENERATION_URL?: string | undefined;
23
24
  AI21_API_BASE_URL?: string | undefined;
24
25
  AI21_API_KEY?: string | undefined;
25
26
  ANTHROPIC_API_KEY?: string | undefined;
26
27
  AWS_BEDROCK_REGION?: string | undefined;
28
+ FAL_KEY?: string | undefined;
29
+ GROQ_API_KEY?: string | undefined;
30
+ LOCALAI_BASE_URL?: string | undefined;
31
+ WATSONX_AI_APIKEY?: string | undefined;
32
+ WATSONX_AI_PROJECT_ID?: string | undefined;
33
+ WATSONX_AI_BEARER_TOKEN?: string | undefined;
34
+ AZURE_CLIENT_SECRET?: string | undefined;
35
+ AZURE_CLIENT_ID?: string | undefined;
36
+ AZURE_TENANT_ID?: string | undefined;
37
+ AZURE_AUTHORITY_HOST?: string | undefined;
38
+ AZURE_TOKEN_SCOPE?: string | undefined;
39
+ AZURE_DEPLOYMENT_NAME?: string | undefined;
40
+ AZURE_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
41
+ AZURE_OPENAI_DEPLOYMENT_NAME?: string | undefined;
42
+ AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
27
43
  AZURE_OPENAI_API_BASE_URL?: string | undefined;
28
44
  AZURE_OPENAI_API_HOST?: string | undefined;
29
45
  AZURE_OPENAI_API_KEY?: string | undefined;
30
46
  AZURE_API_BASE_URL?: string | undefined;
31
47
  AZURE_API_HOST?: string | undefined;
32
48
  AZURE_API_KEY?: string | undefined;
33
- AZURE_DEPLOYMENT_NAME?: string | undefined;
34
- AZURE_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
35
49
  AZURE_OPENAI_BASE_URL?: string | undefined;
36
- AZURE_OPENAI_DEPLOYMENT_NAME?: string | undefined;
37
- AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
38
- AZURE_CLIENT_SECRET?: string | undefined;
39
- AZURE_CLIENT_ID?: string | undefined;
40
- AZURE_TENANT_ID?: string | undefined;
41
- AZURE_AUTHORITY_HOST?: string | undefined;
42
- AZURE_TOKEN_SCOPE?: string | undefined;
43
50
  BAM_API_HOST?: string | undefined;
44
51
  BAM_API_KEY?: string | undefined;
45
52
  CLOUDFLARE_ACCOUNT_ID?: string | undefined;
46
53
  CLOUDFLARE_API_KEY?: string | undefined;
47
- PROMPTFOO_REMOTE_GENERATION_URL?: string | undefined;
48
54
  COHERE_API_KEY?: string | undefined;
49
- FAL_KEY?: string | undefined;
50
55
  GOOGLE_API_HOST?: string | undefined;
51
56
  GOOGLE_API_KEY?: string | undefined;
52
- GROQ_API_KEY?: string | undefined;
53
- LOCALAI_BASE_URL?: string | undefined;
54
57
  MISTRAL_API_BASE_URL?: string | undefined;
55
58
  MISTRAL_API_HOST?: string | undefined;
56
59
  MISTRAL_API_KEY?: string | undefined;
@@ -68,9 +71,6 @@ export default class EvalFactory {
68
71
  VERTEX_PROJECT_ID?: string | undefined;
69
72
  VERTEX_PUBLISHER?: string | undefined;
70
73
  VERTEX_REGION?: string | undefined;
71
- WATSONX_AI_APIKEY?: string | undefined;
72
- WATSONX_AI_PROJECT_ID?: string | undefined;
73
- WATSONX_AI_BEARER_TOKEN?: string | undefined;
74
74
  } | undefined;
75
75
  metadata?: Record<string, any> | undefined;
76
76
  description?: string | undefined;
@@ -104,37 +104,40 @@ export default class EvalFactory {
104
104
  transform?: string | undefined;
105
105
  delay?: number | undefined;
106
106
  env?: {
107
+ PROMPTFOO_REMOTE_GENERATION_URL?: string | undefined;
107
108
  AI21_API_BASE_URL?: string | undefined;
108
109
  AI21_API_KEY?: string | undefined;
109
110
  ANTHROPIC_API_KEY?: string | undefined;
110
111
  AWS_BEDROCK_REGION?: string | undefined;
112
+ FAL_KEY?: string | undefined;
113
+ GROQ_API_KEY?: string | undefined;
114
+ LOCALAI_BASE_URL?: string | undefined;
115
+ WATSONX_AI_APIKEY?: string | undefined;
116
+ WATSONX_AI_PROJECT_ID?: string | undefined;
117
+ WATSONX_AI_BEARER_TOKEN?: string | undefined;
118
+ AZURE_CLIENT_SECRET?: string | undefined;
119
+ AZURE_CLIENT_ID?: string | undefined;
120
+ AZURE_TENANT_ID?: string | undefined;
121
+ AZURE_AUTHORITY_HOST?: string | undefined;
122
+ AZURE_TOKEN_SCOPE?: string | undefined;
123
+ AZURE_DEPLOYMENT_NAME?: string | undefined;
124
+ AZURE_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
125
+ AZURE_OPENAI_DEPLOYMENT_NAME?: string | undefined;
126
+ AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
111
127
  AZURE_OPENAI_API_BASE_URL?: string | undefined;
112
128
  AZURE_OPENAI_API_HOST?: string | undefined;
113
129
  AZURE_OPENAI_API_KEY?: string | undefined;
114
130
  AZURE_API_BASE_URL?: string | undefined;
115
131
  AZURE_API_HOST?: string | undefined;
116
132
  AZURE_API_KEY?: string | undefined;
117
- AZURE_DEPLOYMENT_NAME?: string | undefined;
118
- AZURE_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
119
133
  AZURE_OPENAI_BASE_URL?: string | undefined;
120
- AZURE_OPENAI_DEPLOYMENT_NAME?: string | undefined;
121
- AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
122
- AZURE_CLIENT_SECRET?: string | undefined;
123
- AZURE_CLIENT_ID?: string | undefined;
124
- AZURE_TENANT_ID?: string | undefined;
125
- AZURE_AUTHORITY_HOST?: string | undefined;
126
- AZURE_TOKEN_SCOPE?: string | undefined;
127
134
  BAM_API_HOST?: string | undefined;
128
135
  BAM_API_KEY?: string | undefined;
129
136
  CLOUDFLARE_ACCOUNT_ID?: string | undefined;
130
137
  CLOUDFLARE_API_KEY?: string | undefined;
131
- PROMPTFOO_REMOTE_GENERATION_URL?: string | undefined;
132
138
  COHERE_API_KEY?: string | undefined;
133
- FAL_KEY?: string | undefined;
134
139
  GOOGLE_API_HOST?: string | undefined;
135
140
  GOOGLE_API_KEY?: string | undefined;
136
- GROQ_API_KEY?: string | undefined;
137
- LOCALAI_BASE_URL?: string | undefined;
138
141
  MISTRAL_API_BASE_URL?: string | undefined;
139
142
  MISTRAL_API_HOST?: string | undefined;
140
143
  MISTRAL_API_KEY?: string | undefined;
@@ -152,9 +155,6 @@ export default class EvalFactory {
152
155
  VERTEX_PROJECT_ID?: string | undefined;
153
156
  VERTEX_PUBLISHER?: string | undefined;
154
157
  VERTEX_REGION?: string | undefined;
155
- WATSONX_AI_APIKEY?: string | undefined;
156
- WATSONX_AI_PROJECT_ID?: string | undefined;
157
- WATSONX_AI_BEARER_TOKEN?: string | undefined;
158
158
  } | undefined;
159
159
  } | (((args_0: string, args_1: {
160
160
  vars: Record<string, string | {}>;
@@ -186,37 +186,40 @@ export default class EvalFactory {
186
186
  transform?: string | undefined;
187
187
  delay?: number | undefined;
188
188
  env?: {
189
+ PROMPTFOO_REMOTE_GENERATION_URL?: string | undefined;
189
190
  AI21_API_BASE_URL?: string | undefined;
190
191
  AI21_API_KEY?: string | undefined;
191
192
  ANTHROPIC_API_KEY?: string | undefined;
192
193
  AWS_BEDROCK_REGION?: string | undefined;
194
+ FAL_KEY?: string | undefined;
195
+ GROQ_API_KEY?: string | undefined;
196
+ LOCALAI_BASE_URL?: string | undefined;
197
+ WATSONX_AI_APIKEY?: string | undefined;
198
+ WATSONX_AI_PROJECT_ID?: string | undefined;
199
+ WATSONX_AI_BEARER_TOKEN?: string | undefined;
200
+ AZURE_CLIENT_SECRET?: string | undefined;
201
+ AZURE_CLIENT_ID?: string | undefined;
202
+ AZURE_TENANT_ID?: string | undefined;
203
+ AZURE_AUTHORITY_HOST?: string | undefined;
204
+ AZURE_TOKEN_SCOPE?: string | undefined;
205
+ AZURE_DEPLOYMENT_NAME?: string | undefined;
206
+ AZURE_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
207
+ AZURE_OPENAI_DEPLOYMENT_NAME?: string | undefined;
208
+ AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
193
209
  AZURE_OPENAI_API_BASE_URL?: string | undefined;
194
210
  AZURE_OPENAI_API_HOST?: string | undefined;
195
211
  AZURE_OPENAI_API_KEY?: string | undefined;
196
212
  AZURE_API_BASE_URL?: string | undefined;
197
213
  AZURE_API_HOST?: string | undefined;
198
214
  AZURE_API_KEY?: string | undefined;
199
- AZURE_DEPLOYMENT_NAME?: string | undefined;
200
- AZURE_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
201
215
  AZURE_OPENAI_BASE_URL?: string | undefined;
202
- AZURE_OPENAI_DEPLOYMENT_NAME?: string | undefined;
203
- AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
204
- AZURE_CLIENT_SECRET?: string | undefined;
205
- AZURE_CLIENT_ID?: string | undefined;
206
- AZURE_TENANT_ID?: string | undefined;
207
- AZURE_AUTHORITY_HOST?: string | undefined;
208
- AZURE_TOKEN_SCOPE?: string | undefined;
209
216
  BAM_API_HOST?: string | undefined;
210
217
  BAM_API_KEY?: string | undefined;
211
218
  CLOUDFLARE_ACCOUNT_ID?: string | undefined;
212
219
  CLOUDFLARE_API_KEY?: string | undefined;
213
- PROMPTFOO_REMOTE_GENERATION_URL?: string | undefined;
214
220
  COHERE_API_KEY?: string | undefined;
215
- FAL_KEY?: string | undefined;
216
221
  GOOGLE_API_HOST?: string | undefined;
217
222
  GOOGLE_API_KEY?: string | undefined;
218
- GROQ_API_KEY?: string | undefined;
219
- LOCALAI_BASE_URL?: string | undefined;
220
223
  MISTRAL_API_BASE_URL?: string | undefined;
221
224
  MISTRAL_API_HOST?: string | undefined;
222
225
  MISTRAL_API_KEY?: string | undefined;
@@ -234,9 +237,6 @@ export default class EvalFactory {
234
237
  VERTEX_PROJECT_ID?: string | undefined;
235
238
  VERTEX_PUBLISHER?: string | undefined;
236
239
  VERTEX_REGION?: string | undefined;
237
- WATSONX_AI_APIKEY?: string | undefined;
238
- WATSONX_AI_PROJECT_ID?: string | undefined;
239
- WATSONX_AI_BEARER_TOKEN?: string | undefined;
240
240
  } | undefined;
241
241
  }>)[] | undefined;
242
242
  tests?: string | (string | {
@@ -275,37 +275,40 @@ export default class EvalFactory {
275
275
  transform?: string | undefined;
276
276
  delay?: number | undefined;
277
277
  env?: {
278
+ PROMPTFOO_REMOTE_GENERATION_URL?: string | undefined;
278
279
  AI21_API_BASE_URL?: string | undefined;
279
280
  AI21_API_KEY?: string | undefined;
280
281
  ANTHROPIC_API_KEY?: string | undefined;
281
282
  AWS_BEDROCK_REGION?: string | undefined;
283
+ FAL_KEY?: string | undefined;
284
+ GROQ_API_KEY?: string | undefined;
285
+ LOCALAI_BASE_URL?: string | undefined;
286
+ WATSONX_AI_APIKEY?: string | undefined;
287
+ WATSONX_AI_PROJECT_ID?: string | undefined;
288
+ WATSONX_AI_BEARER_TOKEN?: string | undefined;
289
+ AZURE_CLIENT_SECRET?: string | undefined;
290
+ AZURE_CLIENT_ID?: string | undefined;
291
+ AZURE_TENANT_ID?: string | undefined;
292
+ AZURE_AUTHORITY_HOST?: string | undefined;
293
+ AZURE_TOKEN_SCOPE?: string | undefined;
294
+ AZURE_DEPLOYMENT_NAME?: string | undefined;
295
+ AZURE_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
296
+ AZURE_OPENAI_DEPLOYMENT_NAME?: string | undefined;
297
+ AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
282
298
  AZURE_OPENAI_API_BASE_URL?: string | undefined;
283
299
  AZURE_OPENAI_API_HOST?: string | undefined;
284
300
  AZURE_OPENAI_API_KEY?: string | undefined;
285
301
  AZURE_API_BASE_URL?: string | undefined;
286
302
  AZURE_API_HOST?: string | undefined;
287
303
  AZURE_API_KEY?: string | undefined;
288
- AZURE_DEPLOYMENT_NAME?: string | undefined;
289
- AZURE_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
290
304
  AZURE_OPENAI_BASE_URL?: string | undefined;
291
- AZURE_OPENAI_DEPLOYMENT_NAME?: string | undefined;
292
- AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
293
- AZURE_CLIENT_SECRET?: string | undefined;
294
- AZURE_CLIENT_ID?: string | undefined;
295
- AZURE_TENANT_ID?: string | undefined;
296
- AZURE_AUTHORITY_HOST?: string | undefined;
297
- AZURE_TOKEN_SCOPE?: string | undefined;
298
305
  BAM_API_HOST?: string | undefined;
299
306
  BAM_API_KEY?: string | undefined;
300
307
  CLOUDFLARE_ACCOUNT_ID?: string | undefined;
301
308
  CLOUDFLARE_API_KEY?: string | undefined;
302
- PROMPTFOO_REMOTE_GENERATION_URL?: string | undefined;
303
309
  COHERE_API_KEY?: string | undefined;
304
- FAL_KEY?: string | undefined;
305
310
  GOOGLE_API_HOST?: string | undefined;
306
311
  GOOGLE_API_KEY?: string | undefined;
307
- GROQ_API_KEY?: string | undefined;
308
- LOCALAI_BASE_URL?: string | undefined;
309
312
  MISTRAL_API_BASE_URL?: string | undefined;
310
313
  MISTRAL_API_HOST?: string | undefined;
311
314
  MISTRAL_API_KEY?: string | undefined;
@@ -323,9 +326,6 @@ export default class EvalFactory {
323
326
  VERTEX_PROJECT_ID?: string | undefined;
324
327
  VERTEX_PUBLISHER?: string | undefined;
325
328
  VERTEX_REGION?: string | undefined;
326
- WATSONX_AI_APIKEY?: string | undefined;
327
- WATSONX_AI_PROJECT_ID?: string | undefined;
328
- WATSONX_AI_BEARER_TOKEN?: string | undefined;
329
329
  } | undefined;
330
330
  } | {
331
331
  callApi: import("../../src").CallApiFunction;
@@ -337,7 +337,10 @@ export default class EvalFactory {
337
337
  callEmbeddingApi?: ((args_0: string, ...args: unknown[]) => Promise<import("../../src").ProviderEmbeddingResponse>) | undefined;
338
338
  callClassificationApi?: ((args_0: string, ...args: unknown[]) => Promise<import("../../src").ProviderClassificationResponse>) | undefined;
339
339
  } | undefined;
340
- metadata?: Record<string, any> | undefined;
340
+ metadata?: (Record<string, any> & {
341
+ pluginConfig?: import("../../src").PluginConfig | undefined;
342
+ strategyConfig?: import("../../src").RedteamObjectConfig | undefined;
343
+ }) | undefined;
341
344
  description?: string | undefined;
342
345
  providerOutput?: string | {} | undefined;
343
346
  assert?: ({
@@ -414,37 +417,40 @@ export default class EvalFactory {
414
417
  transform?: string | undefined;
415
418
  delay?: number | undefined;
416
419
  env?: {
420
+ PROMPTFOO_REMOTE_GENERATION_URL?: string | undefined;
417
421
  AI21_API_BASE_URL?: string | undefined;
418
422
  AI21_API_KEY?: string | undefined;
419
423
  ANTHROPIC_API_KEY?: string | undefined;
420
424
  AWS_BEDROCK_REGION?: string | undefined;
425
+ FAL_KEY?: string | undefined;
426
+ GROQ_API_KEY?: string | undefined;
427
+ LOCALAI_BASE_URL?: string | undefined;
428
+ WATSONX_AI_APIKEY?: string | undefined;
429
+ WATSONX_AI_PROJECT_ID?: string | undefined;
430
+ WATSONX_AI_BEARER_TOKEN?: string | undefined;
431
+ AZURE_CLIENT_SECRET?: string | undefined;
432
+ AZURE_CLIENT_ID?: string | undefined;
433
+ AZURE_TENANT_ID?: string | undefined;
434
+ AZURE_AUTHORITY_HOST?: string | undefined;
435
+ AZURE_TOKEN_SCOPE?: string | undefined;
436
+ AZURE_DEPLOYMENT_NAME?: string | undefined;
437
+ AZURE_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
438
+ AZURE_OPENAI_DEPLOYMENT_NAME?: string | undefined;
439
+ AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
421
440
  AZURE_OPENAI_API_BASE_URL?: string | undefined;
422
441
  AZURE_OPENAI_API_HOST?: string | undefined;
423
442
  AZURE_OPENAI_API_KEY?: string | undefined;
424
443
  AZURE_API_BASE_URL?: string | undefined;
425
444
  AZURE_API_HOST?: string | undefined;
426
445
  AZURE_API_KEY?: string | undefined;
427
- AZURE_DEPLOYMENT_NAME?: string | undefined;
428
- AZURE_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
429
446
  AZURE_OPENAI_BASE_URL?: string | undefined;
430
- AZURE_OPENAI_DEPLOYMENT_NAME?: string | undefined;
431
- AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
432
- AZURE_CLIENT_SECRET?: string | undefined;
433
- AZURE_CLIENT_ID?: string | undefined;
434
- AZURE_TENANT_ID?: string | undefined;
435
- AZURE_AUTHORITY_HOST?: string | undefined;
436
- AZURE_TOKEN_SCOPE?: string | undefined;
437
447
  BAM_API_HOST?: string | undefined;
438
448
  BAM_API_KEY?: string | undefined;
439
449
  CLOUDFLARE_ACCOUNT_ID?: string | undefined;
440
450
  CLOUDFLARE_API_KEY?: string | undefined;
441
- PROMPTFOO_REMOTE_GENERATION_URL?: string | undefined;
442
451
  COHERE_API_KEY?: string | undefined;
443
- FAL_KEY?: string | undefined;
444
452
  GOOGLE_API_HOST?: string | undefined;
445
453
  GOOGLE_API_KEY?: string | undefined;
446
- GROQ_API_KEY?: string | undefined;
447
- LOCALAI_BASE_URL?: string | undefined;
448
454
  MISTRAL_API_BASE_URL?: string | undefined;
449
455
  MISTRAL_API_HOST?: string | undefined;
450
456
  MISTRAL_API_KEY?: string | undefined;
@@ -462,9 +468,6 @@ export default class EvalFactory {
462
468
  VERTEX_PROJECT_ID?: string | undefined;
463
469
  VERTEX_PUBLISHER?: string | undefined;
464
470
  VERTEX_REGION?: string | undefined;
465
- WATSONX_AI_APIKEY?: string | undefined;
466
- WATSONX_AI_PROJECT_ID?: string | undefined;
467
- WATSONX_AI_BEARER_TOKEN?: string | undefined;
468
471
  } | undefined;
469
472
  } | {
470
473
  callApi: import("../../src").CallApiFunction;
@@ -476,7 +479,10 @@ export default class EvalFactory {
476
479
  callEmbeddingApi?: ((args_0: string, ...args: unknown[]) => Promise<import("../../src").ProviderEmbeddingResponse>) | undefined;
477
480
  callClassificationApi?: ((args_0: string, ...args: unknown[]) => Promise<import("../../src").ProviderClassificationResponse>) | undefined;
478
481
  } | undefined;
479
- metadata?: Record<string, any> | undefined;
482
+ metadata?: (Record<string, any> & {
483
+ pluginConfig?: import("../../src").PluginConfig | undefined;
484
+ strategyConfig?: import("../../src").RedteamObjectConfig | undefined;
485
+ }) | undefined;
480
486
  description?: string | undefined;
481
487
  providerOutput?: string | {} | undefined;
482
488
  assert?: ({
@@ -550,37 +556,40 @@ export default class EvalFactory {
550
556
  transform?: string | undefined;
551
557
  delay?: number | undefined;
552
558
  env?: {
559
+ PROMPTFOO_REMOTE_GENERATION_URL?: string | undefined;
553
560
  AI21_API_BASE_URL?: string | undefined;
554
561
  AI21_API_KEY?: string | undefined;
555
562
  ANTHROPIC_API_KEY?: string | undefined;
556
563
  AWS_BEDROCK_REGION?: string | undefined;
564
+ FAL_KEY?: string | undefined;
565
+ GROQ_API_KEY?: string | undefined;
566
+ LOCALAI_BASE_URL?: string | undefined;
567
+ WATSONX_AI_APIKEY?: string | undefined;
568
+ WATSONX_AI_PROJECT_ID?: string | undefined;
569
+ WATSONX_AI_BEARER_TOKEN?: string | undefined;
570
+ AZURE_CLIENT_SECRET?: string | undefined;
571
+ AZURE_CLIENT_ID?: string | undefined;
572
+ AZURE_TENANT_ID?: string | undefined;
573
+ AZURE_AUTHORITY_HOST?: string | undefined;
574
+ AZURE_TOKEN_SCOPE?: string | undefined;
575
+ AZURE_DEPLOYMENT_NAME?: string | undefined;
576
+ AZURE_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
577
+ AZURE_OPENAI_DEPLOYMENT_NAME?: string | undefined;
578
+ AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
557
579
  AZURE_OPENAI_API_BASE_URL?: string | undefined;
558
580
  AZURE_OPENAI_API_HOST?: string | undefined;
559
581
  AZURE_OPENAI_API_KEY?: string | undefined;
560
582
  AZURE_API_BASE_URL?: string | undefined;
561
583
  AZURE_API_HOST?: string | undefined;
562
584
  AZURE_API_KEY?: string | undefined;
563
- AZURE_DEPLOYMENT_NAME?: string | undefined;
564
- AZURE_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
565
585
  AZURE_OPENAI_BASE_URL?: string | undefined;
566
- AZURE_OPENAI_DEPLOYMENT_NAME?: string | undefined;
567
- AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
568
- AZURE_CLIENT_SECRET?: string | undefined;
569
- AZURE_CLIENT_ID?: string | undefined;
570
- AZURE_TENANT_ID?: string | undefined;
571
- AZURE_AUTHORITY_HOST?: string | undefined;
572
- AZURE_TOKEN_SCOPE?: string | undefined;
573
586
  BAM_API_HOST?: string | undefined;
574
587
  BAM_API_KEY?: string | undefined;
575
588
  CLOUDFLARE_ACCOUNT_ID?: string | undefined;
576
589
  CLOUDFLARE_API_KEY?: string | undefined;
577
- PROMPTFOO_REMOTE_GENERATION_URL?: string | undefined;
578
590
  COHERE_API_KEY?: string | undefined;
579
- FAL_KEY?: string | undefined;
580
591
  GOOGLE_API_HOST?: string | undefined;
581
592
  GOOGLE_API_KEY?: string | undefined;
582
- GROQ_API_KEY?: string | undefined;
583
- LOCALAI_BASE_URL?: string | undefined;
584
593
  MISTRAL_API_BASE_URL?: string | undefined;
585
594
  MISTRAL_API_HOST?: string | undefined;
586
595
  MISTRAL_API_KEY?: string | undefined;
@@ -598,9 +607,6 @@ export default class EvalFactory {
598
607
  VERTEX_PROJECT_ID?: string | undefined;
599
608
  VERTEX_PUBLISHER?: string | undefined;
600
609
  VERTEX_REGION?: string | undefined;
601
- WATSONX_AI_APIKEY?: string | undefined;
602
- WATSONX_AI_PROJECT_ID?: string | undefined;
603
- WATSONX_AI_BEARER_TOKEN?: string | undefined;
604
610
  } | undefined;
605
611
  } | {
606
612
  callApi: import("../../src").CallApiFunction;
@@ -612,7 +618,10 @@ export default class EvalFactory {
612
618
  callEmbeddingApi?: ((args_0: string, ...args: unknown[]) => Promise<import("../../src").ProviderEmbeddingResponse>) | undefined;
613
619
  callClassificationApi?: ((args_0: string, ...args: unknown[]) => Promise<import("../../src").ProviderClassificationResponse>) | undefined;
614
620
  } | undefined;
615
- metadata?: Record<string, any> | undefined;
621
+ metadata?: (Record<string, any> & {
622
+ pluginConfig?: import("../../src").PluginConfig | undefined;
623
+ strategyConfig?: import("../../src").RedteamObjectConfig | undefined;
624
+ }) | undefined;
616
625
  description?: string | undefined;
617
626
  providerOutput?: string | {} | undefined;
618
627
  assert?: ({
@@ -688,37 +697,40 @@ export default class EvalFactory {
688
697
  transform?: string | undefined;
689
698
  delay?: number | undefined;
690
699
  env?: {
700
+ PROMPTFOO_REMOTE_GENERATION_URL?: string | undefined;
691
701
  AI21_API_BASE_URL?: string | undefined;
692
702
  AI21_API_KEY?: string | undefined;
693
703
  ANTHROPIC_API_KEY?: string | undefined;
694
704
  AWS_BEDROCK_REGION?: string | undefined;
705
+ FAL_KEY?: string | undefined;
706
+ GROQ_API_KEY?: string | undefined;
707
+ LOCALAI_BASE_URL?: string | undefined;
708
+ WATSONX_AI_APIKEY?: string | undefined;
709
+ WATSONX_AI_PROJECT_ID?: string | undefined;
710
+ WATSONX_AI_BEARER_TOKEN?: string | undefined;
711
+ AZURE_CLIENT_SECRET?: string | undefined;
712
+ AZURE_CLIENT_ID?: string | undefined;
713
+ AZURE_TENANT_ID?: string | undefined;
714
+ AZURE_AUTHORITY_HOST?: string | undefined;
715
+ AZURE_TOKEN_SCOPE?: string | undefined;
716
+ AZURE_DEPLOYMENT_NAME?: string | undefined;
717
+ AZURE_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
718
+ AZURE_OPENAI_DEPLOYMENT_NAME?: string | undefined;
719
+ AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
695
720
  AZURE_OPENAI_API_BASE_URL?: string | undefined;
696
721
  AZURE_OPENAI_API_HOST?: string | undefined;
697
722
  AZURE_OPENAI_API_KEY?: string | undefined;
698
723
  AZURE_API_BASE_URL?: string | undefined;
699
724
  AZURE_API_HOST?: string | undefined;
700
725
  AZURE_API_KEY?: string | undefined;
701
- AZURE_DEPLOYMENT_NAME?: string | undefined;
702
- AZURE_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
703
726
  AZURE_OPENAI_BASE_URL?: string | undefined;
704
- AZURE_OPENAI_DEPLOYMENT_NAME?: string | undefined;
705
- AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
706
- AZURE_CLIENT_SECRET?: string | undefined;
707
- AZURE_CLIENT_ID?: string | undefined;
708
- AZURE_TENANT_ID?: string | undefined;
709
- AZURE_AUTHORITY_HOST?: string | undefined;
710
- AZURE_TOKEN_SCOPE?: string | undefined;
711
727
  BAM_API_HOST?: string | undefined;
712
728
  BAM_API_KEY?: string | undefined;
713
729
  CLOUDFLARE_ACCOUNT_ID?: string | undefined;
714
730
  CLOUDFLARE_API_KEY?: string | undefined;
715
- PROMPTFOO_REMOTE_GENERATION_URL?: string | undefined;
716
731
  COHERE_API_KEY?: string | undefined;
717
- FAL_KEY?: string | undefined;
718
732
  GOOGLE_API_HOST?: string | undefined;
719
733
  GOOGLE_API_KEY?: string | undefined;
720
- GROQ_API_KEY?: string | undefined;
721
- LOCALAI_BASE_URL?: string | undefined;
722
734
  MISTRAL_API_BASE_URL?: string | undefined;
723
735
  MISTRAL_API_HOST?: string | undefined;
724
736
  MISTRAL_API_KEY?: string | undefined;
@@ -736,9 +748,6 @@ export default class EvalFactory {
736
748
  VERTEX_PROJECT_ID?: string | undefined;
737
749
  VERTEX_PUBLISHER?: string | undefined;
738
750
  VERTEX_REGION?: string | undefined;
739
- WATSONX_AI_APIKEY?: string | undefined;
740
- WATSONX_AI_PROJECT_ID?: string | undefined;
741
- WATSONX_AI_BEARER_TOKEN?: string | undefined;
742
751
  } | undefined;
743
752
  } | {
744
753
  callApi: import("../../src").CallApiFunction;
@@ -750,7 +759,10 @@ export default class EvalFactory {
750
759
  callEmbeddingApi?: ((args_0: string, ...args: unknown[]) => Promise<import("../../src").ProviderEmbeddingResponse>) | undefined;
751
760
  callClassificationApi?: ((args_0: string, ...args: unknown[]) => Promise<import("../../src").ProviderClassificationResponse>) | undefined;
752
761
  } | undefined;
753
- metadata?: Record<string, any> | undefined;
762
+ metadata?: (Record<string, any> & {
763
+ pluginConfig?: import("../../src").PluginConfig | undefined;
764
+ strategyConfig?: import("../../src").RedteamObjectConfig | undefined;
765
+ }) | undefined;
754
766
  providerOutput?: string | {} | undefined;
755
767
  assert?: ({
756
768
  type: import("../../src").AssertionType;
@@ -874,37 +886,40 @@ export default class EvalFactory {
874
886
  transform?: string | undefined;
875
887
  delay?: number | undefined;
876
888
  env?: {
889
+ PROMPTFOO_REMOTE_GENERATION_URL?: string | undefined;
877
890
  AI21_API_BASE_URL?: string | undefined;
878
891
  AI21_API_KEY?: string | undefined;
879
892
  ANTHROPIC_API_KEY?: string | undefined;
880
893
  AWS_BEDROCK_REGION?: string | undefined;
894
+ FAL_KEY?: string | undefined;
895
+ GROQ_API_KEY?: string | undefined;
896
+ LOCALAI_BASE_URL?: string | undefined;
897
+ WATSONX_AI_APIKEY?: string | undefined;
898
+ WATSONX_AI_PROJECT_ID?: string | undefined;
899
+ WATSONX_AI_BEARER_TOKEN?: string | undefined;
900
+ AZURE_CLIENT_SECRET?: string | undefined;
901
+ AZURE_CLIENT_ID?: string | undefined;
902
+ AZURE_TENANT_ID?: string | undefined;
903
+ AZURE_AUTHORITY_HOST?: string | undefined;
904
+ AZURE_TOKEN_SCOPE?: string | undefined;
905
+ AZURE_DEPLOYMENT_NAME?: string | undefined;
906
+ AZURE_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
907
+ AZURE_OPENAI_DEPLOYMENT_NAME?: string | undefined;
908
+ AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
881
909
  AZURE_OPENAI_API_BASE_URL?: string | undefined;
882
910
  AZURE_OPENAI_API_HOST?: string | undefined;
883
911
  AZURE_OPENAI_API_KEY?: string | undefined;
884
912
  AZURE_API_BASE_URL?: string | undefined;
885
913
  AZURE_API_HOST?: string | undefined;
886
914
  AZURE_API_KEY?: string | undefined;
887
- AZURE_DEPLOYMENT_NAME?: string | undefined;
888
- AZURE_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
889
915
  AZURE_OPENAI_BASE_URL?: string | undefined;
890
- AZURE_OPENAI_DEPLOYMENT_NAME?: string | undefined;
891
- AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
892
- AZURE_CLIENT_SECRET?: string | undefined;
893
- AZURE_CLIENT_ID?: string | undefined;
894
- AZURE_TENANT_ID?: string | undefined;
895
- AZURE_AUTHORITY_HOST?: string | undefined;
896
- AZURE_TOKEN_SCOPE?: string | undefined;
897
916
  BAM_API_HOST?: string | undefined;
898
917
  BAM_API_KEY?: string | undefined;
899
918
  CLOUDFLARE_ACCOUNT_ID?: string | undefined;
900
919
  CLOUDFLARE_API_KEY?: string | undefined;
901
- PROMPTFOO_REMOTE_GENERATION_URL?: string | undefined;
902
920
  COHERE_API_KEY?: string | undefined;
903
- FAL_KEY?: string | undefined;
904
921
  GOOGLE_API_HOST?: string | undefined;
905
922
  GOOGLE_API_KEY?: string | undefined;
906
- GROQ_API_KEY?: string | undefined;
907
- LOCALAI_BASE_URL?: string | undefined;
908
923
  MISTRAL_API_BASE_URL?: string | undefined;
909
924
  MISTRAL_API_HOST?: string | undefined;
910
925
  MISTRAL_API_KEY?: string | undefined;
@@ -922,9 +937,6 @@ export default class EvalFactory {
922
937
  VERTEX_PROJECT_ID?: string | undefined;
923
938
  VERTEX_PUBLISHER?: string | undefined;
924
939
  VERTEX_REGION?: string | undefined;
925
- WATSONX_AI_APIKEY?: string | undefined;
926
- WATSONX_AI_PROJECT_ID?: string | undefined;
927
- WATSONX_AI_BEARER_TOKEN?: string | undefined;
928
940
  } | undefined;
929
941
  } | (((args_0: string, args_1: {
930
942
  vars: Record<string, string | {}>;
@@ -956,37 +968,40 @@ export default class EvalFactory {
956
968
  transform?: string | undefined;
957
969
  delay?: number | undefined;
958
970
  env?: {
971
+ PROMPTFOO_REMOTE_GENERATION_URL?: string | undefined;
959
972
  AI21_API_BASE_URL?: string | undefined;
960
973
  AI21_API_KEY?: string | undefined;
961
974
  ANTHROPIC_API_KEY?: string | undefined;
962
975
  AWS_BEDROCK_REGION?: string | undefined;
976
+ FAL_KEY?: string | undefined;
977
+ GROQ_API_KEY?: string | undefined;
978
+ LOCALAI_BASE_URL?: string | undefined;
979
+ WATSONX_AI_APIKEY?: string | undefined;
980
+ WATSONX_AI_PROJECT_ID?: string | undefined;
981
+ WATSONX_AI_BEARER_TOKEN?: string | undefined;
982
+ AZURE_CLIENT_SECRET?: string | undefined;
983
+ AZURE_CLIENT_ID?: string | undefined;
984
+ AZURE_TENANT_ID?: string | undefined;
985
+ AZURE_AUTHORITY_HOST?: string | undefined;
986
+ AZURE_TOKEN_SCOPE?: string | undefined;
987
+ AZURE_DEPLOYMENT_NAME?: string | undefined;
988
+ AZURE_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
989
+ AZURE_OPENAI_DEPLOYMENT_NAME?: string | undefined;
990
+ AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
963
991
  AZURE_OPENAI_API_BASE_URL?: string | undefined;
964
992
  AZURE_OPENAI_API_HOST?: string | undefined;
965
993
  AZURE_OPENAI_API_KEY?: string | undefined;
966
994
  AZURE_API_BASE_URL?: string | undefined;
967
995
  AZURE_API_HOST?: string | undefined;
968
996
  AZURE_API_KEY?: string | undefined;
969
- AZURE_DEPLOYMENT_NAME?: string | undefined;
970
- AZURE_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
971
997
  AZURE_OPENAI_BASE_URL?: string | undefined;
972
- AZURE_OPENAI_DEPLOYMENT_NAME?: string | undefined;
973
- AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
974
- AZURE_CLIENT_SECRET?: string | undefined;
975
- AZURE_CLIENT_ID?: string | undefined;
976
- AZURE_TENANT_ID?: string | undefined;
977
- AZURE_AUTHORITY_HOST?: string | undefined;
978
- AZURE_TOKEN_SCOPE?: string | undefined;
979
998
  BAM_API_HOST?: string | undefined;
980
999
  BAM_API_KEY?: string | undefined;
981
1000
  CLOUDFLARE_ACCOUNT_ID?: string | undefined;
982
1001
  CLOUDFLARE_API_KEY?: string | undefined;
983
- PROMPTFOO_REMOTE_GENERATION_URL?: string | undefined;
984
1002
  COHERE_API_KEY?: string | undefined;
985
- FAL_KEY?: string | undefined;
986
1003
  GOOGLE_API_HOST?: string | undefined;
987
1004
  GOOGLE_API_KEY?: string | undefined;
988
- GROQ_API_KEY?: string | undefined;
989
- LOCALAI_BASE_URL?: string | undefined;
990
1005
  MISTRAL_API_BASE_URL?: string | undefined;
991
1006
  MISTRAL_API_HOST?: string | undefined;
992
1007
  MISTRAL_API_KEY?: string | undefined;
@@ -1004,9 +1019,6 @@ export default class EvalFactory {
1004
1019
  VERTEX_PROJECT_ID?: string | undefined;
1005
1020
  VERTEX_PUBLISHER?: string | undefined;
1006
1021
  VERTEX_REGION?: string | undefined;
1007
- WATSONX_AI_APIKEY?: string | undefined;
1008
- WATSONX_AI_PROJECT_ID?: string | undefined;
1009
- WATSONX_AI_BEARER_TOKEN?: string | undefined;
1010
1022
  } | undefined;
1011
1023
  }>)[] | undefined;
1012
1024
  }>;
@@ -1028,6 +1040,7 @@ export default class EvalFactory {
1028
1040
  completion?: number | undefined;
1029
1041
  prompt?: number | undefined;
1030
1042
  total?: number | undefined;
1043
+ numRequests?: number | undefined;
1031
1044
  };
1032
1045
  score: number;
1033
1046
  testPassCount: number;
package/dist/test/factories/evalFactory.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"evalFactory.d.ts","sourceRoot":"","sources":["../../../test/factories/evalFactory.ts"],"names":[],"mappings":"AAGA,OAAO,IAAI,MAAM,uBAAuB,CAAC;AAGzC,MAAM,CAAC,OAAO,OAAO,WAAW;WACjB,MAAM;WAgJN,eAAe;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAI7B"}
1
+ {"version":3,"file":"evalFactory.d.ts","sourceRoot":"","sources":["../../../test/factories/evalFactory.ts"],"names":[],"mappings":"AAGA,OAAO,IAAI,MAAM,uBAAuB,CAAC;AAGzC,MAAM,CAAC,OAAO,OAAO,WAAW;WACjB,MAAM;WAgJN,eAAe;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAI7B"}