promptfoo 0.96.2 → 0.98.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (289) hide show
  1. package/dist/package.json +15 -15
  2. package/dist/src/app/assets/index-D7I51A2q.js +789 -0
  3. package/dist/src/app/assets/{index-YwjS5Vq2.css → index-DatcZSwS.css} +1 -1
  4. package/dist/src/app/assets/{index.es-CrujH26S.js → index.es-COpBxjTX.js} +1 -1
  5. package/dist/src/app/assets/{sync-gFAH4kmH.js → sync-BY_V2XfM.js} +1 -1
  6. package/dist/src/app/index.html +2 -2
  7. package/dist/src/assertions/index.d.ts.map +1 -1
  8. package/dist/src/assertions/index.js +10 -3
  9. package/dist/src/assertions/index.js.map +1 -1
  10. package/dist/src/assertions/redteam.js +2 -2
  11. package/dist/src/assertions/redteam.js.map +1 -1
  12. package/dist/src/cache.d.ts +1 -0
  13. package/dist/src/cache.d.ts.map +1 -1
  14. package/dist/src/cache.js +5 -0
  15. package/dist/src/cache.js.map +1 -1
  16. package/dist/src/commands/eval.d.ts.map +1 -1
  17. package/dist/src/commands/eval.js +7 -2
  18. package/dist/src/commands/eval.js.map +1 -1
  19. package/dist/src/database/tables.d.ts +201 -182
  20. package/dist/src/database/tables.d.ts.map +1 -1
  21. package/dist/src/envars.d.ts +1 -1
  22. package/dist/src/envars.d.ts.map +1 -1
  23. package/dist/src/evaluator.d.ts.map +1 -1
  24. package/dist/src/evaluator.js +13 -3
  25. package/dist/src/evaluator.js.map +1 -1
  26. package/dist/src/index.d.ts +2 -106
  27. package/dist/src/index.d.ts.map +1 -1
  28. package/dist/src/logger.js +10 -10
  29. package/dist/src/logger.js.map +1 -1
  30. package/dist/src/main.js +15 -1
  31. package/dist/src/main.js.map +1 -1
  32. package/dist/src/models/eval.d.ts +2 -0
  33. package/dist/src/models/eval.d.ts.map +1 -1
  34. package/dist/src/models/eval.js +4 -0
  35. package/dist/src/models/eval.js.map +1 -1
  36. package/dist/src/prompts/processors/yaml.d.ts.map +1 -1
  37. package/dist/src/prompts/processors/yaml.js +7 -2
  38. package/dist/src/prompts/processors/yaml.js.map +1 -1
  39. package/dist/src/providers/adaline.gateway.d.ts +2 -1
  40. package/dist/src/providers/adaline.gateway.d.ts.map +1 -1
  41. package/dist/src/providers/adaline.gateway.js +2 -2
  42. package/dist/src/providers/adaline.gateway.js.map +1 -1
  43. package/dist/src/providers/ai21.d.ts +2 -1
  44. package/dist/src/providers/ai21.d.ts.map +1 -1
  45. package/dist/src/providers/ai21.js.map +1 -1
  46. package/dist/src/providers/anthropic.d.ts +2 -1
  47. package/dist/src/providers/anthropic.d.ts.map +1 -1
  48. package/dist/src/providers/anthropic.js.map +1 -1
  49. package/dist/src/providers/azure.d.ts +12 -4
  50. package/dist/src/providers/azure.d.ts.map +1 -1
  51. package/dist/src/providers/azure.js +106 -62
  52. package/dist/src/providers/azure.js.map +1 -1
  53. package/dist/src/providers/bam.d.ts +2 -1
  54. package/dist/src/providers/bam.d.ts.map +1 -1
  55. package/dist/src/providers/bam.js.map +1 -1
  56. package/dist/src/providers/bedrock.d.ts +6 -2
  57. package/dist/src/providers/bedrock.d.ts.map +1 -1
  58. package/dist/src/providers/bedrock.js +13 -0
  59. package/dist/src/providers/bedrock.js.map +1 -1
  60. package/dist/src/providers/cloudflare-ai.d.ts +2 -1
  61. package/dist/src/providers/cloudflare-ai.d.ts.map +1 -1
  62. package/dist/src/providers/cohere.d.ts +2 -1
  63. package/dist/src/providers/cohere.d.ts.map +1 -1
  64. package/dist/src/providers/defaults.d.ts +2 -1
  65. package/dist/src/providers/defaults.d.ts.map +1 -1
  66. package/dist/src/providers/defaults.js.map +1 -1
  67. package/dist/src/providers/fal.d.ts +2 -1
  68. package/dist/src/providers/fal.d.ts.map +1 -1
  69. package/dist/src/providers/fal.js.map +1 -1
  70. package/dist/src/providers/groq.d.ts +2 -1
  71. package/dist/src/providers/groq.d.ts.map +1 -1
  72. package/dist/src/providers/http.d.ts +5 -1
  73. package/dist/src/providers/http.d.ts.map +1 -1
  74. package/dist/src/providers/http.js +67 -10
  75. package/dist/src/providers/http.js.map +1 -1
  76. package/dist/src/providers/localai.d.ts +2 -1
  77. package/dist/src/providers/localai.d.ts.map +1 -1
  78. package/dist/src/providers/localai.js.map +1 -1
  79. package/dist/src/providers/mistral.d.ts +2 -1
  80. package/dist/src/providers/mistral.d.ts.map +1 -1
  81. package/dist/src/providers/openai.d.ts +2 -1
  82. package/dist/src/providers/openai.d.ts.map +1 -1
  83. package/dist/src/providers/openai.js +1 -1
  84. package/dist/src/providers/openai.js.map +1 -1
  85. package/dist/src/providers/palm.d.ts +2 -1
  86. package/dist/src/providers/palm.d.ts.map +1 -1
  87. package/dist/src/providers/palm.js.map +1 -1
  88. package/dist/src/providers/promptfoo.d.ts +9 -4
  89. package/dist/src/providers/promptfoo.d.ts.map +1 -1
  90. package/dist/src/providers/promptfoo.js +7 -7
  91. package/dist/src/providers/promptfoo.js.map +1 -1
  92. package/dist/src/providers/replicate.d.ts +2 -1
  93. package/dist/src/providers/replicate.d.ts.map +1 -1
  94. package/dist/src/providers/simulatedUser.d.ts.map +1 -1
  95. package/dist/src/providers/simulatedUser.js +5 -0
  96. package/dist/src/providers/simulatedUser.js.map +1 -1
  97. package/dist/src/providers/togetherai.d.ts +2 -1
  98. package/dist/src/providers/togetherai.d.ts.map +1 -1
  99. package/dist/src/providers/togetherai.js.map +1 -1
  100. package/dist/src/providers/vertex.d.ts +2 -1
  101. package/dist/src/providers/vertex.d.ts.map +1 -1
  102. package/dist/src/providers/watsonx.d.ts +2 -1
  103. package/dist/src/providers/watsonx.d.ts.map +1 -1
  104. package/dist/src/providers/watsonx.js.map +1 -1
  105. package/dist/src/providers/xai.d.ts +2 -1
  106. package/dist/src/providers/xai.d.ts.map +1 -1
  107. package/dist/src/providers/xai.js.map +1 -1
  108. package/dist/src/providers.d.ts +2 -1
  109. package/dist/src/providers.d.ts.map +1 -1
  110. package/dist/src/providers.js.map +1 -1
  111. package/dist/src/redteam/commands/generate.js +1 -1
  112. package/dist/src/redteam/commands/generate.js.map +1 -1
  113. package/dist/src/redteam/commands/init.d.ts.map +1 -1
  114. package/dist/src/redteam/commands/init.js +57 -49
  115. package/dist/src/redteam/commands/init.js.map +1 -1
  116. package/dist/src/redteam/commands/setup.d.ts +3 -0
  117. package/dist/src/redteam/commands/setup.d.ts.map +1 -0
  118. package/dist/src/redteam/commands/setup.js +31 -0
  119. package/dist/src/redteam/commands/setup.js.map +1 -0
  120. package/dist/src/redteam/constants.d.ts +2 -2
  121. package/dist/src/redteam/constants.d.ts.map +1 -1
  122. package/dist/src/redteam/constants.js +1 -1
  123. package/dist/src/redteam/constants.js.map +1 -1
  124. package/dist/src/redteam/graders.d.ts +2 -77
  125. package/dist/src/redteam/graders.d.ts.map +1 -1
  126. package/dist/src/redteam/graders.js +25 -25
  127. package/dist/src/redteam/graders.js.map +1 -1
  128. package/dist/src/redteam/index.d.ts.map +1 -1
  129. package/dist/src/redteam/index.js +46 -23
  130. package/dist/src/redteam/index.js.map +1 -1
  131. package/dist/src/redteam/plugins/base.d.ts +12 -7
  132. package/dist/src/redteam/plugins/base.d.ts.map +1 -1
  133. package/dist/src/redteam/plugins/base.js +42 -28
  134. package/dist/src/redteam/plugins/base.js.map +1 -1
  135. package/dist/src/redteam/plugins/competitors.d.ts.map +1 -1
  136. package/dist/src/redteam/plugins/competitors.js +2 -0
  137. package/dist/src/redteam/plugins/competitors.js.map +1 -1
  138. package/dist/src/redteam/plugins/debugAccess.d.ts.map +1 -1
  139. package/dist/src/redteam/plugins/debugAccess.js +18 -0
  140. package/dist/src/redteam/plugins/debugAccess.js.map +1 -1
  141. package/dist/src/redteam/plugins/harmful/aligned.d.ts +13 -0
  142. package/dist/src/redteam/plugins/harmful/aligned.d.ts.map +1 -0
  143. package/dist/src/redteam/plugins/harmful/aligned.js +29 -0
  144. package/dist/src/redteam/plugins/harmful/aligned.js.map +1 -0
  145. package/dist/src/redteam/plugins/harmful/common.d.ts +5 -0
  146. package/dist/src/redteam/plugins/harmful/common.d.ts.map +1 -0
  147. package/dist/src/redteam/plugins/harmful/common.js +39 -0
  148. package/dist/src/redteam/plugins/harmful/common.js.map +1 -0
  149. package/dist/src/redteam/plugins/harmful/constants.d.ts +10 -0
  150. package/dist/src/redteam/plugins/harmful/constants.d.ts.map +1 -0
  151. package/dist/src/redteam/plugins/harmful/constants.js +166 -0
  152. package/dist/src/redteam/plugins/harmful/constants.js.map +1 -0
  153. package/dist/src/redteam/plugins/harmful/graders.d.ts +15 -0
  154. package/dist/src/redteam/plugins/harmful/graders.d.ts.map +1 -0
  155. package/dist/src/redteam/plugins/harmful/graders.js +94 -0
  156. package/dist/src/redteam/plugins/harmful/graders.js.map +1 -0
  157. package/dist/src/redteam/plugins/harmful/unaligned.d.ts +5 -0
  158. package/dist/src/redteam/plugins/harmful/unaligned.d.ts.map +1 -0
  159. package/dist/src/redteam/plugins/harmful/unaligned.js +29 -0
  160. package/dist/src/redteam/plugins/harmful/unaligned.js.map +1 -0
  161. package/dist/src/redteam/plugins/index.d.ts.map +1 -1
  162. package/dist/src/redteam/plugins/index.js +29 -26
  163. package/dist/src/redteam/plugins/index.js.map +1 -1
  164. package/dist/src/redteam/plugins/intent.d.ts.map +1 -1
  165. package/dist/src/redteam/plugins/intent.js +6 -0
  166. package/dist/src/redteam/plugins/intent.js.map +1 -1
  167. package/dist/src/redteam/providers/crescendo/index.d.ts +9 -0
  168. package/dist/src/redteam/providers/crescendo/index.d.ts.map +1 -1
  169. package/dist/src/redteam/providers/crescendo/index.js +96 -39
  170. package/dist/src/redteam/providers/crescendo/index.js.map +1 -1
  171. package/dist/src/redteam/providers/goat.d.ts.map +1 -1
  172. package/dist/src/redteam/providers/goat.js +33 -1
  173. package/dist/src/redteam/providers/goat.js.map +1 -1
  174. package/dist/src/redteam/providers/iterative.d.ts +7 -0
  175. package/dist/src/redteam/providers/iterative.d.ts.map +1 -1
  176. package/dist/src/redteam/providers/iterative.js +55 -3
  177. package/dist/src/redteam/providers/iterative.js.map +1 -1
  178. package/dist/src/redteam/providers/iterativeImage.d.ts +9 -1
  179. package/dist/src/redteam/providers/iterativeImage.d.ts.map +1 -1
  180. package/dist/src/redteam/providers/iterativeImage.js +42 -12
  181. package/dist/src/redteam/providers/iterativeImage.js.map +1 -1
  182. package/dist/src/redteam/providers/iterativeTree.d.ts +7 -2
  183. package/dist/src/redteam/providers/iterativeTree.d.ts.map +1 -1
  184. package/dist/src/redteam/providers/iterativeTree.js +56 -11
  185. package/dist/src/redteam/providers/iterativeTree.js.map +1 -1
  186. package/dist/src/redteam/providers/shared.d.ts +9 -3
  187. package/dist/src/redteam/providers/shared.d.ts.map +1 -1
  188. package/dist/src/redteam/providers/shared.js +15 -6
  189. package/dist/src/redteam/providers/shared.js.map +1 -1
  190. package/dist/src/redteam/strategies/index.d.ts +3 -2
  191. package/dist/src/redteam/strategies/index.d.ts.map +1 -1
  192. package/dist/src/redteam/strategies/index.js +48 -13
  193. package/dist/src/redteam/strategies/index.js.map +1 -1
  194. package/dist/src/redteam/strategies/multilingual.d.ts.map +1 -1
  195. package/dist/src/redteam/strategies/multilingual.js +1 -0
  196. package/dist/src/redteam/strategies/multilingual.js.map +1 -1
  197. package/dist/src/redteam/types.d.ts +6 -0
  198. package/dist/src/redteam/types.d.ts.map +1 -1
  199. package/dist/src/server/routes/providers.js +1 -0
  200. package/dist/src/server/routes/providers.js.map +1 -1
  201. package/dist/src/server/server.d.ts +2 -1
  202. package/dist/src/server/server.d.ts.map +1 -1
  203. package/dist/src/server/server.js +13 -3
  204. package/dist/src/server/server.js.map +1 -1
  205. package/dist/src/share.d.ts.map +1 -1
  206. package/dist/src/share.js +11 -0
  207. package/dist/src/share.js.map +1 -1
  208. package/dist/src/types/env.d.ts +54 -0
  209. package/dist/src/types/env.d.ts.map +1 -0
  210. package/dist/src/types/env.js +3 -0
  211. package/dist/src/types/env.js.map +1 -0
  212. package/dist/src/types/index.d.ts +8185 -6164
  213. package/dist/src/types/index.d.ts.map +1 -1
  214. package/dist/src/types/index.js +8 -2
  215. package/dist/src/types/index.js.map +1 -1
  216. package/dist/src/types/providers.d.ts +3 -53
  217. package/dist/src/types/providers.d.ts.map +1 -1
  218. package/dist/src/types/providers.js.map +1 -1
  219. package/dist/src/types/shared.d.ts +1 -0
  220. package/dist/src/types/shared.d.ts.map +1 -1
  221. package/dist/src/util/config/load.js +1 -1
  222. package/dist/src/util/config/load.js.map +1 -1
  223. package/dist/src/util/exportToFile/index.d.ts +1 -0
  224. package/dist/src/util/exportToFile/index.d.ts.map +1 -1
  225. package/dist/src/util/index.d.ts +42 -34
  226. package/dist/src/util/index.d.ts.map +1 -1
  227. package/dist/src/util/transform.d.ts +1 -0
  228. package/dist/src/util/transform.d.ts.map +1 -1
  229. package/dist/src/util/transform.js.map +1 -1
  230. package/dist/src/validators/providers.d.ts +311 -296
  231. package/dist/src/validators/providers.d.ts.map +1 -1
  232. package/dist/src/validators/redteam.d.ts +126 -126
  233. package/dist/src/validators/redteam.d.ts.map +1 -1
  234. package/dist/src/validators/redteam.js +14 -2
  235. package/dist/src/validators/redteam.js.map +1 -1
  236. package/dist/src/validators/shared.d.ts +3 -0
  237. package/dist/src/validators/shared.d.ts.map +1 -1
  238. package/dist/src/validators/shared.js +1 -0
  239. package/dist/src/validators/shared.js.map +1 -1
  240. package/dist/test/cache.test.js +47 -7
  241. package/dist/test/cache.test.js.map +1 -1
  242. package/dist/test/commands/init.test.js +14 -0
  243. package/dist/test/commands/init.test.js.map +1 -1
  244. package/dist/test/evaluator.test.js +136 -32
  245. package/dist/test/evaluator.test.js.map +1 -1
  246. package/dist/test/factories/evalFactory.d.ts +161 -148
  247. package/dist/test/factories/evalFactory.d.ts.map +1 -1
  248. package/dist/test/prompts/index.test.js +38 -25
  249. package/dist/test/prompts/index.test.js.map +1 -1
  250. package/dist/test/prompts/processors/yaml.test.js +50 -4
  251. package/dist/test/prompts/processors/yaml.test.js.map +1 -1
  252. package/dist/test/providers/http.test.js +0 -17
  253. package/dist/test/providers/http.test.js.map +1 -1
  254. package/dist/test/providers/index.test.js +1 -0
  255. package/dist/test/providers/index.test.js.map +1 -1
  256. package/dist/test/providers/togetherai.test.js.map +1 -1
  257. package/dist/test/redteam/index.test.js +2 -2
  258. package/dist/test/redteam/index.test.js.map +1 -1
  259. package/dist/test/redteam/plugins/base.test.js +61 -23
  260. package/dist/test/redteam/plugins/base.test.js.map +1 -1
  261. package/dist/test/redteam/plugins/harmful/aligned.test.d.ts +2 -0
  262. package/dist/test/redteam/plugins/harmful/aligned.test.d.ts.map +1 -0
  263. package/dist/test/redteam/plugins/harmful/aligned.test.js +123 -0
  264. package/dist/test/redteam/plugins/harmful/aligned.test.js.map +1 -0
  265. package/dist/test/redteam/plugins/harmful/common.test.d.ts +2 -0
  266. package/dist/test/redteam/plugins/harmful/common.test.d.ts.map +1 -0
  267. package/dist/test/redteam/plugins/harmful/common.test.js +105 -0
  268. package/dist/test/redteam/plugins/harmful/common.test.js.map +1 -0
  269. package/dist/test/redteam/plugins/harmful/unaligned.test.d.ts +2 -0
  270. package/dist/test/redteam/plugins/harmful/unaligned.test.d.ts.map +1 -0
  271. package/dist/test/redteam/plugins/harmful/unaligned.test.js +150 -0
  272. package/dist/test/redteam/plugins/harmful/unaligned.test.js.map +1 -0
  273. package/dist/test/redteam/plugins/index.test.d.ts +2 -0
  274. package/dist/test/redteam/plugins/index.test.d.ts.map +1 -0
  275. package/dist/test/redteam/plugins/index.test.js +172 -0
  276. package/dist/test/redteam/plugins/index.test.js.map +1 -0
  277. package/dist/test/redteam/providers/iterativeTree.test.js +13 -6
  278. package/dist/test/redteam/providers/iterativeTree.test.js.map +1 -1
  279. package/dist/test/share.test.js +45 -1
  280. package/dist/test/share.test.js.map +1 -1
  281. package/dist/test/types.test.js +15 -1
  282. package/dist/test/types.test.js.map +1 -1
  283. package/dist/tsconfig.tsbuildinfo +1 -1
  284. package/package.json +15 -15
  285. package/dist/src/app/assets/index-CAof5cIE.js +0 -769
  286. package/dist/src/redteam/plugins/harmful.d.ts +0 -18
  287. package/dist/src/redteam/plugins/harmful.d.ts.map +0 -1
  288. package/dist/src/redteam/plugins/harmful.js +0 -384
  289. package/dist/src/redteam/plugins/harmful.js.map +0 -1
package/dist/src/database/tables.d.ts CHANGED
@@ -215,37 +215,40 @@ export declare const evalsTable: import("drizzle-orm/sqlite-core").SQLiteTableWi
215
215
  label?: string | undefined;
216
216
  })[];
217
217
  env?: {
218
+ PROMPTFOO_REMOTE_GENERATION_URL?: string | undefined;
218
219
  AI21_API_BASE_URL?: string | undefined;
219
220
  AI21_API_KEY?: string | undefined;
220
221
  ANTHROPIC_API_KEY?: string | undefined;
221
222
  AWS_BEDROCK_REGION?: string | undefined;
223
+ FAL_KEY?: string | undefined;
224
+ GROQ_API_KEY?: string | undefined;
225
+ LOCALAI_BASE_URL?: string | undefined;
226
+ WATSONX_AI_APIKEY?: string | undefined;
227
+ WATSONX_AI_PROJECT_ID?: string | undefined;
228
+ WATSONX_AI_BEARER_TOKEN?: string | undefined;
229
+ AZURE_CLIENT_SECRET?: string | undefined;
230
+ AZURE_CLIENT_ID?: string | undefined;
231
+ AZURE_TENANT_ID?: string | undefined;
232
+ AZURE_AUTHORITY_HOST?: string | undefined;
233
+ AZURE_TOKEN_SCOPE?: string | undefined;
234
+ AZURE_DEPLOYMENT_NAME?: string | undefined;
235
+ AZURE_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
236
+ AZURE_OPENAI_DEPLOYMENT_NAME?: string | undefined;
237
+ AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
222
238
  AZURE_OPENAI_API_BASE_URL?: string | undefined;
223
239
  AZURE_OPENAI_API_HOST?: string | undefined;
224
240
  AZURE_OPENAI_API_KEY?: string | undefined;
225
241
  AZURE_API_BASE_URL?: string | undefined;
226
242
  AZURE_API_HOST?: string | undefined;
227
243
  AZURE_API_KEY?: string | undefined;
228
- AZURE_DEPLOYMENT_NAME?: string | undefined;
229
- AZURE_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
230
244
  AZURE_OPENAI_BASE_URL?: string | undefined;
231
- AZURE_OPENAI_DEPLOYMENT_NAME?: string | undefined;
232
- AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
233
- AZURE_CLIENT_SECRET?: string | undefined;
234
- AZURE_CLIENT_ID?: string | undefined;
235
- AZURE_TENANT_ID?: string | undefined;
236
- AZURE_AUTHORITY_HOST?: string | undefined;
237
- AZURE_TOKEN_SCOPE?: string | undefined;
238
245
  BAM_API_HOST?: string | undefined;
239
246
  BAM_API_KEY?: string | undefined;
240
247
  CLOUDFLARE_ACCOUNT_ID?: string | undefined;
241
248
  CLOUDFLARE_API_KEY?: string | undefined;
242
- PROMPTFOO_REMOTE_GENERATION_URL?: string | undefined;
243
249
  COHERE_API_KEY?: string | undefined;
244
- FAL_KEY?: string | undefined;
245
250
  GOOGLE_API_HOST?: string | undefined;
246
251
  GOOGLE_API_KEY?: string | undefined;
247
- GROQ_API_KEY?: string | undefined;
248
- LOCALAI_BASE_URL?: string | undefined;
249
252
  MISTRAL_API_BASE_URL?: string | undefined;
250
253
  MISTRAL_API_HOST?: string | undefined;
251
254
  MISTRAL_API_KEY?: string | undefined;
@@ -263,9 +266,6 @@ export declare const evalsTable: import("drizzle-orm/sqlite-core").SQLiteTableWi
263
266
  VERTEX_PROJECT_ID?: string | undefined;
264
267
  VERTEX_PUBLISHER?: string | undefined;
265
268
  VERTEX_REGION?: string | undefined;
266
- WATSONX_AI_APIKEY?: string | undefined;
267
- WATSONX_AI_PROJECT_ID?: string | undefined;
268
- WATSONX_AI_BEARER_TOKEN?: string | undefined;
269
269
  } | undefined;
270
270
  metadata?: Record<string, any> | undefined;
271
271
  description?: string | undefined;
@@ -299,37 +299,40 @@ export declare const evalsTable: import("drizzle-orm/sqlite-core").SQLiteTableWi
299
299
  transform?: string | undefined;
300
300
  delay?: number | undefined;
301
301
  env?: {
302
+ PROMPTFOO_REMOTE_GENERATION_URL?: string | undefined;
302
303
  AI21_API_BASE_URL?: string | undefined;
303
304
  AI21_API_KEY?: string | undefined;
304
305
  ANTHROPIC_API_KEY?: string | undefined;
305
306
  AWS_BEDROCK_REGION?: string | undefined;
307
+ FAL_KEY?: string | undefined;
308
+ GROQ_API_KEY?: string | undefined;
309
+ LOCALAI_BASE_URL?: string | undefined;
310
+ WATSONX_AI_APIKEY?: string | undefined;
311
+ WATSONX_AI_PROJECT_ID?: string | undefined;
312
+ WATSONX_AI_BEARER_TOKEN?: string | undefined;
313
+ AZURE_CLIENT_SECRET?: string | undefined;
314
+ AZURE_CLIENT_ID?: string | undefined;
315
+ AZURE_TENANT_ID?: string | undefined;
316
+ AZURE_AUTHORITY_HOST?: string | undefined;
317
+ AZURE_TOKEN_SCOPE?: string | undefined;
318
+ AZURE_DEPLOYMENT_NAME?: string | undefined;
319
+ AZURE_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
320
+ AZURE_OPENAI_DEPLOYMENT_NAME?: string | undefined;
321
+ AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
306
322
  AZURE_OPENAI_API_BASE_URL?: string | undefined;
307
323
  AZURE_OPENAI_API_HOST?: string | undefined;
308
324
  AZURE_OPENAI_API_KEY?: string | undefined;
309
325
  AZURE_API_BASE_URL?: string | undefined;
310
326
  AZURE_API_HOST?: string | undefined;
311
327
  AZURE_API_KEY?: string | undefined;
312
- AZURE_DEPLOYMENT_NAME?: string | undefined;
313
- AZURE_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
314
328
  AZURE_OPENAI_BASE_URL?: string | undefined;
315
- AZURE_OPENAI_DEPLOYMENT_NAME?: string | undefined;
316
- AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
317
- AZURE_CLIENT_SECRET?: string | undefined;
318
- AZURE_CLIENT_ID?: string | undefined;
319
- AZURE_TENANT_ID?: string | undefined;
320
- AZURE_AUTHORITY_HOST?: string | undefined;
321
- AZURE_TOKEN_SCOPE?: string | undefined;
322
329
  BAM_API_HOST?: string | undefined;
323
330
  BAM_API_KEY?: string | undefined;
324
331
  CLOUDFLARE_ACCOUNT_ID?: string | undefined;
325
332
  CLOUDFLARE_API_KEY?: string | undefined;
326
- PROMPTFOO_REMOTE_GENERATION_URL?: string | undefined;
327
333
  COHERE_API_KEY?: string | undefined;
328
- FAL_KEY?: string | undefined;
329
334
  GOOGLE_API_HOST?: string | undefined;
330
335
  GOOGLE_API_KEY?: string | undefined;
331
- GROQ_API_KEY?: string | undefined;
332
- LOCALAI_BASE_URL?: string | undefined;
333
336
  MISTRAL_API_BASE_URL?: string | undefined;
334
337
  MISTRAL_API_HOST?: string | undefined;
335
338
  MISTRAL_API_KEY?: string | undefined;
@@ -347,9 +350,6 @@ export declare const evalsTable: import("drizzle-orm/sqlite-core").SQLiteTableWi
347
350
  VERTEX_PROJECT_ID?: string | undefined;
348
351
  VERTEX_PUBLISHER?: string | undefined;
349
352
  VERTEX_REGION?: string | undefined;
350
- WATSONX_AI_APIKEY?: string | undefined;
351
- WATSONX_AI_PROJECT_ID?: string | undefined;
352
- WATSONX_AI_BEARER_TOKEN?: string | undefined;
353
353
  } | undefined;
354
354
  } | (((args_0: string, args_1: {
355
355
  vars: Record<string, string | {}>;
@@ -381,37 +381,40 @@ export declare const evalsTable: import("drizzle-orm/sqlite-core").SQLiteTableWi
381
381
  transform?: string | undefined;
382
382
  delay?: number | undefined;
383
383
  env?: {
384
+ PROMPTFOO_REMOTE_GENERATION_URL?: string | undefined;
384
385
  AI21_API_BASE_URL?: string | undefined;
385
386
  AI21_API_KEY?: string | undefined;
386
387
  ANTHROPIC_API_KEY?: string | undefined;
387
388
  AWS_BEDROCK_REGION?: string | undefined;
389
+ FAL_KEY?: string | undefined;
390
+ GROQ_API_KEY?: string | undefined;
391
+ LOCALAI_BASE_URL?: string | undefined;
392
+ WATSONX_AI_APIKEY?: string | undefined;
393
+ WATSONX_AI_PROJECT_ID?: string | undefined;
394
+ WATSONX_AI_BEARER_TOKEN?: string | undefined;
395
+ AZURE_CLIENT_SECRET?: string | undefined;
396
+ AZURE_CLIENT_ID?: string | undefined;
397
+ AZURE_TENANT_ID?: string | undefined;
398
+ AZURE_AUTHORITY_HOST?: string | undefined;
399
+ AZURE_TOKEN_SCOPE?: string | undefined;
400
+ AZURE_DEPLOYMENT_NAME?: string | undefined;
401
+ AZURE_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
402
+ AZURE_OPENAI_DEPLOYMENT_NAME?: string | undefined;
403
+ AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
388
404
  AZURE_OPENAI_API_BASE_URL?: string | undefined;
389
405
  AZURE_OPENAI_API_HOST?: string | undefined;
390
406
  AZURE_OPENAI_API_KEY?: string | undefined;
391
407
  AZURE_API_BASE_URL?: string | undefined;
392
408
  AZURE_API_HOST?: string | undefined;
393
409
  AZURE_API_KEY?: string | undefined;
394
- AZURE_DEPLOYMENT_NAME?: string | undefined;
395
- AZURE_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
396
410
  AZURE_OPENAI_BASE_URL?: string | undefined;
397
- AZURE_OPENAI_DEPLOYMENT_NAME?: string | undefined;
398
- AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
399
- AZURE_CLIENT_SECRET?: string | undefined;
400
- AZURE_CLIENT_ID?: string | undefined;
401
- AZURE_TENANT_ID?: string | undefined;
402
- AZURE_AUTHORITY_HOST?: string | undefined;
403
- AZURE_TOKEN_SCOPE?: string | undefined;
404
411
  BAM_API_HOST?: string | undefined;
405
412
  BAM_API_KEY?: string | undefined;
406
413
  CLOUDFLARE_ACCOUNT_ID?: string | undefined;
407
414
  CLOUDFLARE_API_KEY?: string | undefined;
408
- PROMPTFOO_REMOTE_GENERATION_URL?: string | undefined;
409
415
  COHERE_API_KEY?: string | undefined;
410
- FAL_KEY?: string | undefined;
411
416
  GOOGLE_API_HOST?: string | undefined;
412
417
  GOOGLE_API_KEY?: string | undefined;
413
- GROQ_API_KEY?: string | undefined;
414
- LOCALAI_BASE_URL?: string | undefined;
415
418
  MISTRAL_API_BASE_URL?: string | undefined;
416
419
  MISTRAL_API_HOST?: string | undefined;
417
420
  MISTRAL_API_KEY?: string | undefined;
@@ -429,9 +432,6 @@ export declare const evalsTable: import("drizzle-orm/sqlite-core").SQLiteTableWi
429
432
  VERTEX_PROJECT_ID?: string | undefined;
430
433
  VERTEX_PUBLISHER?: string | undefined;
431
434
  VERTEX_REGION?: string | undefined;
432
- WATSONX_AI_APIKEY?: string | undefined;
433
- WATSONX_AI_PROJECT_ID?: string | undefined;
434
- WATSONX_AI_BEARER_TOKEN?: string | undefined;
435
435
  } | undefined;
436
436
  }>)[] | undefined;
437
437
  tests?: string | (string | {
@@ -470,37 +470,40 @@ export declare const evalsTable: import("drizzle-orm/sqlite-core").SQLiteTableWi
470
470
  transform?: string | undefined;
471
471
  delay?: number | undefined;
472
472
  env?: {
473
+ PROMPTFOO_REMOTE_GENERATION_URL?: string | undefined;
473
474
  AI21_API_BASE_URL?: string | undefined;
474
475
  AI21_API_KEY?: string | undefined;
475
476
  ANTHROPIC_API_KEY?: string | undefined;
476
477
  AWS_BEDROCK_REGION?: string | undefined;
478
+ FAL_KEY?: string | undefined;
479
+ GROQ_API_KEY?: string | undefined;
480
+ LOCALAI_BASE_URL?: string | undefined;
481
+ WATSONX_AI_APIKEY?: string | undefined;
482
+ WATSONX_AI_PROJECT_ID?: string | undefined;
483
+ WATSONX_AI_BEARER_TOKEN?: string | undefined;
484
+ AZURE_CLIENT_SECRET?: string | undefined;
485
+ AZURE_CLIENT_ID?: string | undefined;
486
+ AZURE_TENANT_ID?: string | undefined;
487
+ AZURE_AUTHORITY_HOST?: string | undefined;
488
+ AZURE_TOKEN_SCOPE?: string | undefined;
489
+ AZURE_DEPLOYMENT_NAME?: string | undefined;
490
+ AZURE_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
491
+ AZURE_OPENAI_DEPLOYMENT_NAME?: string | undefined;
492
+ AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
477
493
  AZURE_OPENAI_API_BASE_URL?: string | undefined;
478
494
  AZURE_OPENAI_API_HOST?: string | undefined;
479
495
  AZURE_OPENAI_API_KEY?: string | undefined;
480
496
  AZURE_API_BASE_URL?: string | undefined;
481
497
  AZURE_API_HOST?: string | undefined;
482
498
  AZURE_API_KEY?: string | undefined;
483
- AZURE_DEPLOYMENT_NAME?: string | undefined;
484
- AZURE_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
485
499
  AZURE_OPENAI_BASE_URL?: string | undefined;
486
- AZURE_OPENAI_DEPLOYMENT_NAME?: string | undefined;
487
- AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
488
- AZURE_CLIENT_SECRET?: string | undefined;
489
- AZURE_CLIENT_ID?: string | undefined;
490
- AZURE_TENANT_ID?: string | undefined;
491
- AZURE_AUTHORITY_HOST?: string | undefined;
492
- AZURE_TOKEN_SCOPE?: string | undefined;
493
500
  BAM_API_HOST?: string | undefined;
494
501
  BAM_API_KEY?: string | undefined;
495
502
  CLOUDFLARE_ACCOUNT_ID?: string | undefined;
496
503
  CLOUDFLARE_API_KEY?: string | undefined;
497
- PROMPTFOO_REMOTE_GENERATION_URL?: string | undefined;
498
504
  COHERE_API_KEY?: string | undefined;
499
- FAL_KEY?: string | undefined;
500
505
  GOOGLE_API_HOST?: string | undefined;
501
506
  GOOGLE_API_KEY?: string | undefined;
502
- GROQ_API_KEY?: string | undefined;
503
- LOCALAI_BASE_URL?: string | undefined;
504
507
  MISTRAL_API_BASE_URL?: string | undefined;
505
508
  MISTRAL_API_HOST?: string | undefined;
506
509
  MISTRAL_API_KEY?: string | undefined;
@@ -518,9 +521,6 @@ export declare const evalsTable: import("drizzle-orm/sqlite-core").SQLiteTableWi
518
521
  VERTEX_PROJECT_ID?: string | undefined;
519
522
  VERTEX_PUBLISHER?: string | undefined;
520
523
  VERTEX_REGION?: string | undefined;
521
- WATSONX_AI_APIKEY?: string | undefined;
522
- WATSONX_AI_PROJECT_ID?: string | undefined;
523
- WATSONX_AI_BEARER_TOKEN?: string | undefined;
524
524
  } | undefined;
525
525
  } | {
526
526
  callApi: import("../types").CallApiFunction;
@@ -532,7 +532,10 @@ export declare const evalsTable: import("drizzle-orm/sqlite-core").SQLiteTableWi
532
532
  callEmbeddingApi?: ((args_0: string, ...args: unknown[]) => Promise<import("../types").ProviderEmbeddingResponse>) | undefined;
533
533
  callClassificationApi?: ((args_0: string, ...args: unknown[]) => Promise<import("../types").ProviderClassificationResponse>) | undefined;
534
534
  } | undefined;
535
- metadata?: Record<string, any> | undefined;
535
+ metadata?: (Record<string, any> & {
536
+ pluginConfig?: import("../types").PluginConfig | undefined;
537
+ strategyConfig?: import("../types").RedteamObjectConfig | undefined;
538
+ }) | undefined;
536
539
  description?: string | undefined;
537
540
  providerOutput?: string | {} | undefined;
538
541
  assert?: ({
@@ -609,37 +612,40 @@ export declare const evalsTable: import("drizzle-orm/sqlite-core").SQLiteTableWi
609
612
  transform?: string | undefined;
610
613
  delay?: number | undefined;
611
614
  env?: {
615
+ PROMPTFOO_REMOTE_GENERATION_URL?: string | undefined;
612
616
  AI21_API_BASE_URL?: string | undefined;
613
617
  AI21_API_KEY?: string | undefined;
614
618
  ANTHROPIC_API_KEY?: string | undefined;
615
619
  AWS_BEDROCK_REGION?: string | undefined;
620
+ FAL_KEY?: string | undefined;
621
+ GROQ_API_KEY?: string | undefined;
622
+ LOCALAI_BASE_URL?: string | undefined;
623
+ WATSONX_AI_APIKEY?: string | undefined;
624
+ WATSONX_AI_PROJECT_ID?: string | undefined;
625
+ WATSONX_AI_BEARER_TOKEN?: string | undefined;
626
+ AZURE_CLIENT_SECRET?: string | undefined;
627
+ AZURE_CLIENT_ID?: string | undefined;
628
+ AZURE_TENANT_ID?: string | undefined;
629
+ AZURE_AUTHORITY_HOST?: string | undefined;
630
+ AZURE_TOKEN_SCOPE?: string | undefined;
631
+ AZURE_DEPLOYMENT_NAME?: string | undefined;
632
+ AZURE_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
633
+ AZURE_OPENAI_DEPLOYMENT_NAME?: string | undefined;
634
+ AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
616
635
  AZURE_OPENAI_API_BASE_URL?: string | undefined;
617
636
  AZURE_OPENAI_API_HOST?: string | undefined;
618
637
  AZURE_OPENAI_API_KEY?: string | undefined;
619
638
  AZURE_API_BASE_URL?: string | undefined;
620
639
  AZURE_API_HOST?: string | undefined;
621
640
  AZURE_API_KEY?: string | undefined;
622
- AZURE_DEPLOYMENT_NAME?: string | undefined;
623
- AZURE_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
624
641
  AZURE_OPENAI_BASE_URL?: string | undefined;
625
- AZURE_OPENAI_DEPLOYMENT_NAME?: string | undefined;
626
- AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
627
- AZURE_CLIENT_SECRET?: string | undefined;
628
- AZURE_CLIENT_ID?: string | undefined;
629
- AZURE_TENANT_ID?: string | undefined;
630
- AZURE_AUTHORITY_HOST?: string | undefined;
631
- AZURE_TOKEN_SCOPE?: string | undefined;
632
642
  BAM_API_HOST?: string | undefined;
633
643
  BAM_API_KEY?: string | undefined;
634
644
  CLOUDFLARE_ACCOUNT_ID?: string | undefined;
635
645
  CLOUDFLARE_API_KEY?: string | undefined;
636
- PROMPTFOO_REMOTE_GENERATION_URL?: string | undefined;
637
646
  COHERE_API_KEY?: string | undefined;
638
- FAL_KEY?: string | undefined;
639
647
  GOOGLE_API_HOST?: string | undefined;
640
648
  GOOGLE_API_KEY?: string | undefined;
641
- GROQ_API_KEY?: string | undefined;
642
- LOCALAI_BASE_URL?: string | undefined;
643
649
  MISTRAL_API_BASE_URL?: string | undefined;
644
650
  MISTRAL_API_HOST?: string | undefined;
645
651
  MISTRAL_API_KEY?: string | undefined;
@@ -657,9 +663,6 @@ export declare const evalsTable: import("drizzle-orm/sqlite-core").SQLiteTableWi
657
663
  VERTEX_PROJECT_ID?: string | undefined;
658
664
  VERTEX_PUBLISHER?: string | undefined;
659
665
  VERTEX_REGION?: string | undefined;
660
- WATSONX_AI_APIKEY?: string | undefined;
661
- WATSONX_AI_PROJECT_ID?: string | undefined;
662
- WATSONX_AI_BEARER_TOKEN?: string | undefined;
663
666
  } | undefined;
664
667
  } | {
665
668
  callApi: import("../types").CallApiFunction;
@@ -671,7 +674,10 @@ export declare const evalsTable: import("drizzle-orm/sqlite-core").SQLiteTableWi
671
674
  callEmbeddingApi?: ((args_0: string, ...args: unknown[]) => Promise<import("../types").ProviderEmbeddingResponse>) | undefined;
672
675
  callClassificationApi?: ((args_0: string, ...args: unknown[]) => Promise<import("../types").ProviderClassificationResponse>) | undefined;
673
676
  } | undefined;
674
- metadata?: Record<string, any> | undefined;
677
+ metadata?: (Record<string, any> & {
678
+ pluginConfig?: import("../types").PluginConfig | undefined;
679
+ strategyConfig?: import("../types").RedteamObjectConfig | undefined;
680
+ }) | undefined;
675
681
  description?: string | undefined;
676
682
  providerOutput?: string | {} | undefined;
677
683
  assert?: ({
@@ -745,37 +751,40 @@ export declare const evalsTable: import("drizzle-orm/sqlite-core").SQLiteTableWi
745
751
  transform?: string | undefined;
746
752
  delay?: number | undefined;
747
753
  env?: {
754
+ PROMPTFOO_REMOTE_GENERATION_URL?: string | undefined;
748
755
  AI21_API_BASE_URL?: string | undefined;
749
756
  AI21_API_KEY?: string | undefined;
750
757
  ANTHROPIC_API_KEY?: string | undefined;
751
758
  AWS_BEDROCK_REGION?: string | undefined;
759
+ FAL_KEY?: string | undefined;
760
+ GROQ_API_KEY?: string | undefined;
761
+ LOCALAI_BASE_URL?: string | undefined;
762
+ WATSONX_AI_APIKEY?: string | undefined;
763
+ WATSONX_AI_PROJECT_ID?: string | undefined;
764
+ WATSONX_AI_BEARER_TOKEN?: string | undefined;
765
+ AZURE_CLIENT_SECRET?: string | undefined;
766
+ AZURE_CLIENT_ID?: string | undefined;
767
+ AZURE_TENANT_ID?: string | undefined;
768
+ AZURE_AUTHORITY_HOST?: string | undefined;
769
+ AZURE_TOKEN_SCOPE?: string | undefined;
770
+ AZURE_DEPLOYMENT_NAME?: string | undefined;
771
+ AZURE_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
772
+ AZURE_OPENAI_DEPLOYMENT_NAME?: string | undefined;
773
+ AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
752
774
  AZURE_OPENAI_API_BASE_URL?: string | undefined;
753
775
  AZURE_OPENAI_API_HOST?: string | undefined;
754
776
  AZURE_OPENAI_API_KEY?: string | undefined;
755
777
  AZURE_API_BASE_URL?: string | undefined;
756
778
  AZURE_API_HOST?: string | undefined;
757
779
  AZURE_API_KEY?: string | undefined;
758
- AZURE_DEPLOYMENT_NAME?: string | undefined;
759
- AZURE_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
760
780
  AZURE_OPENAI_BASE_URL?: string | undefined;
761
- AZURE_OPENAI_DEPLOYMENT_NAME?: string | undefined;
762
- AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
763
- AZURE_CLIENT_SECRET?: string | undefined;
764
- AZURE_CLIENT_ID?: string | undefined;
765
- AZURE_TENANT_ID?: string | undefined;
766
- AZURE_AUTHORITY_HOST?: string | undefined;
767
- AZURE_TOKEN_SCOPE?: string | undefined;
768
781
  BAM_API_HOST?: string | undefined;
769
782
  BAM_API_KEY?: string | undefined;
770
783
  CLOUDFLARE_ACCOUNT_ID?: string | undefined;
771
784
  CLOUDFLARE_API_KEY?: string | undefined;
772
- PROMPTFOO_REMOTE_GENERATION_URL?: string | undefined;
773
785
  COHERE_API_KEY?: string | undefined;
774
- FAL_KEY?: string | undefined;
775
786
  GOOGLE_API_HOST?: string | undefined;
776
787
  GOOGLE_API_KEY?: string | undefined;
777
- GROQ_API_KEY?: string | undefined;
778
- LOCALAI_BASE_URL?: string | undefined;
779
788
  MISTRAL_API_BASE_URL?: string | undefined;
780
789
  MISTRAL_API_HOST?: string | undefined;
781
790
  MISTRAL_API_KEY?: string | undefined;
@@ -793,9 +802,6 @@ export declare const evalsTable: import("drizzle-orm/sqlite-core").SQLiteTableWi
793
802
  VERTEX_PROJECT_ID?: string | undefined;
794
803
  VERTEX_PUBLISHER?: string | undefined;
795
804
  VERTEX_REGION?: string | undefined;
796
- WATSONX_AI_APIKEY?: string | undefined;
797
- WATSONX_AI_PROJECT_ID?: string | undefined;
798
- WATSONX_AI_BEARER_TOKEN?: string | undefined;
799
805
  } | undefined;
800
806
  } | {
801
807
  callApi: import("../types").CallApiFunction;
@@ -807,7 +813,10 @@ export declare const evalsTable: import("drizzle-orm/sqlite-core").SQLiteTableWi
807
813
  callEmbeddingApi?: ((args_0: string, ...args: unknown[]) => Promise<import("../types").ProviderEmbeddingResponse>) | undefined;
808
814
  callClassificationApi?: ((args_0: string, ...args: unknown[]) => Promise<import("../types").ProviderClassificationResponse>) | undefined;
809
815
  } | undefined;
810
- metadata?: Record<string, any> | undefined;
816
+ metadata?: (Record<string, any> & {
817
+ pluginConfig?: import("../types").PluginConfig | undefined;
818
+ strategyConfig?: import("../types").RedteamObjectConfig | undefined;
819
+ }) | undefined;
811
820
  description?: string | undefined;
812
821
  providerOutput?: string | {} | undefined;
813
822
  assert?: ({
@@ -883,37 +892,40 @@ export declare const evalsTable: import("drizzle-orm/sqlite-core").SQLiteTableWi
883
892
  transform?: string | undefined;
884
893
  delay?: number | undefined;
885
894
  env?: {
895
+ PROMPTFOO_REMOTE_GENERATION_URL?: string | undefined;
886
896
  AI21_API_BASE_URL?: string | undefined;
887
897
  AI21_API_KEY?: string | undefined;
888
898
  ANTHROPIC_API_KEY?: string | undefined;
889
899
  AWS_BEDROCK_REGION?: string | undefined;
900
+ FAL_KEY?: string | undefined;
901
+ GROQ_API_KEY?: string | undefined;
902
+ LOCALAI_BASE_URL?: string | undefined;
903
+ WATSONX_AI_APIKEY?: string | undefined;
904
+ WATSONX_AI_PROJECT_ID?: string | undefined;
905
+ WATSONX_AI_BEARER_TOKEN?: string | undefined;
906
+ AZURE_CLIENT_SECRET?: string | undefined;
907
+ AZURE_CLIENT_ID?: string | undefined;
908
+ AZURE_TENANT_ID?: string | undefined;
909
+ AZURE_AUTHORITY_HOST?: string | undefined;
910
+ AZURE_TOKEN_SCOPE?: string | undefined;
911
+ AZURE_DEPLOYMENT_NAME?: string | undefined;
912
+ AZURE_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
913
+ AZURE_OPENAI_DEPLOYMENT_NAME?: string | undefined;
914
+ AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
890
915
  AZURE_OPENAI_API_BASE_URL?: string | undefined;
891
916
  AZURE_OPENAI_API_HOST?: string | undefined;
892
917
  AZURE_OPENAI_API_KEY?: string | undefined;
893
918
  AZURE_API_BASE_URL?: string | undefined;
894
919
  AZURE_API_HOST?: string | undefined;
895
920
  AZURE_API_KEY?: string | undefined;
896
- AZURE_DEPLOYMENT_NAME?: string | undefined;
897
- AZURE_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
898
921
  AZURE_OPENAI_BASE_URL?: string | undefined;
899
- AZURE_OPENAI_DEPLOYMENT_NAME?: string | undefined;
900
- AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
901
- AZURE_CLIENT_SECRET?: string | undefined;
902
- AZURE_CLIENT_ID?: string | undefined;
903
- AZURE_TENANT_ID?: string | undefined;
904
- AZURE_AUTHORITY_HOST?: string | undefined;
905
- AZURE_TOKEN_SCOPE?: string | undefined;
906
922
  BAM_API_HOST?: string | undefined;
907
923
  BAM_API_KEY?: string | undefined;
908
924
  CLOUDFLARE_ACCOUNT_ID?: string | undefined;
909
925
  CLOUDFLARE_API_KEY?: string | undefined;
910
- PROMPTFOO_REMOTE_GENERATION_URL?: string | undefined;
911
926
  COHERE_API_KEY?: string | undefined;
912
- FAL_KEY?: string | undefined;
913
927
  GOOGLE_API_HOST?: string | undefined;
914
928
  GOOGLE_API_KEY?: string | undefined;
915
- GROQ_API_KEY?: string | undefined;
916
- LOCALAI_BASE_URL?: string | undefined;
917
929
  MISTRAL_API_BASE_URL?: string | undefined;
918
930
  MISTRAL_API_HOST?: string | undefined;
919
931
  MISTRAL_API_KEY?: string | undefined;
@@ -931,9 +943,6 @@ export declare const evalsTable: import("drizzle-orm/sqlite-core").SQLiteTableWi
931
943
  VERTEX_PROJECT_ID?: string | undefined;
932
944
  VERTEX_PUBLISHER?: string | undefined;
933
945
  VERTEX_REGION?: string | undefined;
934
- WATSONX_AI_APIKEY?: string | undefined;
935
- WATSONX_AI_PROJECT_ID?: string | undefined;
936
- WATSONX_AI_BEARER_TOKEN?: string | undefined;
937
946
  } | undefined;
938
947
  } | {
939
948
  callApi: import("../types").CallApiFunction;
@@ -945,7 +954,10 @@ export declare const evalsTable: import("drizzle-orm/sqlite-core").SQLiteTableWi
945
954
  callEmbeddingApi?: ((args_0: string, ...args: unknown[]) => Promise<import("../types").ProviderEmbeddingResponse>) | undefined;
946
955
  callClassificationApi?: ((args_0: string, ...args: unknown[]) => Promise<import("../types").ProviderClassificationResponse>) | undefined;
947
956
  } | undefined;
948
- metadata?: Record<string, any> | undefined;
957
+ metadata?: (Record<string, any> & {
958
+ pluginConfig?: import("../types").PluginConfig | undefined;
959
+ strategyConfig?: import("../types").RedteamObjectConfig | undefined;
960
+ }) | undefined;
949
961
  providerOutput?: string | {} | undefined;
950
962
  assert?: ({
951
963
  type: import("../types").AssertionType;
@@ -1069,37 +1081,40 @@ export declare const evalsTable: import("drizzle-orm/sqlite-core").SQLiteTableWi
1069
1081
  transform?: string | undefined;
1070
1082
  delay?: number | undefined;
1071
1083
  env?: {
1084
+ PROMPTFOO_REMOTE_GENERATION_URL?: string | undefined;
1072
1085
  AI21_API_BASE_URL?: string | undefined;
1073
1086
  AI21_API_KEY?: string | undefined;
1074
1087
  ANTHROPIC_API_KEY?: string | undefined;
1075
1088
  AWS_BEDROCK_REGION?: string | undefined;
1089
+ FAL_KEY?: string | undefined;
1090
+ GROQ_API_KEY?: string | undefined;
1091
+ LOCALAI_BASE_URL?: string | undefined;
1092
+ WATSONX_AI_APIKEY?: string | undefined;
1093
+ WATSONX_AI_PROJECT_ID?: string | undefined;
1094
+ WATSONX_AI_BEARER_TOKEN?: string | undefined;
1095
+ AZURE_CLIENT_SECRET?: string | undefined;
1096
+ AZURE_CLIENT_ID?: string | undefined;
1097
+ AZURE_TENANT_ID?: string | undefined;
1098
+ AZURE_AUTHORITY_HOST?: string | undefined;
1099
+ AZURE_TOKEN_SCOPE?: string | undefined;
1100
+ AZURE_DEPLOYMENT_NAME?: string | undefined;
1101
+ AZURE_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
1102
+ AZURE_OPENAI_DEPLOYMENT_NAME?: string | undefined;
1103
+ AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
1076
1104
  AZURE_OPENAI_API_BASE_URL?: string | undefined;
1077
1105
  AZURE_OPENAI_API_HOST?: string | undefined;
1078
1106
  AZURE_OPENAI_API_KEY?: string | undefined;
1079
1107
  AZURE_API_BASE_URL?: string | undefined;
1080
1108
  AZURE_API_HOST?: string | undefined;
1081
1109
  AZURE_API_KEY?: string | undefined;
1082
- AZURE_DEPLOYMENT_NAME?: string | undefined;
1083
- AZURE_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
1084
1110
  AZURE_OPENAI_BASE_URL?: string | undefined;
1085
- AZURE_OPENAI_DEPLOYMENT_NAME?: string | undefined;
1086
- AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
1087
- AZURE_CLIENT_SECRET?: string | undefined;
1088
- AZURE_CLIENT_ID?: string | undefined;
1089
- AZURE_TENANT_ID?: string | undefined;
1090
- AZURE_AUTHORITY_HOST?: string | undefined;
1091
- AZURE_TOKEN_SCOPE?: string | undefined;
1092
1111
  BAM_API_HOST?: string | undefined;
1093
1112
  BAM_API_KEY?: string | undefined;
1094
1113
  CLOUDFLARE_ACCOUNT_ID?: string | undefined;
1095
1114
  CLOUDFLARE_API_KEY?: string | undefined;
1096
- PROMPTFOO_REMOTE_GENERATION_URL?: string | undefined;
1097
1115
  COHERE_API_KEY?: string | undefined;
1098
- FAL_KEY?: string | undefined;
1099
1116
  GOOGLE_API_HOST?: string | undefined;
1100
1117
  GOOGLE_API_KEY?: string | undefined;
1101
- GROQ_API_KEY?: string | undefined;
1102
- LOCALAI_BASE_URL?: string | undefined;
1103
1118
  MISTRAL_API_BASE_URL?: string | undefined;
1104
1119
  MISTRAL_API_HOST?: string | undefined;
1105
1120
  MISTRAL_API_KEY?: string | undefined;
@@ -1117,9 +1132,6 @@ export declare const evalsTable: import("drizzle-orm/sqlite-core").SQLiteTableWi
1117
1132
  VERTEX_PROJECT_ID?: string | undefined;
1118
1133
  VERTEX_PUBLISHER?: string | undefined;
1119
1134
  VERTEX_REGION?: string | undefined;
1120
- WATSONX_AI_APIKEY?: string | undefined;
1121
- WATSONX_AI_PROJECT_ID?: string | undefined;
1122
- WATSONX_AI_BEARER_TOKEN?: string | undefined;
1123
1135
  } | undefined;
1124
1136
  } | (((args_0: string, args_1: {
1125
1137
  vars: Record<string, string | {}>;
@@ -1151,37 +1163,40 @@ export declare const evalsTable: import("drizzle-orm/sqlite-core").SQLiteTableWi
1151
1163
  transform?: string | undefined;
1152
1164
  delay?: number | undefined;
1153
1165
  env?: {
1166
+ PROMPTFOO_REMOTE_GENERATION_URL?: string | undefined;
1154
1167
  AI21_API_BASE_URL?: string | undefined;
1155
1168
  AI21_API_KEY?: string | undefined;
1156
1169
  ANTHROPIC_API_KEY?: string | undefined;
1157
1170
  AWS_BEDROCK_REGION?: string | undefined;
1171
+ FAL_KEY?: string | undefined;
1172
+ GROQ_API_KEY?: string | undefined;
1173
+ LOCALAI_BASE_URL?: string | undefined;
1174
+ WATSONX_AI_APIKEY?: string | undefined;
1175
+ WATSONX_AI_PROJECT_ID?: string | undefined;
1176
+ WATSONX_AI_BEARER_TOKEN?: string | undefined;
1177
+ AZURE_CLIENT_SECRET?: string | undefined;
1178
+ AZURE_CLIENT_ID?: string | undefined;
1179
+ AZURE_TENANT_ID?: string | undefined;
1180
+ AZURE_AUTHORITY_HOST?: string | undefined;
1181
+ AZURE_TOKEN_SCOPE?: string | undefined;
1182
+ AZURE_DEPLOYMENT_NAME?: string | undefined;
1183
+ AZURE_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
1184
+ AZURE_OPENAI_DEPLOYMENT_NAME?: string | undefined;
1185
+ AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
1158
1186
  AZURE_OPENAI_API_BASE_URL?: string | undefined;
1159
1187
  AZURE_OPENAI_API_HOST?: string | undefined;
1160
1188
  AZURE_OPENAI_API_KEY?: string | undefined;
1161
1189
  AZURE_API_BASE_URL?: string | undefined;
1162
1190
  AZURE_API_HOST?: string | undefined;
1163
1191
  AZURE_API_KEY?: string | undefined;
1164
- AZURE_DEPLOYMENT_NAME?: string | undefined;
1165
- AZURE_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
1166
1192
  AZURE_OPENAI_BASE_URL?: string | undefined;
1167
- AZURE_OPENAI_DEPLOYMENT_NAME?: string | undefined;
1168
- AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
1169
- AZURE_CLIENT_SECRET?: string | undefined;
1170
- AZURE_CLIENT_ID?: string | undefined;
1171
- AZURE_TENANT_ID?: string | undefined;
1172
- AZURE_AUTHORITY_HOST?: string | undefined;
1173
- AZURE_TOKEN_SCOPE?: string | undefined;
1174
1193
  BAM_API_HOST?: string | undefined;
1175
1194
  BAM_API_KEY?: string | undefined;
1176
1195
  CLOUDFLARE_ACCOUNT_ID?: string | undefined;
1177
1196
  CLOUDFLARE_API_KEY?: string | undefined;
1178
- PROMPTFOO_REMOTE_GENERATION_URL?: string | undefined;
1179
1197
  COHERE_API_KEY?: string | undefined;
1180
- FAL_KEY?: string | undefined;
1181
1198
  GOOGLE_API_HOST?: string | undefined;
1182
1199
  GOOGLE_API_KEY?: string | undefined;
1183
- GROQ_API_KEY?: string | undefined;
1184
- LOCALAI_BASE_URL?: string | undefined;
1185
1200
  MISTRAL_API_BASE_URL?: string | undefined;
1186
1201
  MISTRAL_API_HOST?: string | undefined;
1187
1202
  MISTRAL_API_KEY?: string | undefined;
@@ -1199,9 +1214,6 @@ export declare const evalsTable: import("drizzle-orm/sqlite-core").SQLiteTableWi
1199
1214
  VERTEX_PROJECT_ID?: string | undefined;
1200
1215
  VERTEX_PUBLISHER?: string | undefined;
1201
1216
  VERTEX_REGION?: string | undefined;
1202
- WATSONX_AI_APIKEY?: string | undefined;
1203
- WATSONX_AI_PROJECT_ID?: string | undefined;
1204
- WATSONX_AI_BEARER_TOKEN?: string | undefined;
1205
1217
  } | undefined;
1206
1218
  }>)[] | undefined;
1207
1219
  }>;
@@ -1238,6 +1250,7 @@ export declare const evalsTable: import("drizzle-orm/sqlite-core").SQLiteTableWi
1238
1250
  completion?: number | undefined;
1239
1251
  prompt?: number | undefined;
1240
1252
  total?: number | undefined;
1253
+ numRequests?: number | undefined;
1241
1254
  };
1242
1255
  score: number;
1243
1256
  testPassCount: number;
@@ -1409,37 +1422,40 @@ export declare const evalResultsTable: import("drizzle-orm/sqlite-core").SQLiteT
1409
1422
  transform?: string | undefined;
1410
1423
  delay?: number | undefined;
1411
1424
  env?: {
1425
+ PROMPTFOO_REMOTE_GENERATION_URL?: string | undefined;
1412
1426
  AI21_API_BASE_URL?: string | undefined;
1413
1427
  AI21_API_KEY?: string | undefined;
1414
1428
  ANTHROPIC_API_KEY?: string | undefined;
1415
1429
  AWS_BEDROCK_REGION?: string | undefined;
1430
+ FAL_KEY?: string | undefined;
1431
+ GROQ_API_KEY?: string | undefined;
1432
+ LOCALAI_BASE_URL?: string | undefined;
1433
+ WATSONX_AI_APIKEY?: string | undefined;
1434
+ WATSONX_AI_PROJECT_ID?: string | undefined;
1435
+ WATSONX_AI_BEARER_TOKEN?: string | undefined;
1436
+ AZURE_CLIENT_SECRET?: string | undefined;
1437
+ AZURE_CLIENT_ID?: string | undefined;
1438
+ AZURE_TENANT_ID?: string | undefined;
1439
+ AZURE_AUTHORITY_HOST?: string | undefined;
1440
+ AZURE_TOKEN_SCOPE?: string | undefined;
1441
+ AZURE_DEPLOYMENT_NAME?: string | undefined;
1442
+ AZURE_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
1443
+ AZURE_OPENAI_DEPLOYMENT_NAME?: string | undefined;
1444
+ AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
1416
1445
  AZURE_OPENAI_API_BASE_URL?: string | undefined;
1417
1446
  AZURE_OPENAI_API_HOST?: string | undefined;
1418
1447
  AZURE_OPENAI_API_KEY?: string | undefined;
1419
1448
  AZURE_API_BASE_URL?: string | undefined;
1420
1449
  AZURE_API_HOST?: string | undefined;
1421
1450
  AZURE_API_KEY?: string | undefined;
1422
- AZURE_DEPLOYMENT_NAME?: string | undefined;
1423
- AZURE_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
1424
1451
  AZURE_OPENAI_BASE_URL?: string | undefined;
1425
- AZURE_OPENAI_DEPLOYMENT_NAME?: string | undefined;
1426
- AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
1427
- AZURE_CLIENT_SECRET?: string | undefined;
1428
- AZURE_CLIENT_ID?: string | undefined;
1429
- AZURE_TENANT_ID?: string | undefined;
1430
- AZURE_AUTHORITY_HOST?: string | undefined;
1431
- AZURE_TOKEN_SCOPE?: string | undefined;
1432
1452
  BAM_API_HOST?: string | undefined;
1433
1453
  BAM_API_KEY?: string | undefined;
1434
1454
  CLOUDFLARE_ACCOUNT_ID?: string | undefined;
1435
1455
  CLOUDFLARE_API_KEY?: string | undefined;
1436
- PROMPTFOO_REMOTE_GENERATION_URL?: string | undefined;
1437
1456
  COHERE_API_KEY?: string | undefined;
1438
- FAL_KEY?: string | undefined;
1439
1457
  GOOGLE_API_HOST?: string | undefined;
1440
1458
  GOOGLE_API_KEY?: string | undefined;
1441
- GROQ_API_KEY?: string | undefined;
1442
- LOCALAI_BASE_URL?: string | undefined;
1443
1459
  MISTRAL_API_BASE_URL?: string | undefined;
1444
1460
  MISTRAL_API_HOST?: string | undefined;
1445
1461
  MISTRAL_API_KEY?: string | undefined;
@@ -1457,9 +1473,6 @@ export declare const evalResultsTable: import("drizzle-orm/sqlite-core").SQLiteT
1457
1473
  VERTEX_PROJECT_ID?: string | undefined;
1458
1474
  VERTEX_PUBLISHER?: string | undefined;
1459
1475
  VERTEX_REGION?: string | undefined;
1460
- WATSONX_AI_APIKEY?: string | undefined;
1461
- WATSONX_AI_PROJECT_ID?: string | undefined;
1462
- WATSONX_AI_BEARER_TOKEN?: string | undefined;
1463
1476
  } | undefined;
1464
1477
  } | {
1465
1478
  callApi: import("../types").CallApiFunction;
@@ -1471,7 +1484,10 @@ export declare const evalResultsTable: import("drizzle-orm/sqlite-core").SQLiteT
1471
1484
  callEmbeddingApi?: ((args_0: string, ...args: unknown[]) => Promise<import("../types").ProviderEmbeddingResponse>) | undefined;
1472
1485
  callClassificationApi?: ((args_0: string, ...args: unknown[]) => Promise<import("../types").ProviderClassificationResponse>) | undefined;
1473
1486
  } | undefined;
1474
- metadata?: Record<string, any> | undefined;
1487
+ metadata?: (Record<string, any> & {
1488
+ pluginConfig?: import("../types").PluginConfig | undefined;
1489
+ strategyConfig?: import("../types").RedteamObjectConfig | undefined;
1490
+ }) | undefined;
1475
1491
  description?: string | undefined;
1476
1492
  providerOutput?: string | {} | undefined;
1477
1493
  assert?: ({
@@ -1863,37 +1879,40 @@ export declare const datasetsTable: import("drizzle-orm/sqlite-core").SQLiteTabl
1863
1879
  transform?: string | undefined;
1864
1880
  delay?: number | undefined;
1865
1881
  env?: {
1882
+ PROMPTFOO_REMOTE_GENERATION_URL?: string | undefined;
1866
1883
  AI21_API_BASE_URL?: string | undefined;
1867
1884
  AI21_API_KEY?: string | undefined;
1868
1885
  ANTHROPIC_API_KEY?: string | undefined;
1869
1886
  AWS_BEDROCK_REGION?: string | undefined;
1887
+ FAL_KEY?: string | undefined;
1888
+ GROQ_API_KEY?: string | undefined;
1889
+ LOCALAI_BASE_URL?: string | undefined;
1890
+ WATSONX_AI_APIKEY?: string | undefined;
1891
+ WATSONX_AI_PROJECT_ID?: string | undefined;
1892
+ WATSONX_AI_BEARER_TOKEN?: string | undefined;
1893
+ AZURE_CLIENT_SECRET?: string | undefined;
1894
+ AZURE_CLIENT_ID?: string | undefined;
1895
+ AZURE_TENANT_ID?: string | undefined;
1896
+ AZURE_AUTHORITY_HOST?: string | undefined;
1897
+ AZURE_TOKEN_SCOPE?: string | undefined;
1898
+ AZURE_DEPLOYMENT_NAME?: string | undefined;
1899
+ AZURE_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
1900
+ AZURE_OPENAI_DEPLOYMENT_NAME?: string | undefined;
1901
+ AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
1870
1902
  AZURE_OPENAI_API_BASE_URL?: string | undefined;
1871
1903
  AZURE_OPENAI_API_HOST?: string | undefined;
1872
1904
  AZURE_OPENAI_API_KEY?: string | undefined;
1873
1905
  AZURE_API_BASE_URL?: string | undefined;
1874
1906
  AZURE_API_HOST?: string | undefined;
1875
1907
  AZURE_API_KEY?: string | undefined;
1876
- AZURE_DEPLOYMENT_NAME?: string | undefined;
1877
- AZURE_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
1878
1908
  AZURE_OPENAI_BASE_URL?: string | undefined;
1879
- AZURE_OPENAI_DEPLOYMENT_NAME?: string | undefined;
1880
- AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
1881
- AZURE_CLIENT_SECRET?: string | undefined;
1882
- AZURE_CLIENT_ID?: string | undefined;
1883
- AZURE_TENANT_ID?: string | undefined;
1884
- AZURE_AUTHORITY_HOST?: string | undefined;
1885
- AZURE_TOKEN_SCOPE?: string | undefined;
1886
1909
  BAM_API_HOST?: string | undefined;
1887
1910
  BAM_API_KEY?: string | undefined;
1888
1911
  CLOUDFLARE_ACCOUNT_ID?: string | undefined;
1889
1912
  CLOUDFLARE_API_KEY?: string | undefined;
1890
- PROMPTFOO_REMOTE_GENERATION_URL?: string | undefined;
1891
1913
  COHERE_API_KEY?: string | undefined;
1892
- FAL_KEY?: string | undefined;
1893
1914
  GOOGLE_API_HOST?: string | undefined;
1894
1915
  GOOGLE_API_KEY?: string | undefined;
1895
- GROQ_API_KEY?: string | undefined;
1896
- LOCALAI_BASE_URL?: string | undefined;
1897
1916
  MISTRAL_API_BASE_URL?: string | undefined;
1898
1917
  MISTRAL_API_HOST?: string | undefined;
1899
1918
  MISTRAL_API_KEY?: string | undefined;
@@ -1911,9 +1930,6 @@ export declare const datasetsTable: import("drizzle-orm/sqlite-core").SQLiteTabl
1911
1930
  VERTEX_PROJECT_ID?: string | undefined;
1912
1931
  VERTEX_PUBLISHER?: string | undefined;
1913
1932
  VERTEX_REGION?: string | undefined;
1914
- WATSONX_AI_APIKEY?: string | undefined;
1915
- WATSONX_AI_PROJECT_ID?: string | undefined;
1916
- WATSONX_AI_BEARER_TOKEN?: string | undefined;
1917
1933
  } | undefined;
1918
1934
  } | {
1919
1935
  callApi: import("../types").CallApiFunction;
@@ -1925,7 +1941,10 @@ export declare const datasetsTable: import("drizzle-orm/sqlite-core").SQLiteTabl
1925
1941
  callEmbeddingApi?: ((args_0: string, ...args: unknown[]) => Promise<import("../types").ProviderEmbeddingResponse>) | undefined;
1926
1942
  callClassificationApi?: ((args_0: string, ...args: unknown[]) => Promise<import("../types").ProviderClassificationResponse>) | undefined;
1927
1943
  } | undefined;
1928
- metadata?: Record<string, any> | undefined;
1944
+ metadata?: (Record<string, any> & {
1945
+ pluginConfig?: import("../types").PluginConfig | undefined;
1946
+ strategyConfig?: import("../types").RedteamObjectConfig | undefined;
1947
+ }) | undefined;
1929
1948
  description?: string | undefined;
1930
1949
  providerOutput?: string | {} | undefined;
1931
1950
  assert?: ({