promptfoo 0.97.0 → 0.99.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (256) hide show
  1. package/README.md +1 -1
  2. package/dist/package.json +19 -17
  3. package/dist/src/app/assets/index-CJGQrbZb.js +787 -0
  4. package/dist/src/app/assets/{index-YwjS5Vq2.css → index-DatcZSwS.css} +1 -1
  5. package/dist/src/app/assets/{index.es-Cc9N3uII.js → index.es-BCwhwRu5.js} +1 -1
  6. package/dist/src/app/assets/{sync-BoVzjiAe.js → sync-hJWHXawF.js} +1 -1
  7. package/dist/src/app/index.html +2 -2
  8. package/dist/src/assertions/index.d.ts.map +1 -1
  9. package/dist/src/assertions/index.js +9 -1
  10. package/dist/src/assertions/index.js.map +1 -1
  11. package/dist/src/cache.d.ts +1 -0
  12. package/dist/src/cache.d.ts.map +1 -1
  13. package/dist/src/cache.js +5 -0
  14. package/dist/src/cache.js.map +1 -1
  15. package/dist/src/commands/debug.d.ts +4 -0
  16. package/dist/src/commands/debug.d.ts.map +1 -0
  17. package/dist/src/commands/debug.js +84 -0
  18. package/dist/src/commands/debug.js.map +1 -0
  19. package/dist/src/commands/view.d.ts.map +1 -1
  20. package/dist/src/commands/view.js +6 -4
  21. package/dist/src/commands/view.js.map +1 -1
  22. package/dist/src/constants.d.ts +1 -0
  23. package/dist/src/constants.d.ts.map +1 -1
  24. package/dist/src/constants.js +2 -1
  25. package/dist/src/constants.js.map +1 -1
  26. package/dist/src/database/tables.d.ts +176 -176
  27. package/dist/src/envars.d.ts +1 -1
  28. package/dist/src/envars.d.ts.map +1 -1
  29. package/dist/src/evaluator.d.ts.map +1 -1
  30. package/dist/src/evaluator.js +7 -3
  31. package/dist/src/evaluator.js.map +1 -1
  32. package/dist/src/evaluatorHelpers.d.ts +1 -0
  33. package/dist/src/evaluatorHelpers.d.ts.map +1 -1
  34. package/dist/src/evaluatorHelpers.js +19 -0
  35. package/dist/src/evaluatorHelpers.js.map +1 -1
  36. package/dist/src/logger.js +10 -10
  37. package/dist/src/logger.js.map +1 -1
  38. package/dist/src/main.js +17 -1
  39. package/dist/src/main.js.map +1 -1
  40. package/dist/src/models/eval.d.ts +1 -0
  41. package/dist/src/models/eval.d.ts.map +1 -1
  42. package/dist/src/models/eval.js +2 -0
  43. package/dist/src/models/eval.js.map +1 -1
  44. package/dist/src/prompts/processors/yaml.d.ts.map +1 -1
  45. package/dist/src/prompts/processors/yaml.js +7 -2
  46. package/dist/src/prompts/processors/yaml.js.map +1 -1
  47. package/dist/src/providers/adaline.gateway.d.ts +2 -1
  48. package/dist/src/providers/adaline.gateway.d.ts.map +1 -1
  49. package/dist/src/providers/ai21.d.ts +2 -1
  50. package/dist/src/providers/ai21.d.ts.map +1 -1
  51. package/dist/src/providers/ai21.js.map +1 -1
  52. package/dist/src/providers/anthropic.d.ts +2 -1
  53. package/dist/src/providers/anthropic.d.ts.map +1 -1
  54. package/dist/src/providers/anthropic.js.map +1 -1
  55. package/dist/src/providers/azure.d.ts +2 -1
  56. package/dist/src/providers/azure.d.ts.map +1 -1
  57. package/dist/src/providers/bam.d.ts +2 -1
  58. package/dist/src/providers/bam.d.ts.map +1 -1
  59. package/dist/src/providers/bam.js.map +1 -1
  60. package/dist/src/providers/bedrock.d.ts +2 -1
  61. package/dist/src/providers/bedrock.d.ts.map +1 -1
  62. package/dist/src/providers/cloudflare-ai.d.ts +2 -1
  63. package/dist/src/providers/cloudflare-ai.d.ts.map +1 -1
  64. package/dist/src/providers/cohere.d.ts +2 -1
  65. package/dist/src/providers/cohere.d.ts.map +1 -1
  66. package/dist/src/providers/defaults.d.ts +2 -1
  67. package/dist/src/providers/defaults.d.ts.map +1 -1
  68. package/dist/src/providers/defaults.js.map +1 -1
  69. package/dist/src/providers/fal.d.ts +2 -1
  70. package/dist/src/providers/fal.d.ts.map +1 -1
  71. package/dist/src/providers/fal.js.map +1 -1
  72. package/dist/src/providers/groq.d.ts +2 -1
  73. package/dist/src/providers/groq.d.ts.map +1 -1
  74. package/dist/src/providers/http.d.ts +5 -1
  75. package/dist/src/providers/http.d.ts.map +1 -1
  76. package/dist/src/providers/http.js +67 -10
  77. package/dist/src/providers/http.js.map +1 -1
  78. package/dist/src/providers/localai.d.ts +2 -1
  79. package/dist/src/providers/localai.d.ts.map +1 -1
  80. package/dist/src/providers/localai.js.map +1 -1
  81. package/dist/src/providers/mistral.d.ts +2 -1
  82. package/dist/src/providers/mistral.d.ts.map +1 -1
  83. package/dist/src/providers/openai.d.ts +2 -1
  84. package/dist/src/providers/openai.d.ts.map +1 -1
  85. package/dist/src/providers/openai.js +1 -1
  86. package/dist/src/providers/openai.js.map +1 -1
  87. package/dist/src/providers/palm.d.ts +2 -1
  88. package/dist/src/providers/palm.d.ts.map +1 -1
  89. package/dist/src/providers/palm.js.map +1 -1
  90. package/dist/src/providers/promptfoo.d.ts +2 -1
  91. package/dist/src/providers/promptfoo.d.ts.map +1 -1
  92. package/dist/src/providers/replicate.d.ts +2 -1
  93. package/dist/src/providers/replicate.d.ts.map +1 -1
  94. package/dist/src/providers/sequence.d.ts +11 -0
  95. package/dist/src/providers/sequence.d.ts.map +1 -0
  96. package/dist/src/providers/sequence.js +66 -0
  97. package/dist/src/providers/sequence.js.map +1 -0
  98. package/dist/src/providers/togetherai.d.ts +2 -1
  99. package/dist/src/providers/togetherai.d.ts.map +1 -1
  100. package/dist/src/providers/togetherai.js.map +1 -1
  101. package/dist/src/providers/vertex.d.ts +2 -1
  102. package/dist/src/providers/vertex.d.ts.map +1 -1
  103. package/dist/src/providers/watsonx.d.ts +2 -1
  104. package/dist/src/providers/watsonx.d.ts.map +1 -1
  105. package/dist/src/providers/watsonx.js.map +1 -1
  106. package/dist/src/providers/xai.d.ts +2 -1
  107. package/dist/src/providers/xai.d.ts.map +1 -1
  108. package/dist/src/providers/xai.js.map +1 -1
  109. package/dist/src/providers.d.ts +2 -1
  110. package/dist/src/providers.d.ts.map +1 -1
  111. package/dist/src/providers.js +4 -0
  112. package/dist/src/providers.js.map +1 -1
  113. package/dist/src/redteam/commands/generate.d.ts.map +1 -1
  114. package/dist/src/redteam/commands/generate.js +25 -4
  115. package/dist/src/redteam/commands/generate.js.map +1 -1
  116. package/dist/src/redteam/commands/init.d.ts.map +1 -1
  117. package/dist/src/redteam/commands/init.js +79 -64
  118. package/dist/src/redteam/commands/init.js.map +1 -1
  119. package/dist/src/redteam/commands/report.d.ts.map +1 -1
  120. package/dist/src/redteam/commands/report.js +11 -3
  121. package/dist/src/redteam/commands/report.js.map +1 -1
  122. package/dist/src/redteam/commands/setup.d.ts +3 -0
  123. package/dist/src/redteam/commands/setup.d.ts.map +1 -0
  124. package/dist/src/redteam/commands/setup.js +33 -0
  125. package/dist/src/redteam/commands/setup.js.map +1 -0
  126. package/dist/src/redteam/constants.d.ts +4 -4
  127. package/dist/src/redteam/constants.d.ts.map +1 -1
  128. package/dist/src/redteam/constants.js +12 -2
  129. package/dist/src/redteam/constants.js.map +1 -1
  130. package/dist/src/redteam/index.d.ts.map +1 -1
  131. package/dist/src/redteam/index.js +95 -37
  132. package/dist/src/redteam/index.js.map +1 -1
  133. package/dist/src/redteam/plugins/base.d.ts.map +1 -1
  134. package/dist/src/redteam/plugins/base.js +2 -0
  135. package/dist/src/redteam/plugins/base.js.map +1 -1
  136. package/dist/src/redteam/plugins/competitors.d.ts.map +1 -1
  137. package/dist/src/redteam/plugins/competitors.js +23 -1
  138. package/dist/src/redteam/plugins/competitors.js.map +1 -1
  139. package/dist/src/redteam/plugins/debugAccess.d.ts.map +1 -1
  140. package/dist/src/redteam/plugins/debugAccess.js +18 -0
  141. package/dist/src/redteam/plugins/debugAccess.js.map +1 -1
  142. package/dist/src/redteam/plugins/harmful/constants.d.ts.map +1 -1
  143. package/dist/src/redteam/plugins/harmful/constants.js +32 -8
  144. package/dist/src/redteam/plugins/harmful/constants.js.map +1 -1
  145. package/dist/src/redteam/plugins/harmful/unaligned.js +1 -1
  146. package/dist/src/redteam/plugins/harmful/unaligned.js.map +1 -1
  147. package/dist/src/redteam/plugins/intent.d.ts.map +1 -1
  148. package/dist/src/redteam/plugins/intent.js +6 -0
  149. package/dist/src/redteam/plugins/intent.js.map +1 -1
  150. package/dist/src/redteam/providers/crescendo/index.d.ts +3 -0
  151. package/dist/src/redteam/providers/crescendo/index.d.ts.map +1 -1
  152. package/dist/src/redteam/providers/crescendo/index.js +66 -37
  153. package/dist/src/redteam/providers/crescendo/index.js.map +1 -1
  154. package/dist/src/redteam/providers/goat.d.ts.map +1 -1
  155. package/dist/src/redteam/providers/goat.js +56 -41
  156. package/dist/src/redteam/providers/goat.js.map +1 -1
  157. package/dist/src/redteam/providers/iterative.d.ts +1 -0
  158. package/dist/src/redteam/providers/iterative.d.ts.map +1 -1
  159. package/dist/src/redteam/providers/iterative.js +14 -9
  160. package/dist/src/redteam/providers/iterative.js.map +1 -1
  161. package/dist/src/redteam/providers/iterativeImage.d.ts +3 -1
  162. package/dist/src/redteam/providers/iterativeImage.d.ts.map +1 -1
  163. package/dist/src/redteam/providers/iterativeImage.js +14 -8
  164. package/dist/src/redteam/providers/iterativeImage.js.map +1 -1
  165. package/dist/src/redteam/providers/iterativeTree.d.ts.map +1 -1
  166. package/dist/src/redteam/providers/iterativeTree.js +26 -19
  167. package/dist/src/redteam/providers/iterativeTree.js.map +1 -1
  168. package/dist/src/redteam/providers/shared.d.ts +9 -6
  169. package/dist/src/redteam/providers/shared.d.ts.map +1 -1
  170. package/dist/src/redteam/providers/shared.js +9 -16
  171. package/dist/src/redteam/providers/shared.js.map +1 -1
  172. package/dist/src/redteam/strategies/citation.d.ts +3 -0
  173. package/dist/src/redteam/strategies/citation.d.ts.map +1 -0
  174. package/dist/src/redteam/strategies/citation.js +96 -0
  175. package/dist/src/redteam/strategies/citation.js.map +1 -0
  176. package/dist/src/redteam/strategies/index.d.ts +3 -2
  177. package/dist/src/redteam/strategies/index.d.ts.map +1 -1
  178. package/dist/src/redteam/strategies/index.js +68 -13
  179. package/dist/src/redteam/strategies/index.js.map +1 -1
  180. package/dist/src/redteam/strategies/singleTurnComposite.d.ts +3 -0
  181. package/dist/src/redteam/strategies/singleTurnComposite.d.ts.map +1 -0
  182. package/dist/src/redteam/strategies/singleTurnComposite.js +89 -0
  183. package/dist/src/redteam/strategies/singleTurnComposite.js.map +1 -0
  184. package/dist/src/redteam/types.d.ts +5 -1
  185. package/dist/src/redteam/types.d.ts.map +1 -1
  186. package/dist/src/server/index.js +4 -2
  187. package/dist/src/server/index.js.map +1 -1
  188. package/dist/src/server/routes/providers.js +1 -0
  189. package/dist/src/server/routes/providers.js.map +1 -1
  190. package/dist/src/server/server.d.ts +1 -6
  191. package/dist/src/server/server.d.ts.map +1 -1
  192. package/dist/src/server/server.js +13 -43
  193. package/dist/src/server/server.js.map +1 -1
  194. package/dist/src/testCases.js +1 -1
  195. package/dist/src/testCases.js.map +1 -1
  196. package/dist/src/types/env.d.ts +54 -0
  197. package/dist/src/types/env.d.ts.map +1 -0
  198. package/dist/src/types/env.js +3 -0
  199. package/dist/src/types/env.js.map +1 -0
  200. package/dist/src/types/index.d.ts +8395 -6838
  201. package/dist/src/types/index.d.ts.map +1 -1
  202. package/dist/src/types/index.js +2 -1
  203. package/dist/src/types/index.js.map +1 -1
  204. package/dist/src/types/providers.d.ts +3 -53
  205. package/dist/src/types/providers.d.ts.map +1 -1
  206. package/dist/src/types/providers.js.map +1 -1
  207. package/dist/src/util/config/load.js +1 -1
  208. package/dist/src/util/config/load.js.map +1 -1
  209. package/dist/src/util/index.d.ts +32 -32
  210. package/dist/src/util/server.d.ts +10 -0
  211. package/dist/src/util/server.d.ts.map +1 -0
  212. package/dist/src/util/server.js +74 -0
  213. package/dist/src/util/server.js.map +1 -0
  214. package/dist/src/util/transform.d.ts +1 -0
  215. package/dist/src/util/transform.d.ts.map +1 -1
  216. package/dist/src/util/transform.js.map +1 -1
  217. package/dist/src/validators/providers.d.ts +296 -296
  218. package/dist/src/validators/redteam.d.ts +130 -126
  219. package/dist/src/validators/redteam.d.ts.map +1 -1
  220. package/dist/src/validators/redteam.js +18 -2
  221. package/dist/src/validators/redteam.js.map +1 -1
  222. package/dist/test/cache.test.js +47 -7
  223. package/dist/test/cache.test.js.map +1 -1
  224. package/dist/test/commands/init.test.js +14 -0
  225. package/dist/test/commands/init.test.js.map +1 -1
  226. package/dist/test/evaluator.test.js +50 -0
  227. package/dist/test/evaluator.test.js.map +1 -1
  228. package/dist/test/factories/evalFactory.d.ts +144 -144
  229. package/dist/test/prompts/index.test.js +38 -25
  230. package/dist/test/prompts/index.test.js.map +1 -1
  231. package/dist/test/prompts/processors/yaml.test.js +50 -4
  232. package/dist/test/prompts/processors/yaml.test.js.map +1 -1
  233. package/dist/test/providers/http.test.js +0 -17
  234. package/dist/test/providers/http.test.js.map +1 -1
  235. package/dist/test/providers/index.test.js +1 -0
  236. package/dist/test/providers/index.test.js.map +1 -1
  237. package/dist/test/providers/togetherai.test.js.map +1 -1
  238. package/dist/test/redteam/commands/generate.test.js +58 -0
  239. package/dist/test/redteam/commands/generate.test.js.map +1 -1
  240. package/dist/test/redteam/index.test.js +2 -2
  241. package/dist/test/redteam/index.test.js.map +1 -1
  242. package/dist/test/redteam/plugins/base.test.js +8 -0
  243. package/dist/test/redteam/plugins/base.test.js.map +1 -1
  244. package/dist/test/redteam/providers/iterativeTree.test.js +11 -3
  245. package/dist/test/redteam/providers/iterativeTree.test.js.map +1 -1
  246. package/dist/test/testCases.test.js +21 -0
  247. package/dist/test/testCases.test.js.map +1 -1
  248. package/dist/test/types.test.js +17 -4
  249. package/dist/test/types.test.js.map +1 -1
  250. package/dist/test/util/server.test.d.ts +2 -0
  251. package/dist/test/util/server.test.d.ts.map +1 -0
  252. package/dist/test/util/server.test.js +114 -0
  253. package/dist/test/util/server.test.js.map +1 -0
  254. package/dist/tsconfig.tsbuildinfo +1 -1
  255. package/package.json +19 -17
  256. package/dist/src/app/assets/index-CbxDbiAR.js +0 -769
package/dist/test/factories/evalFactory.d.ts CHANGED
@@ -20,37 +20,40 @@ export default class EvalFactory {
20
20
  label?: string | undefined;
21
21
  })[];
22
22
  env?: {
23
+ PROMPTFOO_REMOTE_GENERATION_URL?: string | undefined;
23
24
  AI21_API_BASE_URL?: string | undefined;
24
25
  AI21_API_KEY?: string | undefined;
25
26
  ANTHROPIC_API_KEY?: string | undefined;
26
27
  AWS_BEDROCK_REGION?: string | undefined;
28
+ FAL_KEY?: string | undefined;
29
+ GROQ_API_KEY?: string | undefined;
30
+ LOCALAI_BASE_URL?: string | undefined;
31
+ WATSONX_AI_APIKEY?: string | undefined;
32
+ WATSONX_AI_PROJECT_ID?: string | undefined;
33
+ WATSONX_AI_BEARER_TOKEN?: string | undefined;
34
+ AZURE_CLIENT_SECRET?: string | undefined;
35
+ AZURE_CLIENT_ID?: string | undefined;
36
+ AZURE_TENANT_ID?: string | undefined;
37
+ AZURE_AUTHORITY_HOST?: string | undefined;
38
+ AZURE_TOKEN_SCOPE?: string | undefined;
39
+ AZURE_DEPLOYMENT_NAME?: string | undefined;
40
+ AZURE_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
41
+ AZURE_OPENAI_DEPLOYMENT_NAME?: string | undefined;
42
+ AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
27
43
  AZURE_OPENAI_API_BASE_URL?: string | undefined;
28
44
  AZURE_OPENAI_API_HOST?: string | undefined;
29
45
  AZURE_OPENAI_API_KEY?: string | undefined;
30
46
  AZURE_API_BASE_URL?: string | undefined;
31
47
  AZURE_API_HOST?: string | undefined;
32
48
  AZURE_API_KEY?: string | undefined;
33
- AZURE_DEPLOYMENT_NAME?: string | undefined;
34
- AZURE_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
35
49
  AZURE_OPENAI_BASE_URL?: string | undefined;
36
- AZURE_OPENAI_DEPLOYMENT_NAME?: string | undefined;
37
- AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
38
- AZURE_CLIENT_SECRET?: string | undefined;
39
- AZURE_CLIENT_ID?: string | undefined;
40
- AZURE_TENANT_ID?: string | undefined;
41
- AZURE_AUTHORITY_HOST?: string | undefined;
42
- AZURE_TOKEN_SCOPE?: string | undefined;
43
50
  BAM_API_HOST?: string | undefined;
44
51
  BAM_API_KEY?: string | undefined;
45
52
  CLOUDFLARE_ACCOUNT_ID?: string | undefined;
46
53
  CLOUDFLARE_API_KEY?: string | undefined;
47
- PROMPTFOO_REMOTE_GENERATION_URL?: string | undefined;
48
54
  COHERE_API_KEY?: string | undefined;
49
- FAL_KEY?: string | undefined;
50
55
  GOOGLE_API_HOST?: string | undefined;
51
56
  GOOGLE_API_KEY?: string | undefined;
52
- GROQ_API_KEY?: string | undefined;
53
- LOCALAI_BASE_URL?: string | undefined;
54
57
  MISTRAL_API_BASE_URL?: string | undefined;
55
58
  MISTRAL_API_HOST?: string | undefined;
56
59
  MISTRAL_API_KEY?: string | undefined;
@@ -68,9 +71,6 @@ export default class EvalFactory {
68
71
  VERTEX_PROJECT_ID?: string | undefined;
69
72
  VERTEX_PUBLISHER?: string | undefined;
70
73
  VERTEX_REGION?: string | undefined;
71
- WATSONX_AI_APIKEY?: string | undefined;
72
- WATSONX_AI_PROJECT_ID?: string | undefined;
73
- WATSONX_AI_BEARER_TOKEN?: string | undefined;
74
74
  } | undefined;
75
75
  metadata?: Record<string, any> | undefined;
76
76
  description?: string | undefined;
@@ -104,37 +104,40 @@ export default class EvalFactory {
104
104
  transform?: string | undefined;
105
105
  delay?: number | undefined;
106
106
  env?: {
107
+ PROMPTFOO_REMOTE_GENERATION_URL?: string | undefined;
107
108
  AI21_API_BASE_URL?: string | undefined;
108
109
  AI21_API_KEY?: string | undefined;
109
110
  ANTHROPIC_API_KEY?: string | undefined;
110
111
  AWS_BEDROCK_REGION?: string | undefined;
112
+ FAL_KEY?: string | undefined;
113
+ GROQ_API_KEY?: string | undefined;
114
+ LOCALAI_BASE_URL?: string | undefined;
115
+ WATSONX_AI_APIKEY?: string | undefined;
116
+ WATSONX_AI_PROJECT_ID?: string | undefined;
117
+ WATSONX_AI_BEARER_TOKEN?: string | undefined;
118
+ AZURE_CLIENT_SECRET?: string | undefined;
119
+ AZURE_CLIENT_ID?: string | undefined;
120
+ AZURE_TENANT_ID?: string | undefined;
121
+ AZURE_AUTHORITY_HOST?: string | undefined;
122
+ AZURE_TOKEN_SCOPE?: string | undefined;
123
+ AZURE_DEPLOYMENT_NAME?: string | undefined;
124
+ AZURE_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
125
+ AZURE_OPENAI_DEPLOYMENT_NAME?: string | undefined;
126
+ AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
111
127
  AZURE_OPENAI_API_BASE_URL?: string | undefined;
112
128
  AZURE_OPENAI_API_HOST?: string | undefined;
113
129
  AZURE_OPENAI_API_KEY?: string | undefined;
114
130
  AZURE_API_BASE_URL?: string | undefined;
115
131
  AZURE_API_HOST?: string | undefined;
116
132
  AZURE_API_KEY?: string | undefined;
117
- AZURE_DEPLOYMENT_NAME?: string | undefined;
118
- AZURE_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
119
133
  AZURE_OPENAI_BASE_URL?: string | undefined;
120
- AZURE_OPENAI_DEPLOYMENT_NAME?: string | undefined;
121
- AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
122
- AZURE_CLIENT_SECRET?: string | undefined;
123
- AZURE_CLIENT_ID?: string | undefined;
124
- AZURE_TENANT_ID?: string | undefined;
125
- AZURE_AUTHORITY_HOST?: string | undefined;
126
- AZURE_TOKEN_SCOPE?: string | undefined;
127
134
  BAM_API_HOST?: string | undefined;
128
135
  BAM_API_KEY?: string | undefined;
129
136
  CLOUDFLARE_ACCOUNT_ID?: string | undefined;
130
137
  CLOUDFLARE_API_KEY?: string | undefined;
131
- PROMPTFOO_REMOTE_GENERATION_URL?: string | undefined;
132
138
  COHERE_API_KEY?: string | undefined;
133
- FAL_KEY?: string | undefined;
134
139
  GOOGLE_API_HOST?: string | undefined;
135
140
  GOOGLE_API_KEY?: string | undefined;
136
- GROQ_API_KEY?: string | undefined;
137
- LOCALAI_BASE_URL?: string | undefined;
138
141
  MISTRAL_API_BASE_URL?: string | undefined;
139
142
  MISTRAL_API_HOST?: string | undefined;
140
143
  MISTRAL_API_KEY?: string | undefined;
@@ -152,9 +155,6 @@ export default class EvalFactory {
152
155
  VERTEX_PROJECT_ID?: string | undefined;
153
156
  VERTEX_PUBLISHER?: string | undefined;
154
157
  VERTEX_REGION?: string | undefined;
155
- WATSONX_AI_APIKEY?: string | undefined;
156
- WATSONX_AI_PROJECT_ID?: string | undefined;
157
- WATSONX_AI_BEARER_TOKEN?: string | undefined;
158
158
  } | undefined;
159
159
  } | (((args_0: string, args_1: {
160
160
  vars: Record<string, string | {}>;
@@ -186,37 +186,40 @@ export default class EvalFactory {
186
186
  transform?: string | undefined;
187
187
  delay?: number | undefined;
188
188
  env?: {
189
+ PROMPTFOO_REMOTE_GENERATION_URL?: string | undefined;
189
190
  AI21_API_BASE_URL?: string | undefined;
190
191
  AI21_API_KEY?: string | undefined;
191
192
  ANTHROPIC_API_KEY?: string | undefined;
192
193
  AWS_BEDROCK_REGION?: string | undefined;
194
+ FAL_KEY?: string | undefined;
195
+ GROQ_API_KEY?: string | undefined;
196
+ LOCALAI_BASE_URL?: string | undefined;
197
+ WATSONX_AI_APIKEY?: string | undefined;
198
+ WATSONX_AI_PROJECT_ID?: string | undefined;
199
+ WATSONX_AI_BEARER_TOKEN?: string | undefined;
200
+ AZURE_CLIENT_SECRET?: string | undefined;
201
+ AZURE_CLIENT_ID?: string | undefined;
202
+ AZURE_TENANT_ID?: string | undefined;
203
+ AZURE_AUTHORITY_HOST?: string | undefined;
204
+ AZURE_TOKEN_SCOPE?: string | undefined;
205
+ AZURE_DEPLOYMENT_NAME?: string | undefined;
206
+ AZURE_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
207
+ AZURE_OPENAI_DEPLOYMENT_NAME?: string | undefined;
208
+ AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
193
209
  AZURE_OPENAI_API_BASE_URL?: string | undefined;
194
210
  AZURE_OPENAI_API_HOST?: string | undefined;
195
211
  AZURE_OPENAI_API_KEY?: string | undefined;
196
212
  AZURE_API_BASE_URL?: string | undefined;
197
213
  AZURE_API_HOST?: string | undefined;
198
214
  AZURE_API_KEY?: string | undefined;
199
- AZURE_DEPLOYMENT_NAME?: string | undefined;
200
- AZURE_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
201
215
  AZURE_OPENAI_BASE_URL?: string | undefined;
202
- AZURE_OPENAI_DEPLOYMENT_NAME?: string | undefined;
203
- AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
204
- AZURE_CLIENT_SECRET?: string | undefined;
205
- AZURE_CLIENT_ID?: string | undefined;
206
- AZURE_TENANT_ID?: string | undefined;
207
- AZURE_AUTHORITY_HOST?: string | undefined;
208
- AZURE_TOKEN_SCOPE?: string | undefined;
209
216
  BAM_API_HOST?: string | undefined;
210
217
  BAM_API_KEY?: string | undefined;
211
218
  CLOUDFLARE_ACCOUNT_ID?: string | undefined;
212
219
  CLOUDFLARE_API_KEY?: string | undefined;
213
- PROMPTFOO_REMOTE_GENERATION_URL?: string | undefined;
214
220
  COHERE_API_KEY?: string | undefined;
215
- FAL_KEY?: string | undefined;
216
221
  GOOGLE_API_HOST?: string | undefined;
217
222
  GOOGLE_API_KEY?: string | undefined;
218
- GROQ_API_KEY?: string | undefined;
219
- LOCALAI_BASE_URL?: string | undefined;
220
223
  MISTRAL_API_BASE_URL?: string | undefined;
221
224
  MISTRAL_API_HOST?: string | undefined;
222
225
  MISTRAL_API_KEY?: string | undefined;
@@ -234,9 +237,6 @@ export default class EvalFactory {
234
237
  VERTEX_PROJECT_ID?: string | undefined;
235
238
  VERTEX_PUBLISHER?: string | undefined;
236
239
  VERTEX_REGION?: string | undefined;
237
- WATSONX_AI_APIKEY?: string | undefined;
238
- WATSONX_AI_PROJECT_ID?: string | undefined;
239
- WATSONX_AI_BEARER_TOKEN?: string | undefined;
240
240
  } | undefined;
241
241
  }>)[] | undefined;
242
242
  tests?: string | (string | {
@@ -275,37 +275,40 @@ export default class EvalFactory {
275
275
  transform?: string | undefined;
276
276
  delay?: number | undefined;
277
277
  env?: {
278
+ PROMPTFOO_REMOTE_GENERATION_URL?: string | undefined;
278
279
  AI21_API_BASE_URL?: string | undefined;
279
280
  AI21_API_KEY?: string | undefined;
280
281
  ANTHROPIC_API_KEY?: string | undefined;
281
282
  AWS_BEDROCK_REGION?: string | undefined;
283
+ FAL_KEY?: string | undefined;
284
+ GROQ_API_KEY?: string | undefined;
285
+ LOCALAI_BASE_URL?: string | undefined;
286
+ WATSONX_AI_APIKEY?: string | undefined;
287
+ WATSONX_AI_PROJECT_ID?: string | undefined;
288
+ WATSONX_AI_BEARER_TOKEN?: string | undefined;
289
+ AZURE_CLIENT_SECRET?: string | undefined;
290
+ AZURE_CLIENT_ID?: string | undefined;
291
+ AZURE_TENANT_ID?: string | undefined;
292
+ AZURE_AUTHORITY_HOST?: string | undefined;
293
+ AZURE_TOKEN_SCOPE?: string | undefined;
294
+ AZURE_DEPLOYMENT_NAME?: string | undefined;
295
+ AZURE_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
296
+ AZURE_OPENAI_DEPLOYMENT_NAME?: string | undefined;
297
+ AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
282
298
  AZURE_OPENAI_API_BASE_URL?: string | undefined;
283
299
  AZURE_OPENAI_API_HOST?: string | undefined;
284
300
  AZURE_OPENAI_API_KEY?: string | undefined;
285
301
  AZURE_API_BASE_URL?: string | undefined;
286
302
  AZURE_API_HOST?: string | undefined;
287
303
  AZURE_API_KEY?: string | undefined;
288
- AZURE_DEPLOYMENT_NAME?: string | undefined;
289
- AZURE_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
290
304
  AZURE_OPENAI_BASE_URL?: string | undefined;
291
- AZURE_OPENAI_DEPLOYMENT_NAME?: string | undefined;
292
- AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
293
- AZURE_CLIENT_SECRET?: string | undefined;
294
- AZURE_CLIENT_ID?: string | undefined;
295
- AZURE_TENANT_ID?: string | undefined;
296
- AZURE_AUTHORITY_HOST?: string | undefined;
297
- AZURE_TOKEN_SCOPE?: string | undefined;
298
305
  BAM_API_HOST?: string | undefined;
299
306
  BAM_API_KEY?: string | undefined;
300
307
  CLOUDFLARE_ACCOUNT_ID?: string | undefined;
301
308
  CLOUDFLARE_API_KEY?: string | undefined;
302
- PROMPTFOO_REMOTE_GENERATION_URL?: string | undefined;
303
309
  COHERE_API_KEY?: string | undefined;
304
- FAL_KEY?: string | undefined;
305
310
  GOOGLE_API_HOST?: string | undefined;
306
311
  GOOGLE_API_KEY?: string | undefined;
307
- GROQ_API_KEY?: string | undefined;
308
- LOCALAI_BASE_URL?: string | undefined;
309
312
  MISTRAL_API_BASE_URL?: string | undefined;
310
313
  MISTRAL_API_HOST?: string | undefined;
311
314
  MISTRAL_API_KEY?: string | undefined;
@@ -323,9 +326,6 @@ export default class EvalFactory {
323
326
  VERTEX_PROJECT_ID?: string | undefined;
324
327
  VERTEX_PUBLISHER?: string | undefined;
325
328
  VERTEX_REGION?: string | undefined;
326
- WATSONX_AI_APIKEY?: string | undefined;
327
- WATSONX_AI_PROJECT_ID?: string | undefined;
328
- WATSONX_AI_BEARER_TOKEN?: string | undefined;
329
329
  } | undefined;
330
330
  } | {
331
331
  callApi: import("../../src").CallApiFunction;
@@ -417,37 +417,40 @@ export default class EvalFactory {
417
417
  transform?: string | undefined;
418
418
  delay?: number | undefined;
419
419
  env?: {
420
+ PROMPTFOO_REMOTE_GENERATION_URL?: string | undefined;
420
421
  AI21_API_BASE_URL?: string | undefined;
421
422
  AI21_API_KEY?: string | undefined;
422
423
  ANTHROPIC_API_KEY?: string | undefined;
423
424
  AWS_BEDROCK_REGION?: string | undefined;
425
+ FAL_KEY?: string | undefined;
426
+ GROQ_API_KEY?: string | undefined;
427
+ LOCALAI_BASE_URL?: string | undefined;
428
+ WATSONX_AI_APIKEY?: string | undefined;
429
+ WATSONX_AI_PROJECT_ID?: string | undefined;
430
+ WATSONX_AI_BEARER_TOKEN?: string | undefined;
431
+ AZURE_CLIENT_SECRET?: string | undefined;
432
+ AZURE_CLIENT_ID?: string | undefined;
433
+ AZURE_TENANT_ID?: string | undefined;
434
+ AZURE_AUTHORITY_HOST?: string | undefined;
435
+ AZURE_TOKEN_SCOPE?: string | undefined;
436
+ AZURE_DEPLOYMENT_NAME?: string | undefined;
437
+ AZURE_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
438
+ AZURE_OPENAI_DEPLOYMENT_NAME?: string | undefined;
439
+ AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
424
440
  AZURE_OPENAI_API_BASE_URL?: string | undefined;
425
441
  AZURE_OPENAI_API_HOST?: string | undefined;
426
442
  AZURE_OPENAI_API_KEY?: string | undefined;
427
443
  AZURE_API_BASE_URL?: string | undefined;
428
444
  AZURE_API_HOST?: string | undefined;
429
445
  AZURE_API_KEY?: string | undefined;
430
- AZURE_DEPLOYMENT_NAME?: string | undefined;
431
- AZURE_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
432
446
  AZURE_OPENAI_BASE_URL?: string | undefined;
433
- AZURE_OPENAI_DEPLOYMENT_NAME?: string | undefined;
434
- AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
435
- AZURE_CLIENT_SECRET?: string | undefined;
436
- AZURE_CLIENT_ID?: string | undefined;
437
- AZURE_TENANT_ID?: string | undefined;
438
- AZURE_AUTHORITY_HOST?: string | undefined;
439
- AZURE_TOKEN_SCOPE?: string | undefined;
440
447
  BAM_API_HOST?: string | undefined;
441
448
  BAM_API_KEY?: string | undefined;
442
449
  CLOUDFLARE_ACCOUNT_ID?: string | undefined;
443
450
  CLOUDFLARE_API_KEY?: string | undefined;
444
- PROMPTFOO_REMOTE_GENERATION_URL?: string | undefined;
445
451
  COHERE_API_KEY?: string | undefined;
446
- FAL_KEY?: string | undefined;
447
452
  GOOGLE_API_HOST?: string | undefined;
448
453
  GOOGLE_API_KEY?: string | undefined;
449
- GROQ_API_KEY?: string | undefined;
450
- LOCALAI_BASE_URL?: string | undefined;
451
454
  MISTRAL_API_BASE_URL?: string | undefined;
452
455
  MISTRAL_API_HOST?: string | undefined;
453
456
  MISTRAL_API_KEY?: string | undefined;
@@ -465,9 +468,6 @@ export default class EvalFactory {
465
468
  VERTEX_PROJECT_ID?: string | undefined;
466
469
  VERTEX_PUBLISHER?: string | undefined;
467
470
  VERTEX_REGION?: string | undefined;
468
- WATSONX_AI_APIKEY?: string | undefined;
469
- WATSONX_AI_PROJECT_ID?: string | undefined;
470
- WATSONX_AI_BEARER_TOKEN?: string | undefined;
471
471
  } | undefined;
472
472
  } | {
473
473
  callApi: import("../../src").CallApiFunction;
@@ -556,37 +556,40 @@ export default class EvalFactory {
556
556
  transform?: string | undefined;
557
557
  delay?: number | undefined;
558
558
  env?: {
559
+ PROMPTFOO_REMOTE_GENERATION_URL?: string | undefined;
559
560
  AI21_API_BASE_URL?: string | undefined;
560
561
  AI21_API_KEY?: string | undefined;
561
562
  ANTHROPIC_API_KEY?: string | undefined;
562
563
  AWS_BEDROCK_REGION?: string | undefined;
564
+ FAL_KEY?: string | undefined;
565
+ GROQ_API_KEY?: string | undefined;
566
+ LOCALAI_BASE_URL?: string | undefined;
567
+ WATSONX_AI_APIKEY?: string | undefined;
568
+ WATSONX_AI_PROJECT_ID?: string | undefined;
569
+ WATSONX_AI_BEARER_TOKEN?: string | undefined;
570
+ AZURE_CLIENT_SECRET?: string | undefined;
571
+ AZURE_CLIENT_ID?: string | undefined;
572
+ AZURE_TENANT_ID?: string | undefined;
573
+ AZURE_AUTHORITY_HOST?: string | undefined;
574
+ AZURE_TOKEN_SCOPE?: string | undefined;
575
+ AZURE_DEPLOYMENT_NAME?: string | undefined;
576
+ AZURE_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
577
+ AZURE_OPENAI_DEPLOYMENT_NAME?: string | undefined;
578
+ AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
563
579
  AZURE_OPENAI_API_BASE_URL?: string | undefined;
564
580
  AZURE_OPENAI_API_HOST?: string | undefined;
565
581
  AZURE_OPENAI_API_KEY?: string | undefined;
566
582
  AZURE_API_BASE_URL?: string | undefined;
567
583
  AZURE_API_HOST?: string | undefined;
568
584
  AZURE_API_KEY?: string | undefined;
569
- AZURE_DEPLOYMENT_NAME?: string | undefined;
570
- AZURE_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
571
585
  AZURE_OPENAI_BASE_URL?: string | undefined;
572
- AZURE_OPENAI_DEPLOYMENT_NAME?: string | undefined;
573
- AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
574
- AZURE_CLIENT_SECRET?: string | undefined;
575
- AZURE_CLIENT_ID?: string | undefined;
576
- AZURE_TENANT_ID?: string | undefined;
577
- AZURE_AUTHORITY_HOST?: string | undefined;
578
- AZURE_TOKEN_SCOPE?: string | undefined;
579
586
  BAM_API_HOST?: string | undefined;
580
587
  BAM_API_KEY?: string | undefined;
581
588
  CLOUDFLARE_ACCOUNT_ID?: string | undefined;
582
589
  CLOUDFLARE_API_KEY?: string | undefined;
583
- PROMPTFOO_REMOTE_GENERATION_URL?: string | undefined;
584
590
  COHERE_API_KEY?: string | undefined;
585
- FAL_KEY?: string | undefined;
586
591
  GOOGLE_API_HOST?: string | undefined;
587
592
  GOOGLE_API_KEY?: string | undefined;
588
- GROQ_API_KEY?: string | undefined;
589
- LOCALAI_BASE_URL?: string | undefined;
590
593
  MISTRAL_API_BASE_URL?: string | undefined;
591
594
  MISTRAL_API_HOST?: string | undefined;
592
595
  MISTRAL_API_KEY?: string | undefined;
@@ -604,9 +607,6 @@ export default class EvalFactory {
604
607
  VERTEX_PROJECT_ID?: string | undefined;
605
608
  VERTEX_PUBLISHER?: string | undefined;
606
609
  VERTEX_REGION?: string | undefined;
607
- WATSONX_AI_APIKEY?: string | undefined;
608
- WATSONX_AI_PROJECT_ID?: string | undefined;
609
- WATSONX_AI_BEARER_TOKEN?: string | undefined;
610
610
  } | undefined;
611
611
  } | {
612
612
  callApi: import("../../src").CallApiFunction;
@@ -697,37 +697,40 @@ export default class EvalFactory {
697
697
  transform?: string | undefined;
698
698
  delay?: number | undefined;
699
699
  env?: {
700
+ PROMPTFOO_REMOTE_GENERATION_URL?: string | undefined;
700
701
  AI21_API_BASE_URL?: string | undefined;
701
702
  AI21_API_KEY?: string | undefined;
702
703
  ANTHROPIC_API_KEY?: string | undefined;
703
704
  AWS_BEDROCK_REGION?: string | undefined;
705
+ FAL_KEY?: string | undefined;
706
+ GROQ_API_KEY?: string | undefined;
707
+ LOCALAI_BASE_URL?: string | undefined;
708
+ WATSONX_AI_APIKEY?: string | undefined;
709
+ WATSONX_AI_PROJECT_ID?: string | undefined;
710
+ WATSONX_AI_BEARER_TOKEN?: string | undefined;
711
+ AZURE_CLIENT_SECRET?: string | undefined;
712
+ AZURE_CLIENT_ID?: string | undefined;
713
+ AZURE_TENANT_ID?: string | undefined;
714
+ AZURE_AUTHORITY_HOST?: string | undefined;
715
+ AZURE_TOKEN_SCOPE?: string | undefined;
716
+ AZURE_DEPLOYMENT_NAME?: string | undefined;
717
+ AZURE_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
718
+ AZURE_OPENAI_DEPLOYMENT_NAME?: string | undefined;
719
+ AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
704
720
  AZURE_OPENAI_API_BASE_URL?: string | undefined;
705
721
  AZURE_OPENAI_API_HOST?: string | undefined;
706
722
  AZURE_OPENAI_API_KEY?: string | undefined;
707
723
  AZURE_API_BASE_URL?: string | undefined;
708
724
  AZURE_API_HOST?: string | undefined;
709
725
  AZURE_API_KEY?: string | undefined;
710
- AZURE_DEPLOYMENT_NAME?: string | undefined;
711
- AZURE_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
712
726
  AZURE_OPENAI_BASE_URL?: string | undefined;
713
- AZURE_OPENAI_DEPLOYMENT_NAME?: string | undefined;
714
- AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
715
- AZURE_CLIENT_SECRET?: string | undefined;
716
- AZURE_CLIENT_ID?: string | undefined;
717
- AZURE_TENANT_ID?: string | undefined;
718
- AZURE_AUTHORITY_HOST?: string | undefined;
719
- AZURE_TOKEN_SCOPE?: string | undefined;
720
727
  BAM_API_HOST?: string | undefined;
721
728
  BAM_API_KEY?: string | undefined;
722
729
  CLOUDFLARE_ACCOUNT_ID?: string | undefined;
723
730
  CLOUDFLARE_API_KEY?: string | undefined;
724
- PROMPTFOO_REMOTE_GENERATION_URL?: string | undefined;
725
731
  COHERE_API_KEY?: string | undefined;
726
- FAL_KEY?: string | undefined;
727
732
  GOOGLE_API_HOST?: string | undefined;
728
733
  GOOGLE_API_KEY?: string | undefined;
729
- GROQ_API_KEY?: string | undefined;
730
- LOCALAI_BASE_URL?: string | undefined;
731
734
  MISTRAL_API_BASE_URL?: string | undefined;
732
735
  MISTRAL_API_HOST?: string | undefined;
733
736
  MISTRAL_API_KEY?: string | undefined;
@@ -745,9 +748,6 @@ export default class EvalFactory {
745
748
  VERTEX_PROJECT_ID?: string | undefined;
746
749
  VERTEX_PUBLISHER?: string | undefined;
747
750
  VERTEX_REGION?: string | undefined;
748
- WATSONX_AI_APIKEY?: string | undefined;
749
- WATSONX_AI_PROJECT_ID?: string | undefined;
750
- WATSONX_AI_BEARER_TOKEN?: string | undefined;
751
751
  } | undefined;
752
752
  } | {
753
753
  callApi: import("../../src").CallApiFunction;
@@ -886,37 +886,40 @@ export default class EvalFactory {
886
886
  transform?: string | undefined;
887
887
  delay?: number | undefined;
888
888
  env?: {
889
+ PROMPTFOO_REMOTE_GENERATION_URL?: string | undefined;
889
890
  AI21_API_BASE_URL?: string | undefined;
890
891
  AI21_API_KEY?: string | undefined;
891
892
  ANTHROPIC_API_KEY?: string | undefined;
892
893
  AWS_BEDROCK_REGION?: string | undefined;
894
+ FAL_KEY?: string | undefined;
895
+ GROQ_API_KEY?: string | undefined;
896
+ LOCALAI_BASE_URL?: string | undefined;
897
+ WATSONX_AI_APIKEY?: string | undefined;
898
+ WATSONX_AI_PROJECT_ID?: string | undefined;
899
+ WATSONX_AI_BEARER_TOKEN?: string | undefined;
900
+ AZURE_CLIENT_SECRET?: string | undefined;
901
+ AZURE_CLIENT_ID?: string | undefined;
902
+ AZURE_TENANT_ID?: string | undefined;
903
+ AZURE_AUTHORITY_HOST?: string | undefined;
904
+ AZURE_TOKEN_SCOPE?: string | undefined;
905
+ AZURE_DEPLOYMENT_NAME?: string | undefined;
906
+ AZURE_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
907
+ AZURE_OPENAI_DEPLOYMENT_NAME?: string | undefined;
908
+ AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
893
909
  AZURE_OPENAI_API_BASE_URL?: string | undefined;
894
910
  AZURE_OPENAI_API_HOST?: string | undefined;
895
911
  AZURE_OPENAI_API_KEY?: string | undefined;
896
912
  AZURE_API_BASE_URL?: string | undefined;
897
913
  AZURE_API_HOST?: string | undefined;
898
914
  AZURE_API_KEY?: string | undefined;
899
- AZURE_DEPLOYMENT_NAME?: string | undefined;
900
- AZURE_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
901
915
  AZURE_OPENAI_BASE_URL?: string | undefined;
902
- AZURE_OPENAI_DEPLOYMENT_NAME?: string | undefined;
903
- AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
904
- AZURE_CLIENT_SECRET?: string | undefined;
905
- AZURE_CLIENT_ID?: string | undefined;
906
- AZURE_TENANT_ID?: string | undefined;
907
- AZURE_AUTHORITY_HOST?: string | undefined;
908
- AZURE_TOKEN_SCOPE?: string | undefined;
909
916
  BAM_API_HOST?: string | undefined;
910
917
  BAM_API_KEY?: string | undefined;
911
918
  CLOUDFLARE_ACCOUNT_ID?: string | undefined;
912
919
  CLOUDFLARE_API_KEY?: string | undefined;
913
- PROMPTFOO_REMOTE_GENERATION_URL?: string | undefined;
914
920
  COHERE_API_KEY?: string | undefined;
915
- FAL_KEY?: string | undefined;
916
921
  GOOGLE_API_HOST?: string | undefined;
917
922
  GOOGLE_API_KEY?: string | undefined;
918
- GROQ_API_KEY?: string | undefined;
919
- LOCALAI_BASE_URL?: string | undefined;
920
923
  MISTRAL_API_BASE_URL?: string | undefined;
921
924
  MISTRAL_API_HOST?: string | undefined;
922
925
  MISTRAL_API_KEY?: string | undefined;
@@ -934,9 +937,6 @@ export default class EvalFactory {
934
937
  VERTEX_PROJECT_ID?: string | undefined;
935
938
  VERTEX_PUBLISHER?: string | undefined;
936
939
  VERTEX_REGION?: string | undefined;
937
- WATSONX_AI_APIKEY?: string | undefined;
938
- WATSONX_AI_PROJECT_ID?: string | undefined;
939
- WATSONX_AI_BEARER_TOKEN?: string | undefined;
940
940
  } | undefined;
941
941
  } | (((args_0: string, args_1: {
942
942
  vars: Record<string, string | {}>;
@@ -968,37 +968,40 @@ export default class EvalFactory {
968
968
  transform?: string | undefined;
969
969
  delay?: number | undefined;
970
970
  env?: {
971
+ PROMPTFOO_REMOTE_GENERATION_URL?: string | undefined;
971
972
  AI21_API_BASE_URL?: string | undefined;
972
973
  AI21_API_KEY?: string | undefined;
973
974
  ANTHROPIC_API_KEY?: string | undefined;
974
975
  AWS_BEDROCK_REGION?: string | undefined;
976
+ FAL_KEY?: string | undefined;
977
+ GROQ_API_KEY?: string | undefined;
978
+ LOCALAI_BASE_URL?: string | undefined;
979
+ WATSONX_AI_APIKEY?: string | undefined;
980
+ WATSONX_AI_PROJECT_ID?: string | undefined;
981
+ WATSONX_AI_BEARER_TOKEN?: string | undefined;
982
+ AZURE_CLIENT_SECRET?: string | undefined;
983
+ AZURE_CLIENT_ID?: string | undefined;
984
+ AZURE_TENANT_ID?: string | undefined;
985
+ AZURE_AUTHORITY_HOST?: string | undefined;
986
+ AZURE_TOKEN_SCOPE?: string | undefined;
987
+ AZURE_DEPLOYMENT_NAME?: string | undefined;
988
+ AZURE_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
989
+ AZURE_OPENAI_DEPLOYMENT_NAME?: string | undefined;
990
+ AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
975
991
  AZURE_OPENAI_API_BASE_URL?: string | undefined;
976
992
  AZURE_OPENAI_API_HOST?: string | undefined;
977
993
  AZURE_OPENAI_API_KEY?: string | undefined;
978
994
  AZURE_API_BASE_URL?: string | undefined;
979
995
  AZURE_API_HOST?: string | undefined;
980
996
  AZURE_API_KEY?: string | undefined;
981
- AZURE_DEPLOYMENT_NAME?: string | undefined;
982
- AZURE_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
983
997
  AZURE_OPENAI_BASE_URL?: string | undefined;
984
- AZURE_OPENAI_DEPLOYMENT_NAME?: string | undefined;
985
- AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
986
- AZURE_CLIENT_SECRET?: string | undefined;
987
- AZURE_CLIENT_ID?: string | undefined;
988
- AZURE_TENANT_ID?: string | undefined;
989
- AZURE_AUTHORITY_HOST?: string | undefined;
990
- AZURE_TOKEN_SCOPE?: string | undefined;
991
998
  BAM_API_HOST?: string | undefined;
992
999
  BAM_API_KEY?: string | undefined;
993
1000
  CLOUDFLARE_ACCOUNT_ID?: string | undefined;
994
1001
  CLOUDFLARE_API_KEY?: string | undefined;
995
- PROMPTFOO_REMOTE_GENERATION_URL?: string | undefined;
996
1002
  COHERE_API_KEY?: string | undefined;
997
- FAL_KEY?: string | undefined;
998
1003
  GOOGLE_API_HOST?: string | undefined;
999
1004
  GOOGLE_API_KEY?: string | undefined;
1000
- GROQ_API_KEY?: string | undefined;
1001
- LOCALAI_BASE_URL?: string | undefined;
1002
1005
  MISTRAL_API_BASE_URL?: string | undefined;
1003
1006
  MISTRAL_API_HOST?: string | undefined;
1004
1007
  MISTRAL_API_KEY?: string | undefined;
@@ -1016,9 +1019,6 @@ export default class EvalFactory {
1016
1019
  VERTEX_PROJECT_ID?: string | undefined;
1017
1020
  VERTEX_PUBLISHER?: string | undefined;
1018
1021
  VERTEX_REGION?: string | undefined;
1019
- WATSONX_AI_APIKEY?: string | undefined;
1020
- WATSONX_AI_PROJECT_ID?: string | undefined;
1021
- WATSONX_AI_BEARER_TOKEN?: string | undefined;
1022
1022
  } | undefined;
1023
1023
  }>)[] | undefined;
1024
1024
  }>;
package/dist/test/prompts/index.test.js CHANGED
@@ -250,50 +250,63 @@ describe('readPrompts', () => {
250
250
  ]);
251
251
  expect(fs.readFileSync).toHaveBeenCalledTimes(1);
252
252
  });
253
- it('should read a .yaml file', async () => {
254
- const yamlContent = (0, dedent_1.default) `
253
+ const yamlContent = (0, dedent_1.default) `
255
254
  - role: user
256
255
  content:
257
256
  - type: text
258
257
  text: "What's in this image?"
259
258
  - type: image_url
260
259
  image_url:
261
- url: "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg"
262
- `;
260
+ url: "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg"`;
261
+ it('should read a .yaml file', async () => {
262
+ const expectedJson = JSON.stringify([
263
+ {
264
+ role: 'user',
265
+ content: [
266
+ { type: 'text', text: "What's in this image?" },
267
+ {
268
+ type: 'image_url',
269
+ image_url: {
270
+ url: 'https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg',
271
+ },
272
+ },
273
+ ],
274
+ },
275
+ ]);
263
276
  jest.mocked(fs.readFileSync).mockReturnValueOnce(yamlContent);
264
277
  jest.mocked(fs.statSync).mockReturnValueOnce({ isDirectory: () => false });
265
278
  await expect((0, prompts_1.readPrompts)('prompts.yaml')).resolves.toEqual([
266
279
  {
267
- raw: yamlContent,
268
- label: `prompts.yaml: ${yamlContent}`,
280
+ raw: expectedJson,
281
+ label: expect.stringContaining('prompts.yaml: [{"role":"user","content":[{"type":"text","text":"What\'s in this image?"}'),
282
+ config: undefined,
269
283
  },
270
284
  ]);
271
- expect(fs.readFileSync).toHaveBeenCalledTimes(1);
272
285
  });
273
286
  it('should read a .yml file', async () => {
274
- const ymlContent = (0, dedent_1.default) `
275
- - role: user
276
- content:
277
- - type: text
278
- text: "What's in this image?"
279
- - type: image_url
280
- image_url:
281
- url: "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg"
282
- `;
283
- jest.mocked(fs.readFileSync).mockReturnValue(ymlContent);
284
- jest.mocked(fs.statSync).mockReturnValueOnce({ isDirectory: () => false });
285
- await expect((0, prompts_1.readPrompts)([
287
+ const expectedJson = JSON.stringify([
286
288
  {
287
- id: 'prompts.yml',
288
- label: 'image-summary',
289
+ role: 'user',
290
+ content: [
291
+ { type: 'text', text: "What's in this image?" },
292
+ {
293
+ type: 'image_url',
294
+ image_url: {
295
+ url: 'https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg',
296
+ },
297
+ },
298
+ ],
289
299
  },
290
- ])).resolves.toEqual([
300
+ ]);
301
+ jest.mocked(fs.readFileSync).mockReturnValueOnce(yamlContent);
302
+ jest.mocked(fs.statSync).mockReturnValueOnce({ isDirectory: () => false });
303
+ await expect((0, prompts_1.readPrompts)('image-summary.yml')).resolves.toEqual([
291
304
  {
292
- raw: ymlContent,
293
- label: `image-summary`,
305
+ raw: expectedJson,
306
+ label: expect.stringContaining('image-summary.yml: [{"role":"user","content":[{"type":"text","text":"What\'s in this image?"}'),
307
+ config: undefined,
294
308
  },
295
309
  ]);
296
- expect(fs.readFileSync).toHaveBeenCalledTimes(1);
297
310
  });
298
311
  it('should read a .py prompt object array', async () => {
299
312
  const prompts = [