promptfoo 0.73.9 → 0.74.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (288) hide show
  1. package/README.md +59 -1
  2. package/dist/package.json +1 -1
  3. package/dist/src/assertions/AssertionsResult.d.ts +1 -1
  4. package/dist/src/assertions/AssertionsResult.d.ts.map +1 -1
  5. package/dist/src/assertions/validateAssertions.d.ts +1 -1
  6. package/dist/src/assertions/validateAssertions.d.ts.map +1 -1
  7. package/dist/src/assertions.d.ts +2 -1
  8. package/dist/src/assertions.d.ts.map +1 -1
  9. package/dist/src/assertions.js.map +1 -1
  10. package/dist/src/commands/cache.d.ts +1 -1
  11. package/dist/src/commands/cache.d.ts.map +1 -1
  12. package/dist/src/commands/config.d.ts +1 -1
  13. package/dist/src/commands/config.d.ts.map +1 -1
  14. package/dist/src/commands/delete.d.ts +1 -1
  15. package/dist/src/commands/delete.d.ts.map +1 -1
  16. package/dist/src/commands/eval/filterFailingTests.d.ts +1 -1
  17. package/dist/src/commands/eval/filterFailingTests.d.ts.map +1 -1
  18. package/dist/src/commands/eval/filterTests.d.ts +1 -1
  19. package/dist/src/commands/eval/filterTests.d.ts.map +1 -1
  20. package/dist/src/commands/eval.d.ts +2 -2
  21. package/dist/src/commands/eval.d.ts.map +1 -1
  22. package/dist/src/commands/eval.js.map +1 -1
  23. package/dist/src/commands/export.d.ts +1 -1
  24. package/dist/src/commands/export.d.ts.map +1 -1
  25. package/dist/src/commands/feedback.d.ts +1 -1
  26. package/dist/src/commands/feedback.d.ts.map +1 -1
  27. package/dist/src/commands/generate/dataset.d.ts +2 -2
  28. package/dist/src/commands/generate/dataset.d.ts.map +1 -1
  29. package/dist/src/commands/generate/redteam.d.ts +3 -3
  30. package/dist/src/commands/generate/redteam.d.ts.map +1 -1
  31. package/dist/src/commands/generate/redteam.js +13 -6
  32. package/dist/src/commands/generate/redteam.js.map +1 -1
  33. package/dist/src/commands/import.d.ts +1 -1
  34. package/dist/src/commands/import.d.ts.map +1 -1
  35. package/dist/src/commands/init.d.ts +1 -1
  36. package/dist/src/commands/init.d.ts.map +1 -1
  37. package/dist/src/commands/list.d.ts +1 -1
  38. package/dist/src/commands/list.d.ts.map +1 -1
  39. package/dist/src/commands/redteam.d.ts +2 -1
  40. package/dist/src/commands/redteam.d.ts.map +1 -1
  41. package/dist/src/commands/redteam.js +200 -193
  42. package/dist/src/commands/redteam.js.map +1 -1
  43. package/dist/src/commands/share.d.ts +1 -1
  44. package/dist/src/commands/share.d.ts.map +1 -1
  45. package/dist/src/commands/show.d.ts +1 -1
  46. package/dist/src/commands/show.d.ts.map +1 -1
  47. package/dist/src/commands/version.d.ts +1 -1
  48. package/dist/src/commands/version.d.ts.map +1 -1
  49. package/dist/src/commands/view.d.ts +1 -1
  50. package/dist/src/commands/view.d.ts.map +1 -1
  51. package/dist/src/config.d.ts +1 -1
  52. package/dist/src/config.d.ts.map +1 -1
  53. package/dist/src/database/operations.d.ts +56 -240
  54. package/dist/src/database/operations.d.ts.map +1 -1
  55. package/dist/src/database/types.d.ts +1 -1
  56. package/dist/src/database/types.d.ts.map +1 -1
  57. package/dist/src/globalConfig.d.ts +1 -1
  58. package/dist/src/globalConfig.d.ts.map +1 -1
  59. package/dist/src/onboarding.d.ts.map +1 -1
  60. package/dist/src/onboarding.js +11 -0
  61. package/dist/src/onboarding.js.map +1 -1
  62. package/dist/src/prompts/grading.js +1 -1
  63. package/dist/src/prompts/processors/javascript.d.ts +1 -1
  64. package/dist/src/prompts/processors/javascript.d.ts.map +1 -1
  65. package/dist/src/prompts/processors/json.d.ts +1 -1
  66. package/dist/src/prompts/processors/json.d.ts.map +1 -1
  67. package/dist/src/prompts/processors/jsonl.d.ts +1 -1
  68. package/dist/src/prompts/processors/jsonl.d.ts.map +1 -1
  69. package/dist/src/prompts/processors/python.d.ts.map +1 -1
  70. package/dist/src/prompts/processors/python.js.map +1 -1
  71. package/dist/src/prompts/processors/string.d.ts +1 -1
  72. package/dist/src/prompts/processors/string.d.ts.map +1 -1
  73. package/dist/src/prompts/processors/text.d.ts +1 -1
  74. package/dist/src/prompts/processors/text.d.ts.map +1 -1
  75. package/dist/src/prompts/processors/yaml.d.ts +1 -1
  76. package/dist/src/prompts/processors/yaml.d.ts.map +1 -1
  77. package/dist/src/prompts/utils.d.ts +1 -1
  78. package/dist/src/prompts/utils.d.ts.map +1 -1
  79. package/dist/src/providers/anthropic.js +1 -1
  80. package/dist/src/providers/anthropic.js.map +1 -1
  81. package/dist/src/providers/azureopenai.d.ts.map +1 -1
  82. package/dist/src/providers/azureopenai.js +6 -2
  83. package/dist/src/providers/azureopenai.js.map +1 -1
  84. package/dist/src/providers/bedrock.d.ts.map +1 -1
  85. package/dist/src/providers/bedrock.js +2 -2
  86. package/dist/src/providers/bedrock.js.map +1 -1
  87. package/dist/src/providers/huggingface.d.ts +1 -1
  88. package/dist/src/providers/huggingface.d.ts.map +1 -1
  89. package/dist/src/providers/manualInput.d.ts +1 -1
  90. package/dist/src/providers/manualInput.d.ts.map +1 -1
  91. package/dist/src/providers/mistral.d.ts +1 -1
  92. package/dist/src/providers/mistral.d.ts.map +1 -1
  93. package/dist/src/providers/openai.d.ts +1 -1
  94. package/dist/src/providers/openai.d.ts.map +1 -1
  95. package/dist/src/providers/openai.js +8 -3
  96. package/dist/src/providers/openai.js.map +1 -1
  97. package/dist/src/providers/shared.d.ts +1 -0
  98. package/dist/src/providers/shared.d.ts.map +1 -1
  99. package/dist/src/providers/shared.js +47 -0
  100. package/dist/src/providers/shared.js.map +1 -1
  101. package/dist/src/providers/vertex.d.ts.map +1 -1
  102. package/dist/src/providers/vertex.js.map +1 -1
  103. package/dist/src/providers.d.ts +2 -1
  104. package/dist/src/providers.d.ts.map +1 -1
  105. package/dist/src/python/pythonUtils.d.ts.map +1 -1
  106. package/dist/src/python/pythonUtils.js.map +1 -1
  107. package/dist/src/redteam/constants.d.ts +1 -1
  108. package/dist/src/redteam/constants.js +3 -3
  109. package/dist/src/redteam/constants.js.map +1 -1
  110. package/dist/src/redteam/extraction/entities.d.ts +1 -1
  111. package/dist/src/redteam/extraction/entities.d.ts.map +1 -1
  112. package/dist/src/redteam/extraction/purpose.d.ts +1 -1
  113. package/dist/src/redteam/extraction/purpose.d.ts.map +1 -1
  114. package/dist/src/redteam/extraction/util.d.ts +1 -1
  115. package/dist/src/redteam/extraction/util.d.ts.map +1 -1
  116. package/dist/src/redteam/graders/base.d.ts +1 -1
  117. package/dist/src/redteam/graders/base.d.ts.map +1 -1
  118. package/dist/src/redteam/graders/harmful.d.ts +1 -1
  119. package/dist/src/redteam/graders/harmful.d.ts.map +1 -1
  120. package/dist/src/redteam/graders/index.d.ts +1 -1
  121. package/dist/src/redteam/graders/index.d.ts.map +1 -1
  122. package/dist/src/redteam/index.d.ts +3 -3
  123. package/dist/src/redteam/index.d.ts.map +1 -1
  124. package/dist/src/redteam/index.js +20 -8
  125. package/dist/src/redteam/index.js.map +1 -1
  126. package/dist/src/redteam/plugins/base.d.ts +4 -4
  127. package/dist/src/redteam/plugins/base.d.ts.map +1 -1
  128. package/dist/src/redteam/plugins/base.js +1 -1
  129. package/dist/src/redteam/plugins/base.js.map +1 -1
  130. package/dist/src/redteam/plugins/competitors.d.ts +2 -2
  131. package/dist/src/redteam/plugins/competitors.d.ts.map +1 -1
  132. package/dist/src/redteam/plugins/competitors.js +2 -3
  133. package/dist/src/redteam/plugins/competitors.js.map +1 -1
  134. package/dist/src/redteam/plugins/contracts.d.ts +2 -2
  135. package/dist/src/redteam/plugins/contracts.d.ts.map +1 -1
  136. package/dist/src/redteam/plugins/contracts.js +2 -3
  137. package/dist/src/redteam/plugins/contracts.js.map +1 -1
  138. package/dist/src/redteam/plugins/debugAccess.d.ts +2 -2
  139. package/dist/src/redteam/plugins/debugAccess.d.ts.map +1 -1
  140. package/dist/src/redteam/plugins/debugAccess.js +2 -3
  141. package/dist/src/redteam/plugins/debugAccess.js.map +1 -1
  142. package/dist/src/redteam/plugins/excessiveAgency.d.ts +2 -2
  143. package/dist/src/redteam/plugins/excessiveAgency.d.ts.map +1 -1
  144. package/dist/src/redteam/plugins/excessiveAgency.js +2 -3
  145. package/dist/src/redteam/plugins/excessiveAgency.js.map +1 -1
  146. package/dist/src/redteam/plugins/hallucination.d.ts +2 -2
  147. package/dist/src/redteam/plugins/hallucination.d.ts.map +1 -1
  148. package/dist/src/redteam/plugins/hallucination.js +2 -3
  149. package/dist/src/redteam/plugins/hallucination.js.map +1 -1
  150. package/dist/src/redteam/plugins/harmful.d.ts.map +1 -1
  151. package/dist/src/redteam/plugins/harmful.js +3 -1
  152. package/dist/src/redteam/plugins/harmful.js.map +1 -1
  153. package/dist/src/redteam/plugins/hijacking.d.ts +2 -2
  154. package/dist/src/redteam/plugins/hijacking.d.ts.map +1 -1
  155. package/dist/src/redteam/plugins/hijacking.js +2 -3
  156. package/dist/src/redteam/plugins/hijacking.js.map +1 -1
  157. package/dist/src/redteam/plugins/imitation.d.ts +1 -1
  158. package/dist/src/redteam/plugins/imitation.d.ts.map +1 -1
  159. package/dist/src/redteam/plugins/imitation.js +2 -3
  160. package/dist/src/redteam/plugins/imitation.js.map +1 -1
  161. package/dist/src/redteam/plugins/overreliance.d.ts +2 -2
  162. package/dist/src/redteam/plugins/overreliance.d.ts.map +1 -1
  163. package/dist/src/redteam/plugins/overreliance.js +2 -3
  164. package/dist/src/redteam/plugins/overreliance.js.map +1 -1
  165. package/dist/src/redteam/plugins/politics.d.ts +2 -2
  166. package/dist/src/redteam/plugins/politics.d.ts.map +1 -1
  167. package/dist/src/redteam/plugins/politics.js +2 -3
  168. package/dist/src/redteam/plugins/politics.js.map +1 -1
  169. package/dist/src/redteam/plugins/rbac.d.ts +2 -2
  170. package/dist/src/redteam/plugins/rbac.d.ts.map +1 -1
  171. package/dist/src/redteam/plugins/rbac.js +2 -3
  172. package/dist/src/redteam/plugins/rbac.js.map +1 -1
  173. package/dist/src/redteam/plugins/shellInjection.d.ts +2 -2
  174. package/dist/src/redteam/plugins/shellInjection.d.ts.map +1 -1
  175. package/dist/src/redteam/plugins/shellInjection.js +2 -3
  176. package/dist/src/redteam/plugins/shellInjection.js.map +1 -1
  177. package/dist/src/redteam/plugins/sqlInjection.d.ts +2 -2
  178. package/dist/src/redteam/plugins/sqlInjection.d.ts.map +1 -1
  179. package/dist/src/redteam/plugins/sqlInjection.js +2 -3
  180. package/dist/src/redteam/plugins/sqlInjection.js.map +1 -1
  181. package/dist/src/redteam/strategies/injections.d.ts +1 -1
  182. package/dist/src/redteam/strategies/injections.d.ts.map +1 -1
  183. package/dist/src/redteam/strategies/iterative.d.ts +1 -1
  184. package/dist/src/redteam/strategies/iterative.d.ts.map +1 -1
  185. package/dist/src/testCases.d.ts.map +1 -1
  186. package/dist/src/{types.d.ts → types/index.d.ts} +1496 -6518
  187. package/dist/src/{types.d.ts.map → types/index.d.ts.map} +1 -1
  188. package/dist/src/{types.js → types/index.js} +38 -172
  189. package/dist/src/types/index.js.map +1 -0
  190. package/dist/src/types/prompts.d.ts +29 -0
  191. package/dist/src/types/prompts.d.ts.map +1 -0
  192. package/dist/src/types/prompts.js +3 -0
  193. package/dist/src/types/prompts.js.map +1 -0
  194. package/dist/src/types/providers.d.ts +126 -0
  195. package/dist/src/types/providers.d.ts.map +1 -0
  196. package/dist/src/types/providers.js +11 -0
  197. package/dist/src/types/providers.js.map +1 -0
  198. package/dist/src/types/redteam.d.ts +51 -0
  199. package/dist/src/types/redteam.d.ts.map +1 -0
  200. package/dist/src/types/redteam.js +3 -0
  201. package/dist/src/types/redteam.js.map +1 -0
  202. package/dist/src/types/shared.d.ts +8 -0
  203. package/dist/src/types/shared.d.ts.map +1 -0
  204. package/dist/src/types/shared.js +3 -0
  205. package/dist/src/types/shared.js.map +1 -0
  206. package/dist/src/util/index.d.ts +20 -96
  207. package/dist/src/util/index.d.ts.map +1 -1
  208. package/dist/src/util/index.js +15 -8
  209. package/dist/src/util/index.js.map +1 -1
  210. package/dist/src/util/templates.d.ts +1 -1
  211. package/dist/src/util/templates.d.ts.map +1 -1
  212. package/dist/src/util/transform.d.ts +1 -1
  213. package/dist/src/util/transform.d.ts.map +1 -1
  214. package/dist/src/validators/prompts.d.ts +60 -0
  215. package/dist/src/validators/prompts.d.ts.map +1 -0
  216. package/dist/src/validators/prompts.js +32 -0
  217. package/dist/src/validators/prompts.js.map +1 -0
  218. package/dist/src/validators/providers.d.ts +1145 -0
  219. package/dist/src/validators/providers.d.ts.map +1 -0
  220. package/dist/src/validators/providers.js +133 -0
  221. package/dist/src/validators/providers.js.map +1 -0
  222. package/dist/src/validators/redteam.d.ts +517 -0
  223. package/dist/src/validators/redteam.d.ts.map +1 -0
  224. package/dist/src/{redteam/types.js → validators/redteam.js} +31 -26
  225. package/dist/src/validators/redteam.js.map +1 -0
  226. package/dist/src/validators/shared.d.ts +19 -0
  227. package/dist/src/validators/shared.d.ts.map +1 -0
  228. package/dist/src/validators/shared.js +12 -0
  229. package/dist/src/validators/shared.js.map +1 -0
  230. package/dist/src/web/nextui/404/index.html +1 -1
  231. package/dist/src/web/nextui/404.html +1 -1
  232. package/dist/src/web/nextui/_next/static/chunks/949-e778f1d346946d00.js +1 -0
  233. package/dist/src/web/nextui/_next/static/chunks/app/eval/[id]/{page-3f5dd9507c5c905d.js → page-9029528e35960a91.js} +1 -1
  234. package/dist/src/web/nextui/_next/static/chunks/app/{layout-881a16d7fe67581c.js → layout-517c4420e852da35.js} +1 -1
  235. package/dist/src/web/nextui/_next/static/chunks/app/page-81e3b8fee37eaf67.js +1 -0
  236. package/dist/src/web/nextui/_next/static/chunks/app/report/{page-95d5e86c47bb3593.js → page-36e98231504ad757.js} +1 -1
  237. package/dist/src/web/nextui/_next/static/chunks/main-app-0dc90092cd7acaaf.js +1 -0
  238. package/dist/src/web/nextui/_next/static/chunks/{webpack-b0363d9598537efc.js → webpack-d3dba20c5c1ca02e.js} +1 -1
  239. package/dist/src/web/nextui/_next/static/css/32384cb640d2265e.css +1 -0
  240. package/dist/src/web/nextui/auth/login/index.html +1 -1
  241. package/dist/src/web/nextui/auth/login/index.txt +4 -4
  242. package/dist/src/web/nextui/auth/signup/index.html +1 -1
  243. package/dist/src/web/nextui/auth/signup/index.txt +4 -4
  244. package/dist/src/web/nextui/datasets/index.html +1 -1
  245. package/dist/src/web/nextui/datasets/index.txt +4 -4
  246. package/dist/src/web/nextui/eval/index.html +1 -1
  247. package/dist/src/web/nextui/eval/index.txt +5 -5
  248. package/dist/src/web/nextui/index.html +1 -1
  249. package/dist/src/web/nextui/index.txt +4 -4
  250. package/dist/src/web/nextui/progress/index.html +1 -1
  251. package/dist/src/web/nextui/progress/index.txt +4 -4
  252. package/dist/src/web/nextui/prompts/index.html +1 -1
  253. package/dist/src/web/nextui/prompts/index.txt +4 -4
  254. package/dist/src/web/nextui/report/index.html +1 -1
  255. package/dist/src/web/nextui/report/index.txt +7 -7
  256. package/dist/src/web/nextui/setup/index.html +1 -1
  257. package/dist/src/web/nextui/setup/index.txt +5 -5
  258. package/dist/src/web/server.d.ts.map +1 -1
  259. package/dist/src/web/server.js.map +1 -1
  260. package/dist/test/cache.test.js.map +1 -1
  261. package/dist/test/providers.bedrock.test.js.map +1 -1
  262. package/dist/test/providers.shared.test.d.ts +2 -0
  263. package/dist/test/providers.shared.test.d.ts.map +1 -0
  264. package/dist/test/providers.shared.test.js +101 -0
  265. package/dist/test/providers.shared.test.js.map +1 -0
  266. package/dist/test/redteam/plugins/base.test.js +2 -3
  267. package/dist/test/redteam/plugins/base.test.js.map +1 -1
  268. package/dist/test/redteam/validators.test.d.ts +2 -0
  269. package/dist/test/redteam/validators.test.d.ts.map +1 -0
  270. package/dist/test/redteam/{types.test.js → validators.test.js} +84 -29
  271. package/dist/test/redteam/validators.test.js.map +1 -0
  272. package/dist/test/utils.d.ts +1 -1
  273. package/dist/test/utils.d.ts.map +1 -1
  274. package/package.json +1 -1
  275. package/dist/src/redteam/types.d.ts +0 -215
  276. package/dist/src/redteam/types.d.ts.map +0 -1
  277. package/dist/src/redteam/types.js.map +0 -1
  278. package/dist/src/types.js.map +0 -1
  279. package/dist/src/web/nextui/_next/static/chunks/347-49b79b8c81884e85.js +0 -1
  280. package/dist/src/web/nextui/_next/static/chunks/app/page-7e8faab5d212f727.js +0 -1
  281. package/dist/src/web/nextui/_next/static/chunks/main-app-4398f7af7ad8387f.js +0 -1
  282. package/dist/src/web/nextui/_next/static/css/e17fdafd8599f69b.css +0 -1
  283. package/dist/test/redteam/types.test.d.ts +0 -2
  284. package/dist/test/redteam/types.test.d.ts.map +0 -1
  285. package/dist/test/redteam/types.test.js.map +0 -1
  286. /package/dist/src/web/nextui/_next/static/{rXJSLz3x5ieAU1Syln-RX → 9g6bpuNNR0WtmBBAmnqTc}/_buildManifest.js +0 -0
  287. /package/dist/src/web/nextui/_next/static/{rXJSLz3x5ieAU1Syln-RX → 9g6bpuNNR0WtmBBAmnqTc}/_ssgManifest.js +0 -0
  288. /package/dist/src/web/nextui/_next/static/chunks/{258-f5d5d6dbebb8ae22.js → 258-280986e9c8fae219.js} +0 -0
@@ -145,34 +145,11 @@ export declare const evals: import("drizzle-orm/sqlite-core").SQLiteTableWithCol
145
145
  columnType: "SQLiteTextJson";
146
146
  data: Partial<{
147
147
  prompts: string | Record<string, string> | (string | {
148
- label: string;
149
148
  raw: string;
149
+ label: string;
150
150
  function?: ((args_0: {
151
151
  vars: Record<string, any>;
152
- provider?: {
153
- id: (...args: unknown[]) => string;
154
- callApi: {
155
- (prompt: string, context?: import("../types").CallApiContextParams, options?: import("../types").CallApiOptionsParams): Promise<import("../types").ProviderResponse>;
156
- label?: string;
157
- };
158
- delay?: number | undefined;
159
- label?: string | undefined;
160
- transform?: string | undefined;
161
- callEmbeddingApi?: ((args_0: string, ...args_1: unknown[]) => Promise<{
162
- error?: string | undefined;
163
- embedding?: number[] | undefined;
164
- tokenUsage?: {
165
- cached?: number | undefined;
166
- completion?: number | undefined;
167
- prompt?: number | undefined;
168
- total?: number | undefined;
169
- } | undefined;
170
- }>) | undefined;
171
- callClassificationApi?: ((args_0: string, ...args_1: unknown[]) => Promise<{
172
- error?: string | undefined;
173
- classification?: Record<string, number> | undefined;
174
- }>) | undefined;
175
- } | undefined;
152
+ provider?: import("../types").ApiProvider | undefined;
176
153
  }, ...args_1: unknown[]) => Promise<any>) | undefined;
177
154
  id?: string | undefined;
178
155
  display?: string | undefined;
@@ -180,34 +157,11 @@ export declare const evals: import("drizzle-orm/sqlite-core").SQLiteTableWithCol
180
157
  providers: string | (((args_0: string, args_1: {
181
158
  vars: Record<string, string | {}>;
182
159
  prompt: {
183
- label: string;
184
160
  raw: string;
161
+ label: string;
185
162
  function?: ((args_0: {
186
163
  vars: Record<string, any>;
187
- provider?: {
188
- id: (...args: unknown[]) => string;
189
- callApi: {
190
- (prompt: string, context?: import("../types").CallApiContextParams, options?: import("../types").CallApiOptionsParams): Promise<import("../types").ProviderResponse>;
191
- label?: string;
192
- };
193
- delay?: number | undefined;
194
- label?: string | undefined;
195
- transform?: string | undefined;
196
- callEmbeddingApi?: ((args_0: string, ...args_1: unknown[]) => Promise<{
197
- error?: string | undefined;
198
- embedding?: number[] | undefined;
199
- tokenUsage?: {
200
- cached?: number | undefined;
201
- completion?: number | undefined;
202
- prompt?: number | undefined;
203
- total?: number | undefined;
204
- } | undefined;
205
- }>) | undefined;
206
- callClassificationApi?: ((args_0: string, ...args_1: unknown[]) => Promise<{
207
- error?: string | undefined;
208
- classification?: Record<string, number> | undefined;
209
- }>) | undefined;
210
- } | undefined;
164
+ provider?: import("../types").ApiProvider | undefined;
211
165
  }, ...args_1: unknown[]) => Promise<any>) | undefined;
212
166
  id?: string | undefined;
213
167
  display?: string | undefined;
@@ -219,30 +173,15 @@ export declare const evals: import("drizzle-orm/sqlite-core").SQLiteTableWithCol
219
173
  originalProvider?: any;
220
174
  } | undefined, args_2: {
221
175
  includeLogProbs?: boolean | undefined;
222
- } | undefined, ...args_3: unknown[]) => Promise<{
223
- output?: any;
224
- error?: string | undefined;
225
- cached?: boolean | undefined;
226
- tokenUsage?: {
227
- cached?: number | undefined;
228
- completion?: number | undefined;
229
- prompt?: number | undefined;
230
- total?: number | undefined;
231
- } | undefined;
232
- cost?: number | undefined;
233
- logProbs?: number[] | undefined;
234
- metadata?: import("zod").objectOutputType<{
235
- redteamFinalPrompt: import("zod").ZodOptional<import("zod").ZodString>;
236
- }, import("zod").ZodAny, "strip"> | undefined;
237
- }>) & {
176
+ } | undefined, ...args_3: unknown[]) => Promise<import("../types").ProviderResponse>) & {
238
177
  label?: string | undefined;
239
178
  }) | (string | {
240
179
  id?: string | undefined;
180
+ label?: string | undefined;
241
181
  config?: any;
242
182
  prompts?: string[] | undefined;
243
- delay?: number | undefined;
244
- label?: string | undefined;
245
183
  transform?: string | undefined;
184
+ delay?: number | undefined;
246
185
  env?: {
247
186
  ANTHROPIC_API_KEY?: string | undefined;
248
187
  BAM_API_KEY?: string | undefined;
@@ -279,34 +218,11 @@ export declare const evals: import("drizzle-orm/sqlite-core").SQLiteTableWithCol
279
218
  } | (((args_0: string, args_1: {
280
219
  vars: Record<string, string | {}>;
281
220
  prompt: {
282
- label: string;
283
221
  raw: string;
222
+ label: string;
284
223
  function?: ((args_0: {
285
224
  vars: Record<string, any>;
286
- provider?: {
287
- id: (...args: unknown[]) => string;
288
- callApi: {
289
- (prompt: string, context?: import("../types").CallApiContextParams, options?: import("../types").CallApiOptionsParams): Promise<import("../types").ProviderResponse>;
290
- label?: string;
291
- };
292
- delay?: number | undefined;
293
- label?: string | undefined;
294
- transform?: string | undefined;
295
- callEmbeddingApi?: ((args_0: string, ...args_1: unknown[]) => Promise<{
296
- error?: string | undefined;
297
- embedding?: number[] | undefined;
298
- tokenUsage?: {
299
- cached?: number | undefined;
300
- completion?: number | undefined;
301
- prompt?: number | undefined;
302
- total?: number | undefined;
303
- } | undefined;
304
- }>) | undefined;
305
- callClassificationApi?: ((args_0: string, ...args_1: unknown[]) => Promise<{
306
- error?: string | undefined;
307
- classification?: Record<string, number> | undefined;
308
- }>) | undefined;
309
- } | undefined;
225
+ provider?: import("../types").ApiProvider | undefined;
310
226
  }, ...args_1: unknown[]) => Promise<any>) | undefined;
311
227
  id?: string | undefined;
312
228
  display?: string | undefined;
@@ -318,30 +234,15 @@ export declare const evals: import("drizzle-orm/sqlite-core").SQLiteTableWithCol
318
234
  originalProvider?: any;
319
235
  } | undefined, args_2: {
320
236
  includeLogProbs?: boolean | undefined;
321
- } | undefined, ...args_3: unknown[]) => Promise<{
322
- output?: any;
323
- error?: string | undefined;
324
- cached?: boolean | undefined;
325
- tokenUsage?: {
326
- cached?: number | undefined;
327
- completion?: number | undefined;
328
- prompt?: number | undefined;
329
- total?: number | undefined;
330
- } | undefined;
331
- cost?: number | undefined;
332
- logProbs?: number[] | undefined;
333
- metadata?: import("zod").objectOutputType<{
334
- redteamFinalPrompt: import("zod").ZodOptional<import("zod").ZodString>;
335
- }, import("zod").ZodAny, "strip"> | undefined;
336
- }>) & {
237
+ } | undefined, ...args_3: unknown[]) => Promise<import("../types").ProviderResponse>) & {
337
238
  label?: string | undefined;
338
239
  }) | Record<string, {
339
240
  id?: string | undefined;
241
+ label?: string | undefined;
340
242
  config?: any;
341
243
  prompts?: string[] | undefined;
342
- delay?: number | undefined;
343
- label?: string | undefined;
344
244
  transform?: string | undefined;
245
+ delay?: number | undefined;
345
246
  env?: {
346
247
  ANTHROPIC_API_KEY?: string | undefined;
347
248
  BAM_API_KEY?: string | undefined;
@@ -401,13 +302,14 @@ export declare const evals: import("drizzle-orm/sqlite-core").SQLiteTableWithCol
401
302
  disableVarExpansion?: boolean | undefined;
402
303
  disableConversationVar?: boolean | undefined;
403
304
  }) | undefined;
305
+ vars?: Record<string, string | any[] | string[] | {}> | undefined;
404
306
  provider?: string | {
405
307
  id?: string | undefined;
308
+ label?: string | undefined;
406
309
  config?: any;
407
310
  prompts?: string[] | undefined;
408
- delay?: number | undefined;
409
- label?: string | undefined;
410
311
  transform?: string | undefined;
312
+ delay?: number | undefined;
411
313
  env?: {
412
314
  ANTHROPIC_API_KEY?: string | undefined;
413
315
  BAM_API_KEY?: string | undefined;
@@ -442,30 +344,14 @@ export declare const evals: import("drizzle-orm/sqlite-core").SQLiteTableWithCol
442
344
  CLOUDFLARE_ACCOUNT_ID?: string | undefined;
443
345
  } | undefined;
444
346
  } | {
347
+ callApi: import("../types").CallApiFunction;
445
348
  id: (...args: unknown[]) => string;
446
- callApi: {
447
- (prompt: string, context?: import("../types").CallApiContextParams, options?: import("../types").CallApiOptionsParams): Promise<import("../types").ProviderResponse>;
448
- label?: string;
449
- };
450
- delay?: number | undefined;
451
349
  label?: string | undefined;
452
350
  transform?: string | undefined;
453
- callEmbeddingApi?: ((args_0: string, ...args_1: unknown[]) => Promise<{
454
- error?: string | undefined;
455
- embedding?: number[] | undefined;
456
- tokenUsage?: {
457
- cached?: number | undefined;
458
- completion?: number | undefined;
459
- prompt?: number | undefined;
460
- total?: number | undefined;
461
- } | undefined;
462
- }>) | undefined;
463
- callClassificationApi?: ((args_0: string, ...args_1: unknown[]) => Promise<{
464
- error?: string | undefined;
465
- classification?: Record<string, number> | undefined;
466
- }>) | undefined;
351
+ delay?: number | undefined;
352
+ callEmbeddingApi?: ((args_0: string, ...args_1: unknown[]) => Promise<import("../types").ProviderEmbeddingResponse>) | undefined;
353
+ callClassificationApi?: ((args_0: string, ...args_1: unknown[]) => Promise<import("../types").ProviderClassificationResponse>) | undefined;
467
354
  } | undefined;
468
- vars?: Record<string, string | any[] | string[] | {}> | undefined;
469
355
  metadata?: Record<string, any> | undefined;
470
356
  description?: string | undefined;
471
357
  providerOutput?: string | {} | undefined;
@@ -563,13 +449,14 @@ export declare const evals: import("drizzle-orm/sqlite-core").SQLiteTableWithCol
563
449
  disableVarExpansion?: boolean | undefined;
564
450
  disableConversationVar?: boolean | undefined;
565
451
  }) | undefined;
452
+ vars?: Record<string, string | any[] | string[] | {}> | undefined;
566
453
  provider?: string | {
567
454
  id?: string | undefined;
455
+ label?: string | undefined;
568
456
  config?: any;
569
457
  prompts?: string[] | undefined;
570
- delay?: number | undefined;
571
- label?: string | undefined;
572
458
  transform?: string | undefined;
459
+ delay?: number | undefined;
573
460
  env?: {
574
461
  ANTHROPIC_API_KEY?: string | undefined;
575
462
  BAM_API_KEY?: string | undefined;
@@ -604,30 +491,14 @@ export declare const evals: import("drizzle-orm/sqlite-core").SQLiteTableWithCol
604
491
  CLOUDFLARE_ACCOUNT_ID?: string | undefined;
605
492
  } | undefined;
606
493
  } | {
494
+ callApi: import("../types").CallApiFunction;
607
495
  id: (...args: unknown[]) => string;
608
- callApi: {
609
- (prompt: string, context?: import("../types").CallApiContextParams, options?: import("../types").CallApiOptionsParams): Promise<import("../types").ProviderResponse>;
610
- label?: string;
611
- };
612
- delay?: number | undefined;
613
496
  label?: string | undefined;
614
497
  transform?: string | undefined;
615
- callEmbeddingApi?: ((args_0: string, ...args_1: unknown[]) => Promise<{
616
- error?: string | undefined;
617
- embedding?: number[] | undefined;
618
- tokenUsage?: {
619
- cached?: number | undefined;
620
- completion?: number | undefined;
621
- prompt?: number | undefined;
622
- total?: number | undefined;
623
- } | undefined;
624
- }>) | undefined;
625
- callClassificationApi?: ((args_0: string, ...args_1: unknown[]) => Promise<{
626
- error?: string | undefined;
627
- classification?: Record<string, number> | undefined;
628
- }>) | undefined;
498
+ delay?: number | undefined;
499
+ callEmbeddingApi?: ((args_0: string, ...args_1: unknown[]) => Promise<import("../types").ProviderEmbeddingResponse>) | undefined;
500
+ callClassificationApi?: ((args_0: string, ...args_1: unknown[]) => Promise<import("../types").ProviderClassificationResponse>) | undefined;
629
501
  } | undefined;
630
- vars?: Record<string, string | any[] | string[] | {}> | undefined;
631
502
  metadata?: Record<string, any> | undefined;
632
503
  description?: string | undefined;
633
504
  providerOutput?: string | {} | undefined;
@@ -689,13 +560,14 @@ export declare const evals: import("drizzle-orm/sqlite-core").SQLiteTableWithCol
689
560
  disableVarExpansion?: boolean | undefined;
690
561
  disableConversationVar?: boolean | undefined;
691
562
  }) | undefined;
563
+ vars?: Record<string, string | any[] | string[] | {}> | undefined;
692
564
  provider?: string | {
693
565
  id?: string | undefined;
566
+ label?: string | undefined;
694
567
  config?: any;
695
568
  prompts?: string[] | undefined;
696
- delay?: number | undefined;
697
- label?: string | undefined;
698
569
  transform?: string | undefined;
570
+ delay?: number | undefined;
699
571
  env?: {
700
572
  ANTHROPIC_API_KEY?: string | undefined;
701
573
  BAM_API_KEY?: string | undefined;
@@ -730,30 +602,14 @@ export declare const evals: import("drizzle-orm/sqlite-core").SQLiteTableWithCol
730
602
  CLOUDFLARE_ACCOUNT_ID?: string | undefined;
731
603
  } | undefined;
732
604
  } | {
605
+ callApi: import("../types").CallApiFunction;
733
606
  id: (...args: unknown[]) => string;
734
- callApi: {
735
- (prompt: string, context?: import("../types").CallApiContextParams, options?: import("../types").CallApiOptionsParams): Promise<import("../types").ProviderResponse>;
736
- label?: string;
737
- };
738
- delay?: number | undefined;
739
607
  label?: string | undefined;
740
608
  transform?: string | undefined;
741
- callEmbeddingApi?: ((args_0: string, ...args_1: unknown[]) => Promise<{
742
- error?: string | undefined;
743
- embedding?: number[] | undefined;
744
- tokenUsage?: {
745
- cached?: number | undefined;
746
- completion?: number | undefined;
747
- prompt?: number | undefined;
748
- total?: number | undefined;
749
- } | undefined;
750
- }>) | undefined;
751
- callClassificationApi?: ((args_0: string, ...args_1: unknown[]) => Promise<{
752
- error?: string | undefined;
753
- classification?: Record<string, number> | undefined;
754
- }>) | undefined;
609
+ delay?: number | undefined;
610
+ callEmbeddingApi?: ((args_0: string, ...args_1: unknown[]) => Promise<import("../types").ProviderEmbeddingResponse>) | undefined;
611
+ callClassificationApi?: ((args_0: string, ...args_1: unknown[]) => Promise<import("../types").ProviderClassificationResponse>) | undefined;
755
612
  } | undefined;
756
- vars?: Record<string, string | any[] | string[] | {}> | undefined;
757
613
  metadata?: Record<string, any> | undefined;
758
614
  description?: string | undefined;
759
615
  providerOutput?: string | {} | undefined;
@@ -817,13 +673,14 @@ export declare const evals: import("drizzle-orm/sqlite-core").SQLiteTableWithCol
817
673
  disableVarExpansion?: boolean | undefined;
818
674
  disableConversationVar?: boolean | undefined;
819
675
  }) | undefined;
676
+ vars?: Record<string, string | any[] | string[] | {}> | undefined;
820
677
  provider?: string | {
821
678
  id?: string | undefined;
679
+ label?: string | undefined;
822
680
  config?: any;
823
681
  prompts?: string[] | undefined;
824
- delay?: number | undefined;
825
- label?: string | undefined;
826
682
  transform?: string | undefined;
683
+ delay?: number | undefined;
827
684
  env?: {
828
685
  ANTHROPIC_API_KEY?: string | undefined;
829
686
  BAM_API_KEY?: string | undefined;
@@ -858,30 +715,14 @@ export declare const evals: import("drizzle-orm/sqlite-core").SQLiteTableWithCol
858
715
  CLOUDFLARE_ACCOUNT_ID?: string | undefined;
859
716
  } | undefined;
860
717
  } | {
718
+ callApi: import("../types").CallApiFunction;
861
719
  id: (...args: unknown[]) => string;
862
- callApi: {
863
- (prompt: string, context?: import("../types").CallApiContextParams, options?: import("../types").CallApiOptionsParams): Promise<import("../types").ProviderResponse>;
864
- label?: string;
865
- };
866
- delay?: number | undefined;
867
720
  label?: string | undefined;
868
721
  transform?: string | undefined;
869
- callEmbeddingApi?: ((args_0: string, ...args_1: unknown[]) => Promise<{
870
- error?: string | undefined;
871
- embedding?: number[] | undefined;
872
- tokenUsage?: {
873
- cached?: number | undefined;
874
- completion?: number | undefined;
875
- prompt?: number | undefined;
876
- total?: number | undefined;
877
- } | undefined;
878
- }>) | undefined;
879
- callClassificationApi?: ((args_0: string, ...args_1: unknown[]) => Promise<{
880
- error?: string | undefined;
881
- classification?: Record<string, number> | undefined;
882
- }>) | undefined;
722
+ delay?: number | undefined;
723
+ callEmbeddingApi?: ((args_0: string, ...args_1: unknown[]) => Promise<import("../types").ProviderEmbeddingResponse>) | undefined;
724
+ callClassificationApi?: ((args_0: string, ...args_1: unknown[]) => Promise<import("../types").ProviderClassificationResponse>) | undefined;
883
725
  } | undefined;
884
- vars?: Record<string, string | any[] | string[] | {}> | undefined;
885
726
  metadata?: Record<string, any> | undefined;
886
727
  providerOutput?: string | {} | undefined;
887
728
  assert?: ({
@@ -922,17 +763,7 @@ export declare const evals: import("drizzle-orm/sqlite-core").SQLiteTableWithCol
922
763
  value: string | ((args_0: Record<string, number>, args_1: import("../types").RunEvalOptions, ...args_2: unknown[]) => number);
923
764
  name: string;
924
765
  }[] | undefined;
925
- redteam?: {
926
- plugins: any[];
927
- strategies: ({
928
- id: string;
929
- } | {
930
- id: string;
931
- })[];
932
- provider?: string | undefined;
933
- injectVar?: string | undefined;
934
- purpose?: string | undefined;
935
- } | undefined;
766
+ redteam?: import("../types").RedteamConfig | undefined;
936
767
  outputPath?: string | string[] | undefined;
937
768
  sharing?: boolean | {
938
769
  apiBaseUrl?: string | undefined;
@@ -940,9 +771,9 @@ export declare const evals: import("drizzle-orm/sqlite-core").SQLiteTableWithCol
940
771
  } | undefined;
941
772
  evaluateOptions?: {
942
773
  repeat?: number | undefined;
943
- cache?: boolean | undefined;
944
- maxConcurrency?: number | undefined;
945
774
  delay?: number | undefined;
775
+ maxConcurrency?: number | undefined;
776
+ cache?: boolean | undefined;
946
777
  interactiveProviders?: boolean | undefined;
947
778
  generateSuggestions?: boolean | undefined;
948
779
  showProgressBar?: boolean | undefined;
@@ -951,22 +782,21 @@ export declare const evals: import("drizzle-orm/sqlite-core").SQLiteTableWithCol
951
782
  } | undefined;
952
783
  commandLineOptions?: {
953
784
  repeat?: string | undefined;
954
- cache?: boolean | undefined;
785
+ vars?: string | undefined;
955
786
  config?: string[] | undefined;
956
- envFile?: string | undefined;
957
- output?: string[] | undefined;
958
- write?: boolean | undefined;
959
787
  prompts?: string[] | undefined;
788
+ delay?: string | undefined;
789
+ output?: string[] | undefined;
960
790
  providers?: string[] | undefined;
961
791
  maxConcurrency?: string | undefined;
962
- delay?: string | undefined;
963
- vars?: string | undefined;
964
792
  tests?: string | undefined;
965
793
  assertions?: string | undefined;
966
794
  modelOutputs?: string | undefined;
967
795
  verbose?: boolean | undefined;
968
796
  grader?: string | undefined;
969
797
  tableCellMaxLength?: string | undefined;
798
+ write?: boolean | undefined;
799
+ cache?: boolean | undefined;
970
800
  table?: boolean | undefined;
971
801
  share?: boolean | undefined;
972
802
  progressBar?: boolean | undefined;
@@ -980,6 +810,7 @@ export declare const evals: import("drizzle-orm/sqlite-core").SQLiteTableWithCol
980
810
  generateSuggestions?: boolean | undefined;
981
811
  promptPrefix?: string | undefined;
982
812
  promptSuffix?: string | undefined;
813
+ envFile?: string | undefined;
983
814
  } | undefined;
984
815
  }>;
985
816
  driverParam: string;
@@ -1087,13 +918,14 @@ export declare const datasets: import("drizzle-orm/sqlite-core").SQLiteTableWith
1087
918
  disableVarExpansion?: boolean | undefined;
1088
919
  disableConversationVar?: boolean | undefined;
1089
920
  }) | undefined;
921
+ vars?: Record<string, string | any[] | string[] | {}> | undefined;
1090
922
  provider?: string | {
1091
923
  id?: string | undefined;
924
+ label?: string | undefined;
1092
925
  config?: any;
1093
926
  prompts?: string[] | undefined;
1094
- delay?: number | undefined;
1095
- label?: string | undefined;
1096
927
  transform?: string | undefined;
928
+ delay?: number | undefined;
1097
929
  env?: {
1098
930
  ANTHROPIC_API_KEY?: string | undefined;
1099
931
  BAM_API_KEY?: string | undefined;
@@ -1128,30 +960,14 @@ export declare const datasets: import("drizzle-orm/sqlite-core").SQLiteTableWith
1128
960
  CLOUDFLARE_ACCOUNT_ID?: string | undefined;
1129
961
  } | undefined;
1130
962
  } | {
963
+ callApi: import("../types").CallApiFunction;
1131
964
  id: (...args: unknown[]) => string;
1132
- callApi: {
1133
- (prompt: string, context?: import("../types").CallApiContextParams, options?: import("../types").CallApiOptionsParams): Promise<import("../types").ProviderResponse>;
1134
- label?: string;
1135
- };
1136
- delay?: number | undefined;
1137
965
  label?: string | undefined;
1138
966
  transform?: string | undefined;
1139
- callEmbeddingApi?: ((args_0: string, ...args_1: unknown[]) => Promise<{
1140
- error?: string | undefined;
1141
- embedding?: number[] | undefined;
1142
- tokenUsage?: {
1143
- cached?: number | undefined;
1144
- completion?: number | undefined;
1145
- prompt?: number | undefined;
1146
- total?: number | undefined;
1147
- } | undefined;
1148
- }>) | undefined;
1149
- callClassificationApi?: ((args_0: string, ...args_1: unknown[]) => Promise<{
1150
- error?: string | undefined;
1151
- classification?: Record<string, number> | undefined;
1152
- }>) | undefined;
967
+ delay?: number | undefined;
968
+ callEmbeddingApi?: ((args_0: string, ...args_1: unknown[]) => Promise<import("../types").ProviderEmbeddingResponse>) | undefined;
969
+ callClassificationApi?: ((args_0: string, ...args_1: unknown[]) => Promise<import("../types").ProviderClassificationResponse>) | undefined;
1153
970
  } | undefined;
1154
- vars?: Record<string, string | any[] | string[] | {}> | undefined;
1155
971
  metadata?: Record<string, any> | undefined;
1156
972
  description?: string | undefined;
1157
973
  providerOutput?: string | {} | undefined;
@@ -1 +1 @@
1
- {"version":3,"file":"operations.d.ts","sourceRoot":"","sources":["../../../src/database/operations.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,eAAe,EAAiB,MAAM,UAAU,CAAC;AAI/D,eAAO,MAAM,OAAO;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAMlB,CAAC;AAEH,eAAO,MAAM,KAAK;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAShB,CAAC;AAEH,eAAO,MAAM,cAAc;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAe1B,CAAC;AAEF,eAAO,MAAM,gBAAgB;;EAE1B,CAAC;AAIJ,eAAO,MAAM,QAAQ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAMnB,CAAC;AAEH,eAAO,MAAM,eAAe;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAe3B,CAAC;AAEF,eAAO,MAAM,iBAAiB;;EAE3B,CAAC;AAIJ,eAAO,MAAM,cAAc;;;EAGxB,CAAC;AAEJ,eAAO,MAAM,uBAAuB;;;EASjC,CAAC;AAEJ,eAAO,MAAM,wBAAwB;;;EASlC,CAAC"}
1
+ {"version":3,"file":"operations.d.ts","sourceRoot":"","sources":["../../../src/database/operations.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,eAAe,EAAiB,MAAM,UAAU,CAAC;AAI/D,eAAO,MAAM,OAAO;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAMlB,CAAC;AAEH,eAAO,MAAM,KAAK;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAShB,CAAC;AAEH,eAAO,MAAM,cAAc;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAe1B,CAAC;AAEF,eAAO,MAAM,gBAAgB;;EAE1B,CAAC;AAIJ,eAAO,MAAM,QAAQ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAMnB,CAAC;AAEH,eAAO,MAAM,eAAe;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAe3B,CAAC;AAEF,eAAO,MAAM,iBAAiB;;EAE3B,CAAC;AAIJ,eAAO,MAAM,cAAc;;;EAGxB,CAAC;AAEJ,eAAO,MAAM,uBAAuB;;;EASjC,CAAC;AAEJ,eAAO,MAAM,wBAAwB;;;EASlC,CAAC"}
@@ -1,4 +1,4 @@
1
- import { EvaluateSummary, UnifiedConfig } from '../types';
1
+ import type { EvaluateSummary, UnifiedConfig } from '../types';
2
2
  export interface ResultsFile {
3
3
  version: number;
4
4
  createdAt: string;
@@ -1 +1 @@
1
- {"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/database/types.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAE1D,MAAM,WAAW,WAAW;IAC1B,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,OAAO,EAAE,eAAe,CAAC;IACzB,MAAM,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;IAC/B,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,gBAAgB;IAC/B,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,IAAI,CAAC;IACX,MAAM,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;IAC/B,OAAO,EAAE,eAAe,CAAC;IACzB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB"}
1
+ {"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/database/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAE/D,MAAM,WAAW,WAAW;IAC1B,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,OAAO,EAAE,eAAe,CAAC;IACzB,MAAM,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;IAC/B,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,gBAAgB;IAC/B,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,IAAI,CAAC;IACX,MAAM,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;IAC/B,OAAO,EAAE,eAAe,CAAC;IACzB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB"}
@@ -1,4 +1,4 @@
1
- import { GlobalConfig } from './configTypes';
1
+ import type { GlobalConfig } from './configTypes';
2
2
  export declare function resetGlobalConfig(): void;
3
3
  export declare function readGlobalConfig(): GlobalConfig;
4
4
  export declare function writeGlobalConfig(config: GlobalConfig): void;
@@ -1 +1 @@
1
- {"version":3,"file":"globalConfig.d.ts","sourceRoot":"","sources":["../../src/globalConfig.ts"],"names":[],"mappings":"AAOA,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAK7C,wBAAgB,iBAAiB,IAAI,IAAI,CAExC;AAED,wBAAgB,gBAAgB,IAAI,YAAY,CAiB/C;AAED,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,YAAY,GAAG,IAAI,CAK5D;AAED;;;GAGG;AACH,wBAAgB,wBAAwB,CAAC,aAAa,EAAE,OAAO,CAAC,YAAY,CAAC,GAAG,IAAI,CAcnF;AAED,wBAAgB,mBAAmB,IAAI,OAAO,CAa7C"}
1
+ {"version":3,"file":"globalConfig.d.ts","sourceRoot":"","sources":["../../src/globalConfig.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAKlD,wBAAgB,iBAAiB,IAAI,IAAI,CAExC;AAED,wBAAgB,gBAAgB,IAAI,YAAY,CAiB/C;AAED,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,YAAY,GAAG,IAAI,CAK5D;AAED;;;GAGG;AACH,wBAAgB,wBAAwB,CAAC,aAAa,EAAE,OAAO,CAAC,YAAY,CAAC,GAAG,IAAI,CAcnF;AAED,wBAAgB,mBAAmB,IAAI,OAAO,CAa7C"}
@@ -1 +1 @@
1
- {"version":3,"file":"onboarding.d.ts","sourceRoot":"","sources":["../../src/onboarding.ts"],"names":[],"mappings":"AAQA,eAAO,MAAM,eAAe,6hFAmF3B,CAAC;AAEF,eAAO,MAAM,eAAe,muCA8B3B,CAAC;AAEF,eAAO,MAAM,mBAAmB,0/BAsC/B,CAAC;AAEF,eAAO,MAAM,aAAa,gTAQzB,CAAC;AAEF,eAAO,MAAM,UAAU,upBAgBtB,CAAC;AAEF,eAAO,MAAM,cAAc,moBAkB1B,CAAC;AAEF,eAAO,MAAM,cAAc,0QAU1B,CAAC;AAEF,wBAAsB,gBAAgB,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,EAAE,WAAW,GAAE,OAAc;;;;;GAqO3F"}
1
+ {"version":3,"file":"onboarding.d.ts","sourceRoot":"","sources":["../../src/onboarding.ts"],"names":[],"mappings":"AASA,eAAO,MAAM,eAAe,6hFAmF3B,CAAC;AAEF,eAAO,MAAM,eAAe,muCA8B3B,CAAC;AAEF,eAAO,MAAM,mBAAmB,0/BAsC/B,CAAC;AAEF,eAAO,MAAM,aAAa,gTAQzB,CAAC;AAEF,eAAO,MAAM,UAAU,upBAgBtB,CAAC;AAEF,eAAO,MAAM,cAAc,moBAkB1B,CAAC;AAEF,eAAO,MAAM,cAAc,0QAU1B,CAAC;AAEF,wBAAsB,gBAAgB,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,EAAE,WAAW,GAAE,OAAc;;;;;GAiP3F"}
@@ -10,6 +10,7 @@ const rawlist_1 = __importDefault(require("@inquirer/rawlist"));
10
10
  const chalk_1 = __importDefault(require("chalk"));
11
11
  const fs_1 = __importDefault(require("fs"));
12
12
  const path_1 = __importDefault(require("path"));
13
+ const redteam_1 = require("./commands/redteam");
13
14
  const logger_1 = __importDefault(require("./logger"));
14
15
  const templates_1 = require("./util/templates");
15
16
  exports.CONFIG_TEMPLATE = `# Learn more about building a configuration: https://promptfoo.dev/docs/configuration/guide
@@ -251,8 +252,18 @@ async function createDummyFiles(directory, interactive = true) {
251
252
  { name: 'Improve prompt and model performance', value: 'compare' },
252
253
  { name: 'Improve RAG performance', value: 'rag' },
253
254
  { name: 'Improve agent/chain of thought performance', value: 'agent' },
255
+ { name: 'Run a red team evaluation', value: 'redteam' },
254
256
  ],
255
257
  });
258
+ if (action === 'redteam') {
259
+ await (0, redteam_1.redteamInit)(directory || '.');
260
+ return {
261
+ numPrompts: 0,
262
+ providerPrefixes: [],
263
+ action: 'redteam',
264
+ language: 'not_applicable',
265
+ };
266
+ }
256
267
  language = 'not_sure';
257
268
  if (action === 'rag' || action === 'agent') {
258
269
  language = await (0, rawlist_1.default)({
@@ -1 +1 @@
1
- {"version":3,"file":"onboarding.js","sourceRoot":"","sources":["../../src/onboarding.ts"],"names":[],"mappings":";;;;;;AAiOA,4CAqOC;AAtcD,kEAA0C;AAC1C,gEAAwC;AACxC,kDAA0B;AAC1B,4CAAoB;AACpB,gDAAwB;AACxB,sDAA8B;AAC9B,gDAAqD;AAExC,QAAA,eAAe,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAmF9B,CAAC;AAEW,QAAA,eAAe,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA8B9B,CAAC;AAEW,QAAA,mBAAmB,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAsClC,CAAC;AAEW,QAAA,aAAa,GAAG;;;;;;;;CAQ5B,CAAC;AAEW,QAAA,UAAU,GAAG;;;;;;;;;;;;;;;;CAgBzB,CAAC;AAEW,QAAA,cAAc,GAAG;;;;;;;;;;;;;;;;;;CAkB7B,CAAC;AAEW,QAAA,cAAc,GAAG;;;;;;;;;;CAU7B,CAAC;AAEK,KAAK,UAAU,gBAAgB,CAAC,SAAwB,EAAE,cAAuB,IAAI;IAC1F,IAAI,SAAS,EAAE,CAAC;QACd,yCAAyC;QACzC,IAAI,CAAC,YAAE,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;YAC9B,YAAE,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC;QAC1B,CAAC;IACH,CAAC;IAED,IAAI,SAAS,EAAE,CAAC;QACd,IAAI,CAAC,YAAE,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;YAC9B,gBAAM,CAAC,IAAI,CAAC,sBAAsB,SAAS,MAAM,CAAC,CAAC;YACnD,YAAE,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC;QAC1B,CAAC;IACH,CAAC;SAAM,CAAC;QACN,SAAS,GAAG,GAAG,CAAC;IAClB,CAAC;IAED,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,MAAM,SAAS,GAAwB,EAAE,CAAC;IAC1C,IAAI,MAAc,CAAC;IACnB,IAAI,QAAgB,CAAC;IACrB,IAAI,WAAW,EAAE,CAAC;QAChB,kBAAkB;QAClB,MAAM,GAAG,MAAM,IAAA,iBAAO,EAAC;YACrB,OAAO,EAAE,4BAA4B;YACrC,OAAO,EAAE;gBACP,EAAE,IAAI,EAAE,cAAc,EAAE,KAAK,EAAE,SAAS,EAAE;gBAC1C,EAAE,IAAI,EAAE,sCAAsC,EAAE,KAAK,EAAE,SAAS,EAAE;gBAClE,EAAE,IAAI,EAAE,yBAAyB,EAAE,KAAK,EAAE,KAAK,EAAE;gBACjD,EAAE,IAAI,EAAE,4CAA4C,EAAE,KAAK,EAAE,OAAO,EAAE;aACvE;SACF,CAAC,CAAC;QACH,QAAQ,GAAG,UAAU,CAAC;QACtB,IAAI,MAAM,KAAK,KAAK,IAAI,MAAM,KAAK,OAAO,EAAE,CAAC;YAC3C,QAAQ,GAAG,MAAM,IAAA,iBAAO,EAAC;gBACvB,OAAO,EAAE,0DAA0D;gBACnE,OAAO,EAAE;oBACP,EAAE,IAAI,EAAE,cAAc,EAAE,KAAK,EAAE,UAAU,EAAE;oBAC3C,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE;oBACnC,EAAE,IAAI,EAAE,YAAY,EAAE,KAAK,EAAE,YAAY,EAAE;iBAC5C;aACF,CAAC,CAAC;QACL,CAAC;QAED,MAAM,OAAO,GAAmD;YAC9D,EAAE,IAAI,EAAE,cAAc,EAAE,KAAK,EAAE,CAAC,oBAAoB,EAAE,eAAe,CAAC,EAAE;YACxE;gBACE,IAAI,EAAE,4CAA4C;gBAClD,KAAK,EACH,MAAM,KAAK,OAAO;oBAChB,CAAC,CAAC;wBACE;4BACE,EAAE,EAAE,eAAe;4BACnB,MAAM,EAAE;gCACN,KAAK,EAAE;oCACL;wCACE,IAAI,EAAE,UAAU;wCAChB,QAAQ,EAAE;4CACR,IAAI,EAAE,qBAAqB;4CAC3B,WAAW,EAAE,6CAA6C;4CAC1D,UAAU,EAAE;gDACV,IAAI,EAAE,QAAQ;gDACd,UAAU,EAAE;oDACV,QAAQ,EAAE;wDACR,IAAI,EAAE,QAAQ;wDACd,WAAW,EAAE,4CAA4C;qDAC1D;iDACF;gDACD,QAAQ,EAAE,CAAC,UAAU,CAAC;6CACvB;yCACF;qCACF;iCACF;6BACF;yBACF;qBACF;oBACH,CAAC,CAAC,CAAC,oBAAoB,EAAE,eAAe,CAAC;aAC9C;YACD;gBACE,IAAI,EAAE,6CAA6C;gBACnD,KAAK,EAAE;oBACL,+CAA+C;oBAC/C,2CAA2C;iBAC5C;aACF;YACD;gBACE,IAAI,EAAE,sCAAsC;gBAC5C,KAAK,EAAE;oBACL,iEAAiE;oBACjE,8DAA8D;oBAC9D,gDAAgD;iBACjD;aACF;YACD;gBACE,IAAI,EAAE,qBAAqB;gBAC3B,KAAK,EAAE,CAAC,oBAAoB,CAAC;aAC9B;YACD;gBACE,IAAI,EAAE,yBAAyB;gBAC/B,KAAK,EAAE,CAAC,oBAAoB,CAAC;aAC9B;YACD;gBACE,IAAI,EAAE,kBAAkB;gBACxB,KAAK,EAAE,CAAC,kBAAkB,CAAC;aAC5B;YACD;gBACE,IAAI,EAAE,eAAe;gBACrB,KAAK,EAAE,CAAC,kCAAkC,CAAC;aAC5C;YACD;gBACE,IAAI,EAAE,yCAAyC;gBAC/C,KAAK,EAAE;oBACL,gDAAgD;oBAChD,+CAA+C;iBAChD;aACF;YACD;gBACE,IAAI,EAAE,qCAAqC;gBAC3C,KAAK,EAAE,CAAC,kBAAkB,EAAE,uBAAuB,CAAC;aACrD;YACD,EAAE,IAAI,EAAE,wCAAwC,EAAE,KAAK,EAAE,CAAC,mBAAmB,CAAC,EAAE;YAChF;gBACE,IAAI,EAAE,gCAAgC;gBACtC,KAAK,EAAE,CAAC,oBAAoB,EAAE,2BAA2B,CAAC;aAC3D;SACF,CAAC;QACF,MAAM,eAAe,GAAwB,MAAM,IAAA,kBAAQ,EAAC;YAC1D,OAAO,EAAE,oEAAoE;YAC7E,OAAO;SACR,CAAC,CAAC;QACH,IAAI,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC/B,MAAM,aAAa,GAAG,eAAe,CAAC,IAAI,EAAE,CAAC;YAC7C,IAAI,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC7B,SAAS,CAAC,IAAI,CACZ,GAAG,eAAe,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CACjF,CAAC;YACJ,CAAC;iBAAM,CAAC;gBACN,SAAS,CAAC,IAAI,CAAC,GAAG,aAAa,CAAC,CAAC;YACnC,CAAC;YAED,IACE,aAAa,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC,EAC1F,CAAC;gBACD,YAAE,CAAC,aAAa,CAAC,cAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,aAAa,CAAC,EAAE,2BAAmB,CAAC,CAAC;gBAC1F,gBAAM,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;YACrC,CAAC;YACD,IACE,aAAa,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,EACxF,CAAC;gBACD,YAAE,CAAC,aAAa,CAAC,cAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,aAAa,CAAC,EAAE,qBAAa,CAAC,CAAC;gBACpF,gBAAM,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;YACrC,CAAC;YACD,IACE,aAAa,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC,EAC1F,CAAC;gBACD,YAAE,CAAC,aAAa,CAAC,cAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,aAAa,CAAC,EAAE,uBAAe,CAAC,CAAC;gBACtF,gBAAM,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;YACrC,CAAC;QACH,CAAC;aAAM,CAAC;YACN,SAAS,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;YACrC,SAAS,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;QAClC,CAAC;QAED,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;YACzB,OAAO,CAAC,IAAI,CAAC,+BAA+B,CAAC,CAAC;YAC9C,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACzB,OAAO,CAAC,IAAI,CAAC,8CAA8C,CAAC,CAAC;YAC/D,CAAC;QACH,CAAC;aAAM,IAAI,MAAM,KAAK,KAAK,EAAE,CAAC;YAC5B,OAAO,CAAC,IAAI,CACV,6FAA6F,CAC9F,CAAC;QACJ,CAAC;aAAM,IAAI,MAAM,KAAK,OAAO,EAAE,CAAC;YAC9B,OAAO,CAAC,IAAI,CAAC,gDAAgD,CAAC,CAAC;QACjE,CAAC;QAED,IAAI,MAAM,KAAK,KAAK,IAAI,MAAM,KAAK,OAAO,EAAE,CAAC;YAC3C,IAAI,QAAQ,KAAK,YAAY,EAAE,CAAC;gBAC9B,YAAE,CAAC,aAAa,CAAC,cAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,YAAY,CAAC,EAAE,sBAAc,CAAC,CAAC;gBACpF,gBAAM,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;YACpC,CAAC;iBAAM,CAAC;gBACN,YAAE,CAAC,aAAa,CAAC,cAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,YAAY,CAAC,EAAE,kBAAU,CAAC,CAAC;gBAChF,gBAAM,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;YACpC,CAAC;QACH,CAAC;IACH,CAAC;SAAM,CAAC;QACN,MAAM,GAAG,SAAS,CAAC;QACnB,QAAQ,GAAG,UAAU,CAAC;QACtB,OAAO,CAAC,IAAI,CAAC,+BAA+B,CAAC,CAAC;QAC9C,OAAO,CAAC,IAAI,CAAC,8CAA8C,CAAC,CAAC;QAC7D,SAAS,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;QACrC,SAAS,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;IAClC,CAAC;IAED,MAAM,QAAQ,GAAG,IAAA,6BAAiB,GAAE,CAAC;IACrC,MAAM,MAAM,GAAG,QAAQ,CAAC,YAAY,CAAC,uBAAe,EAAE;QACpD,OAAO;QACP,SAAS;QACT,IAAI,EAAE,MAAM;QACZ,QAAQ;KACT,CAAC,CAAC;IACH,YAAE,CAAC,aAAa,CAAC,cAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,sBAAsB,CAAC,EAAE,MAAM,CAAC,CAAC;IACtF,YAAE,CAAC,aAAa,CAAC,cAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,WAAW,CAAC,EAAE,sBAAc,CAAC,CAAC;IAEnF,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,YAAY,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAC;IACxD,MAAM,UAAU,GAAG,KAAK,CAAC,CAAC,CAAC,2BAA2B,CAAC,CAAC,CAAC,gBAAgB,CAAC;IAC1E,IAAI,SAAS,KAAK,GAAG,EAAE,CAAC;QACtB,gBAAM,CAAC,IAAI,CACT,eAAK,CAAC,KAAK,CACT,uCAAuC,eAAK,CAAC,IAAI,CAAC,UAAU,CAAC,oBAAoB,CAClF,CACF,CAAC;IACJ,CAAC;SAAM,CAAC;QACN,gBAAM,CAAC,IAAI,CAAC,qCAAqC,SAAS,EAAE,CAAC,CAAC;QAC9D,gBAAM,CAAC,IAAI,CACT,eAAK,CAAC,KAAK,CACT,SAAS,eAAK,CAAC,IAAI,CAAC,MAAM,SAAS,EAAE,CAAC,iBAAiB,eAAK,CAAC,IAAI,CAC/D,UAAU,CACX,oBAAoB,CACtB,CACF,CAAC;IACJ,CAAC;IAED,OAAO;QACL,UAAU,EAAE,OAAO,CAAC,MAAM;QAC1B,gBAAgB,EAAE,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;QAC7F,MAAM;QACN,QAAQ;KACT,CAAC;AACJ,CAAC"}
1
+ {"version":3,"file":"onboarding.js","sourceRoot":"","sources":["../../src/onboarding.ts"],"names":[],"mappings":";;;;;;AAkOA,4CAiPC;AAndD,kEAA0C;AAC1C,gEAAwC;AACxC,kDAA0B;AAC1B,4CAAoB;AACpB,gDAAwB;AACxB,gDAAiD;AACjD,sDAA8B;AAC9B,gDAAqD;AAExC,QAAA,eAAe,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAmF9B,CAAC;AAEW,QAAA,eAAe,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA8B9B,CAAC;AAEW,QAAA,mBAAmB,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAsClC,CAAC;AAEW,QAAA,aAAa,GAAG;;;;;;;;CAQ5B,CAAC;AAEW,QAAA,UAAU,GAAG;;;;;;;;;;;;;;;;CAgBzB,CAAC;AAEW,QAAA,cAAc,GAAG;;;;;;;;;;;;;;;;;;CAkB7B,CAAC;AAEW,QAAA,cAAc,GAAG;;;;;;;;;;CAU7B,CAAC;AAEK,KAAK,UAAU,gBAAgB,CAAC,SAAwB,EAAE,cAAuB,IAAI;IAC1F,IAAI,SAAS,EAAE,CAAC;QACd,yCAAyC;QACzC,IAAI,CAAC,YAAE,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;YAC9B,YAAE,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC;QAC1B,CAAC;IACH,CAAC;IAED,IAAI,SAAS,EAAE,CAAC;QACd,IAAI,CAAC,YAAE,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;YAC9B,gBAAM,CAAC,IAAI,CAAC,sBAAsB,SAAS,MAAM,CAAC,CAAC;YACnD,YAAE,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC;QAC1B,CAAC;IACH,CAAC;SAAM,CAAC;QACN,SAAS,GAAG,GAAG,CAAC;IAClB,CAAC;IAED,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,MAAM,SAAS,GAAwB,EAAE,CAAC;IAC1C,IAAI,MAAc,CAAC;IACnB,IAAI,QAAgB,CAAC;IACrB,IAAI,WAAW,EAAE,CAAC;QAChB,kBAAkB;QAClB,MAAM,GAAG,MAAM,IAAA,iBAAO,EAAC;YACrB,OAAO,EAAE,4BAA4B;YACrC,OAAO,EAAE;gBACP,EAAE,IAAI,EAAE,cAAc,EAAE,KAAK,EAAE,SAAS,EAAE;gBAC1C,EAAE,IAAI,EAAE,sCAAsC,EAAE,KAAK,EAAE,SAAS,EAAE;gBAClE,EAAE,IAAI,EAAE,yBAAyB,EAAE,KAAK,EAAE,KAAK,EAAE;gBACjD,EAAE,IAAI,EAAE,4CAA4C,EAAE,KAAK,EAAE,OAAO,EAAE;gBACtE,EAAE,IAAI,EAAE,2BAA2B,EAAE,KAAK,EAAE,SAAS,EAAE;aACxD;SACF,CAAC,CAAC;QAEH,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;YACzB,MAAM,IAAA,qBAAW,EAAC,SAAS,IAAI,GAAG,CAAC,CAAC;YACpC,OAAO;gBACL,UAAU,EAAE,CAAC;gBACb,gBAAgB,EAAE,EAAE;gBACpB,MAAM,EAAE,SAAS;gBACjB,QAAQ,EAAE,gBAAgB;aAC3B,CAAC;QACJ,CAAC;QAED,QAAQ,GAAG,UAAU,CAAC;QACtB,IAAI,MAAM,KAAK,KAAK,IAAI,MAAM,KAAK,OAAO,EAAE,CAAC;YAC3C,QAAQ,GAAG,MAAM,IAAA,iBAAO,EAAC;gBACvB,OAAO,EAAE,0DAA0D;gBACnE,OAAO,EAAE;oBACP,EAAE,IAAI,EAAE,cAAc,EAAE,KAAK,EAAE,UAAU,EAAE;oBAC3C,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE;oBACnC,EAAE,IAAI,EAAE,YAAY,EAAE,KAAK,EAAE,YAAY,EAAE;iBAC5C;aACF,CAAC,CAAC;QACL,CAAC;QAED,MAAM,OAAO,GAAmD;YAC9D,EAAE,IAAI,EAAE,cAAc,EAAE,KAAK,EAAE,CAAC,oBAAoB,EAAE,eAAe,CAAC,EAAE;YACxE;gBACE,IAAI,EAAE,4CAA4C;gBAClD,KAAK,EACH,MAAM,KAAK,OAAO;oBAChB,CAAC,CAAC;wBACE;4BACE,EAAE,EAAE,eAAe;4BACnB,MAAM,EAAE;gCACN,KAAK,EAAE;oCACL;wCACE,IAAI,EAAE,UAAU;wCAChB,QAAQ,EAAE;4CACR,IAAI,EAAE,qBAAqB;4CAC3B,WAAW,EAAE,6CAA6C;4CAC1D,UAAU,EAAE;gDACV,IAAI,EAAE,QAAQ;gDACd,UAAU,EAAE;oDACV,QAAQ,EAAE;wDACR,IAAI,EAAE,QAAQ;wDACd,WAAW,EAAE,4CAA4C;qDAC1D;iDACF;gDACD,QAAQ,EAAE,CAAC,UAAU,CAAC;6CACvB;yCACF;qCACF;iCACF;6BACF;yBACF;qBACF;oBACH,CAAC,CAAC,CAAC,oBAAoB,EAAE,eAAe,CAAC;aAC9C;YACD;gBACE,IAAI,EAAE,6CAA6C;gBACnD,KAAK,EAAE;oBACL,+CAA+C;oBAC/C,2CAA2C;iBAC5C;aACF;YACD;gBACE,IAAI,EAAE,sCAAsC;gBAC5C,KAAK,EAAE;oBACL,iEAAiE;oBACjE,8DAA8D;oBAC9D,gDAAgD;iBACjD;aACF;YACD;gBACE,IAAI,EAAE,qBAAqB;gBAC3B,KAAK,EAAE,CAAC,oBAAoB,CAAC;aAC9B;YACD;gBACE,IAAI,EAAE,yBAAyB;gBAC/B,KAAK,EAAE,CAAC,oBAAoB,CAAC;aAC9B;YACD;gBACE,IAAI,EAAE,kBAAkB;gBACxB,KAAK,EAAE,CAAC,kBAAkB,CAAC;aAC5B;YACD;gBACE,IAAI,EAAE,eAAe;gBACrB,KAAK,EAAE,CAAC,kCAAkC,CAAC;aAC5C;YACD;gBACE,IAAI,EAAE,yCAAyC;gBAC/C,KAAK,EAAE;oBACL,gDAAgD;oBAChD,+CAA+C;iBAChD;aACF;YACD;gBACE,IAAI,EAAE,qCAAqC;gBAC3C,KAAK,EAAE,CAAC,kBAAkB,EAAE,uBAAuB,CAAC;aACrD;YACD,EAAE,IAAI,EAAE,wCAAwC,EAAE,KAAK,EAAE,CAAC,mBAAmB,CAAC,EAAE;YAChF;gBACE,IAAI,EAAE,gCAAgC;gBACtC,KAAK,EAAE,CAAC,oBAAoB,EAAE,2BAA2B,CAAC;aAC3D;SACF,CAAC;QACF,MAAM,eAAe,GAAwB,MAAM,IAAA,kBAAQ,EAAC;YAC1D,OAAO,EAAE,oEAAoE;YAC7E,OAAO;SACR,CAAC,CAAC;QACH,IAAI,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC/B,MAAM,aAAa,GAAG,eAAe,CAAC,IAAI,EAAE,CAAC;YAC7C,IAAI,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC7B,SAAS,CAAC,IAAI,CACZ,GAAG,eAAe,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CACjF,CAAC;YACJ,CAAC;iBAAM,CAAC;gBACN,SAAS,CAAC,IAAI,CAAC,GAAG,aAAa,CAAC,CAAC;YACnC,CAAC;YAED,IACE,aAAa,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC,EAC1F,CAAC;gBACD,YAAE,CAAC,aAAa,CAAC,cAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,aAAa,CAAC,EAAE,2BAAmB,CAAC,CAAC;gBAC1F,gBAAM,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;YACrC,CAAC;YACD,IACE,aAAa,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,EACxF,CAAC;gBACD,YAAE,CAAC,aAAa,CAAC,cAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,aAAa,CAAC,EAAE,qBAAa,CAAC,CAAC;gBACpF,gBAAM,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;YACrC,CAAC;YACD,IACE,aAAa,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC,EAC1F,CAAC;gBACD,YAAE,CAAC,aAAa,CAAC,cAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,aAAa,CAAC,EAAE,uBAAe,CAAC,CAAC;gBACtF,gBAAM,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;YACrC,CAAC;QACH,CAAC;aAAM,CAAC;YACN,SAAS,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;YACrC,SAAS,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;QAClC,CAAC;QAED,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;YACzB,OAAO,CAAC,IAAI,CAAC,+BAA+B,CAAC,CAAC;YAC9C,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACzB,OAAO,CAAC,IAAI,CAAC,8CAA8C,CAAC,CAAC;YAC/D,CAAC;QACH,CAAC;aAAM,IAAI,MAAM,KAAK,KAAK,EAAE,CAAC;YAC5B,OAAO,CAAC,IAAI,CACV,6FAA6F,CAC9F,CAAC;QACJ,CAAC;aAAM,IAAI,MAAM,KAAK,OAAO,EAAE,CAAC;YAC9B,OAAO,CAAC,IAAI,CAAC,gDAAgD,CAAC,CAAC;QACjE,CAAC;QAED,IAAI,MAAM,KAAK,KAAK,IAAI,MAAM,KAAK,OAAO,EAAE,CAAC;YAC3C,IAAI,QAAQ,KAAK,YAAY,EAAE,CAAC;gBAC9B,YAAE,CAAC,aAAa,CAAC,cAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,YAAY,CAAC,EAAE,sBAAc,CAAC,CAAC;gBACpF,gBAAM,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;YACpC,CAAC;iBAAM,CAAC;gBACN,YAAE,CAAC,aAAa,CAAC,cAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,YAAY,CAAC,EAAE,kBAAU,CAAC,CAAC;gBAChF,gBAAM,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;YACpC,CAAC;QACH,CAAC;IACH,CAAC;SAAM,CAAC;QACN,MAAM,GAAG,SAAS,CAAC;QACnB,QAAQ,GAAG,UAAU,CAAC;QACtB,OAAO,CAAC,IAAI,CAAC,+BAA+B,CAAC,CAAC;QAC9C,OAAO,CAAC,IAAI,CAAC,8CAA8C,CAAC,CAAC;QAC7D,SAAS,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;QACrC,SAAS,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;IAClC,CAAC;IAED,MAAM,QAAQ,GAAG,IAAA,6BAAiB,GAAE,CAAC;IACrC,MAAM,MAAM,GAAG,QAAQ,CAAC,YAAY,CAAC,uBAAe,EAAE;QACpD,OAAO;QACP,SAAS;QACT,IAAI,EAAE,MAAM;QACZ,QAAQ;KACT,CAAC,CAAC;IACH,YAAE,CAAC,aAAa,CAAC,cAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,sBAAsB,CAAC,EAAE,MAAM,CAAC,CAAC;IACtF,YAAE,CAAC,aAAa,CAAC,cAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,WAAW,CAAC,EAAE,sBAAc,CAAC,CAAC;IAEnF,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,YAAY,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAC;IACxD,MAAM,UAAU,GAAG,KAAK,CAAC,CAAC,CAAC,2BAA2B,CAAC,CAAC,CAAC,gBAAgB,CAAC;IAC1E,IAAI,SAAS,KAAK,GAAG,EAAE,CAAC;QACtB,gBAAM,CAAC,IAAI,CACT,eAAK,CAAC,KAAK,CACT,uCAAuC,eAAK,CAAC,IAAI,CAAC,UAAU,CAAC,oBAAoB,CAClF,CACF,CAAC;IACJ,CAAC;SAAM,CAAC;QACN,gBAAM,CAAC,IAAI,CAAC,qCAAqC,SAAS,EAAE,CAAC,CAAC;QAC9D,gBAAM,CAAC,IAAI,CACT,eAAK,CAAC,KAAK,CACT,SAAS,eAAK,CAAC,IAAI,CAAC,MAAM,SAAS,EAAE,CAAC,iBAAiB,eAAK,CAAC,IAAI,CAC/D,UAAU,CACX,oBAAoB,CACtB,CACF,CAAC;IACJ,CAAC;IAED,OAAO;QACL,UAAU,EAAE,OAAO,CAAC,MAAM;QAC1B,gBAAgB,EAAE,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;QAC7F,MAAM;QACN,QAAQ;KACT,CAAC;AACJ,CAAC"}
@@ -35,7 +35,7 @@ exports.DEFAULT_GRADING_PROMPT = JSON.stringify([
35
35
 
36
36
  Output: Hello world
37
37
  Rubric: Content contains a greeting
38
- {"reason": "the content contains the word 'world'", "pass": true, "score": 1.0}
38
+ {"reason": "the content contains the word 'Hello'", "pass": true, "score": 1.0}
39
39
 
40
40
  Output: Avast ye swabs, repel the invaders!
41
41
  Rubric: Does not speak like a pirate
@@ -1,4 +1,4 @@
1
- import { ApiProvider, Prompt, PromptFunctionContext } from '../../types';
1
+ import type { ApiProvider, Prompt, PromptFunctionContext } from '../../types';
2
2
  export declare const transformContext: (context: {
3
3
  vars: Record<string, string | object>;
4
4
  provider?: ApiProvider;
@@ -1 +1 @@
1
- {"version":3,"file":"javascript.d.ts","sourceRoot":"","sources":["../../../../src/prompts/processors/javascript.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,WAAW,EAAE,MAAM,EAAE,qBAAqB,EAAE,MAAM,aAAa,CAAC;AAEzE,eAAO,MAAM,gBAAgB,YAAa;IACxC,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,CAAC;IACtC,QAAQ,CAAC,EAAE,WAAW,CAAC;CACxB,KAAG,qBAMH,CAAC;AAEF;;;;;GAKG;AACH,wBAAsB,aAAa,CACjC,QAAQ,EAAE,MAAM,EAChB,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,EACvB,YAAY,EAAE,MAAM,GAAG,SAAS,GAC/B,OAAO,CAAC,MAAM,EAAE,CAAC,CASnB"}
1
+ {"version":3,"file":"javascript.d.ts","sourceRoot":"","sources":["../../../../src/prompts/processors/javascript.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,EAAE,qBAAqB,EAAE,MAAM,aAAa,CAAC;AAE9E,eAAO,MAAM,gBAAgB,YAAa;IACxC,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,CAAC;IACtC,QAAQ,CAAC,EAAE,WAAW,CAAC;CACxB,KAAG,qBAMH,CAAC;AAEF;;;;;GAKG;AACH,wBAAsB,aAAa,CACjC,QAAQ,EAAE,MAAM,EAChB,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,EACvB,YAAY,EAAE,MAAM,GAAG,SAAS,GAC/B,OAAO,CAAC,MAAM,EAAE,CAAC,CASnB"}