promptfoo 0.95.0 → 0.96.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (350) hide show
  1. package/README.md +3 -1
  2. package/dist/package.json +19 -17
  3. package/dist/src/app/assets/{index-zXzgAsKj.js → index-CL29fyye.js} +185 -185
  4. package/dist/src/app/assets/{index.es-Jztl1qad.js → index.es-CsYHA4xg.js} +1 -1
  5. package/dist/src/app/assets/{sync-BJBBGzPI.js → sync-B9AjROwZ.js} +1 -1
  6. package/dist/src/app/index.html +1 -1
  7. package/dist/src/assertions/answerRelevance.d.ts +3 -0
  8. package/dist/src/assertions/answerRelevance.d.ts.map +1 -0
  9. package/dist/src/assertions/answerRelevance.js +19 -0
  10. package/dist/src/assertions/answerRelevance.js.map +1 -0
  11. package/dist/src/assertions/bleu.d.ts +23 -0
  12. package/dist/src/assertions/bleu.d.ts.map +1 -0
  13. package/dist/src/assertions/bleu.js +132 -0
  14. package/dist/src/assertions/bleu.js.map +1 -0
  15. package/dist/src/assertions/classifier.d.ts +3 -0
  16. package/dist/src/assertions/classifier.d.ts.map +1 -0
  17. package/dist/src/assertions/classifier.js +22 -0
  18. package/dist/src/assertions/classifier.js.map +1 -0
  19. package/dist/src/assertions/contains.d.ts +8 -0
  20. package/dist/src/assertions/contains.d.ts.map +1 -0
  21. package/dist/src/assertions/contains.js +110 -0
  22. package/dist/src/assertions/contains.js.map +1 -0
  23. package/dist/src/assertions/contextFaithfulness.d.ts +3 -0
  24. package/dist/src/assertions/contextFaithfulness.d.ts.map +1 -0
  25. package/dist/src/assertions/contextFaithfulness.js +19 -0
  26. package/dist/src/assertions/contextFaithfulness.js.map +1 -0
  27. package/dist/src/assertions/contextRecall.d.ts +3 -0
  28. package/dist/src/assertions/contextRecall.d.ts.map +1 -0
  29. package/dist/src/assertions/contextRecall.js +18 -0
  30. package/dist/src/assertions/contextRecall.js.map +1 -0
  31. package/dist/src/assertions/contextRelevance.d.ts +3 -0
  32. package/dist/src/assertions/contextRelevance.d.ts.map +1 -0
  33. package/dist/src/assertions/contextRelevance.js +19 -0
  34. package/dist/src/assertions/contextRelevance.js.map +1 -0
  35. package/dist/src/assertions/cost.d.ts +3 -0
  36. package/dist/src/assertions/cost.d.ts.map +1 -0
  37. package/dist/src/assertions/cost.js +22 -0
  38. package/dist/src/assertions/cost.js.map +1 -0
  39. package/dist/src/assertions/equals.d.ts +4 -0
  40. package/dist/src/assertions/equals.d.ts.map +1 -0
  41. package/dist/src/assertions/equals.js +32 -0
  42. package/dist/src/assertions/equals.js.map +1 -0
  43. package/dist/src/assertions/factuality.d.ts +3 -0
  44. package/dist/src/assertions/factuality.d.ts.map +1 -0
  45. package/dist/src/assertions/factuality.js +25 -0
  46. package/dist/src/assertions/factuality.js.map +1 -0
  47. package/dist/src/assertions/index.d.ts +1 -12
  48. package/dist/src/assertions/index.d.ts.map +1 -1
  49. package/dist/src/assertions/index.js +94 -987
  50. package/dist/src/assertions/index.js.map +1 -1
  51. package/dist/src/assertions/javascript.d.ts +4 -0
  52. package/dist/src/assertions/javascript.d.ts.map +1 -0
  53. package/dist/src/assertions/javascript.js +94 -0
  54. package/dist/src/assertions/javascript.js.map +1 -0
  55. package/dist/src/assertions/json.d.ts +7 -0
  56. package/dist/src/assertions/json.d.ts.map +1 -0
  57. package/dist/src/assertions/json.js +121 -0
  58. package/dist/src/assertions/json.js.map +1 -0
  59. package/dist/src/assertions/latency.d.ts +3 -0
  60. package/dist/src/assertions/latency.d.ts.map +1 -0
  61. package/dist/src/assertions/latency.js +22 -0
  62. package/dist/src/assertions/latency.js.map +1 -0
  63. package/dist/src/assertions/levenshtein.d.ts +3 -0
  64. package/dist/src/assertions/levenshtein.d.ts.map +1 -0
  65. package/dist/src/assertions/levenshtein.js +22 -0
  66. package/dist/src/assertions/levenshtein.js.map +1 -0
  67. package/dist/src/assertions/llmRubric.d.ts +3 -0
  68. package/dist/src/assertions/llmRubric.d.ts.map +1 -0
  69. package/dist/src/assertions/llmRubric.js +22 -0
  70. package/dist/src/assertions/llmRubric.js.map +1 -0
  71. package/dist/src/assertions/modelGradedClosedQa.d.ts +3 -0
  72. package/dist/src/assertions/modelGradedClosedQa.d.ts.map +1 -0
  73. package/dist/src/assertions/modelGradedClosedQa.js +25 -0
  74. package/dist/src/assertions/modelGradedClosedQa.js.map +1 -0
  75. package/dist/src/assertions/moderation.d.ts +3 -0
  76. package/dist/src/assertions/moderation.d.ts.map +1 -0
  77. package/dist/src/assertions/moderation.js +41 -0
  78. package/dist/src/assertions/moderation.js.map +1 -0
  79. package/dist/src/assertions/openai.d.ts +5 -0
  80. package/dist/src/assertions/openai.d.ts.map +1 -0
  81. package/dist/src/assertions/openai.js +68 -0
  82. package/dist/src/assertions/openai.js.map +1 -0
  83. package/dist/src/assertions/perplexity.d.ts +4 -0
  84. package/dist/src/assertions/perplexity.d.ts.map +1 -0
  85. package/dist/src/assertions/perplexity.js +40 -0
  86. package/dist/src/assertions/perplexity.js.map +1 -0
  87. package/dist/src/assertions/python.d.ts +4 -0
  88. package/dist/src/assertions/python.d.ts.map +1 -0
  89. package/dist/src/assertions/python.js +107 -0
  90. package/dist/src/assertions/python.js.map +1 -0
  91. package/dist/src/assertions/redteam.d.ts +3 -0
  92. package/dist/src/assertions/redteam.d.ts.map +1 -0
  93. package/dist/src/assertions/redteam.js +29 -0
  94. package/dist/src/assertions/redteam.js.map +1 -0
  95. package/dist/src/assertions/regex.d.ts +3 -0
  96. package/dist/src/assertions/regex.d.ts.map +1 -0
  97. package/dist/src/assertions/regex.js +34 -0
  98. package/dist/src/assertions/regex.js.map +1 -0
  99. package/dist/src/assertions/rouge.d.ts +3 -0
  100. package/dist/src/assertions/rouge.d.ts.map +1 -0
  101. package/dist/src/assertions/rouge.js +47 -0
  102. package/dist/src/assertions/rouge.js.map +1 -0
  103. package/dist/src/assertions/similar.d.ts +3 -0
  104. package/dist/src/assertions/similar.d.ts.map +1 -0
  105. package/dist/src/assertions/similar.js +40 -0
  106. package/dist/src/assertions/similar.js.map +1 -0
  107. package/dist/src/assertions/sql.d.ts +4 -0
  108. package/dist/src/assertions/sql.d.ts.map +1 -0
  109. package/dist/src/assertions/sql.js +98 -0
  110. package/dist/src/assertions/sql.js.map +1 -0
  111. package/dist/src/assertions/startsWith.d.ts +3 -0
  112. package/dist/src/assertions/startsWith.d.ts.map +1 -0
  113. package/dist/src/assertions/startsWith.js +22 -0
  114. package/dist/src/assertions/startsWith.js.map +1 -0
  115. package/dist/src/assertions/utils.d.ts +1 -0
  116. package/dist/src/assertions/utils.d.ts.map +1 -1
  117. package/dist/src/assertions/utils.js +7 -0
  118. package/dist/src/assertions/utils.js.map +1 -1
  119. package/dist/src/assertions/webhook.d.ts +3 -0
  120. package/dist/src/assertions/webhook.d.ts.map +1 -0
  121. package/dist/src/assertions/webhook.js +55 -0
  122. package/dist/src/assertions/webhook.js.map +1 -0
  123. package/dist/src/assertions/xml.d.ts +11 -0
  124. package/dist/src/assertions/xml.d.ts.map +1 -0
  125. package/dist/src/assertions/xml.js +86 -0
  126. package/dist/src/assertions/xml.js.map +1 -0
  127. package/dist/src/commands/eval/filterTests.d.ts +1 -0
  128. package/dist/src/commands/eval/filterTests.d.ts.map +1 -1
  129. package/dist/src/commands/eval/filterTests.js +14 -1
  130. package/dist/src/commands/eval/filterTests.js.map +1 -1
  131. package/dist/src/commands/eval.d.ts.map +1 -1
  132. package/dist/src/commands/eval.js +14 -3
  133. package/dist/src/commands/eval.js.map +1 -1
  134. package/dist/src/commands/import.d.ts.map +1 -1
  135. package/dist/src/commands/import.js +1 -0
  136. package/dist/src/commands/import.js.map +1 -1
  137. package/dist/src/commands/list.d.ts.map +1 -1
  138. package/dist/src/commands/list.js +15 -0
  139. package/dist/src/commands/list.js.map +1 -1
  140. package/dist/src/csv.d.ts.map +1 -1
  141. package/dist/src/csv.js +9 -4
  142. package/dist/src/csv.js.map +1 -1
  143. package/dist/src/database/tables.d.ts +189 -1
  144. package/dist/src/database/tables.d.ts.map +1 -1
  145. package/dist/src/envars.d.ts +9 -0
  146. package/dist/src/envars.d.ts.map +1 -1
  147. package/dist/src/envars.js.map +1 -1
  148. package/dist/src/evaluator.js +2 -2
  149. package/dist/src/evaluator.js.map +1 -1
  150. package/dist/src/evaluatorHelpers.d.ts.map +1 -1
  151. package/dist/src/evaluatorHelpers.js +4 -0
  152. package/dist/src/evaluatorHelpers.js.map +1 -1
  153. package/dist/src/fetch.d.ts.map +1 -1
  154. package/dist/src/fetch.js +8 -1
  155. package/dist/src/fetch.js.map +1 -1
  156. package/dist/src/globalConfig/accounts.d.ts +1 -0
  157. package/dist/src/globalConfig/accounts.d.ts.map +1 -1
  158. package/dist/src/globalConfig/accounts.js +24 -0
  159. package/dist/src/globalConfig/accounts.js.map +1 -1
  160. package/dist/src/index.d.ts +74 -68
  161. package/dist/src/index.d.ts.map +1 -1
  162. package/dist/src/onboarding.js +8 -8
  163. package/dist/src/onboarding.js.map +1 -1
  164. package/dist/src/providers/adaline.gateway.d.ts.map +1 -1
  165. package/dist/src/providers/adaline.gateway.js +4 -4
  166. package/dist/src/providers/adaline.gateway.js.map +1 -1
  167. package/dist/src/providers/anthropic.d.ts.map +1 -1
  168. package/dist/src/providers/anthropic.js +31 -0
  169. package/dist/src/providers/anthropic.js.map +1 -1
  170. package/dist/src/providers/{azureopenai.d.ts → azure.d.ts} +13 -13
  171. package/dist/src/providers/azure.d.ts.map +1 -0
  172. package/dist/src/providers/{azureopenai.js → azure.js} +64 -42
  173. package/dist/src/providers/azure.js.map +1 -0
  174. package/dist/src/providers/{azureopenaiUtil.d.ts → azureUtil.d.ts} +1 -1
  175. package/dist/src/providers/azureUtil.d.ts.map +1 -0
  176. package/dist/src/providers/{azureopenaiUtil.js → azureUtil.js} +3 -3
  177. package/dist/src/providers/azureUtil.js.map +1 -0
  178. package/dist/src/providers/bedrock.d.ts +5 -3
  179. package/dist/src/providers/bedrock.d.ts.map +1 -1
  180. package/dist/src/providers/bedrock.js +38 -9
  181. package/dist/src/providers/bedrock.js.map +1 -1
  182. package/dist/src/providers/defaults.d.ts.map +1 -1
  183. package/dist/src/providers/defaults.js +36 -0
  184. package/dist/src/providers/defaults.js.map +1 -1
  185. package/dist/src/providers/portkey.d.ts +3 -2
  186. package/dist/src/providers/portkey.d.ts.map +1 -1
  187. package/dist/src/providers/portkey.js +17 -10
  188. package/dist/src/providers/portkey.js.map +1 -1
  189. package/dist/src/providers/promptfoo.js +2 -2
  190. package/dist/src/providers/promptfoo.js.map +1 -1
  191. package/dist/src/providers.d.ts +9 -5
  192. package/dist/src/providers.d.ts.map +1 -1
  193. package/dist/src/providers.js +16 -11
  194. package/dist/src/providers.js.map +1 -1
  195. package/dist/src/redteam/commands/generate.d.ts.map +1 -1
  196. package/dist/src/redteam/commands/generate.js +5 -1
  197. package/dist/src/redteam/commands/generate.js.map +1 -1
  198. package/dist/src/redteam/commands/init.d.ts +11 -0
  199. package/dist/src/redteam/commands/init.d.ts.map +1 -1
  200. package/dist/src/redteam/commands/init.js +47 -12
  201. package/dist/src/redteam/commands/init.js.map +1 -1
  202. package/dist/src/redteam/commands/poison.d.ts +3 -0
  203. package/dist/src/redteam/commands/poison.d.ts.map +1 -0
  204. package/dist/src/redteam/commands/poison.js +165 -0
  205. package/dist/src/redteam/commands/poison.js.map +1 -0
  206. package/dist/src/redteam/commands/run.d.ts.map +1 -1
  207. package/dist/src/redteam/commands/run.js +2 -0
  208. package/dist/src/redteam/commands/run.js.map +1 -1
  209. package/dist/src/redteam/constants.d.ts +6 -5
  210. package/dist/src/redteam/constants.d.ts.map +1 -1
  211. package/dist/src/redteam/constants.js +54 -37
  212. package/dist/src/redteam/constants.js.map +1 -1
  213. package/dist/src/redteam/extraction/util.js +1 -1
  214. package/dist/src/redteam/extraction/util.js.map +1 -1
  215. package/dist/src/redteam/graders.d.ts +34 -32
  216. package/dist/src/redteam/graders.d.ts.map +1 -1
  217. package/dist/src/redteam/graders.js +34 -33
  218. package/dist/src/redteam/graders.js.map +1 -1
  219. package/dist/src/redteam/index.d.ts.map +1 -1
  220. package/dist/src/redteam/index.js +9 -2
  221. package/dist/src/redteam/index.js.map +1 -1
  222. package/dist/src/redteam/plugins/bfla.d.ts.map +1 -1
  223. package/dist/src/redteam/plugins/bfla.js +5 -3
  224. package/dist/src/redteam/plugins/bfla.js.map +1 -1
  225. package/dist/src/redteam/plugins/bola.js +3 -3
  226. package/dist/src/redteam/plugins/index.d.ts.map +1 -1
  227. package/dist/src/redteam/plugins/index.js +3 -2
  228. package/dist/src/redteam/plugins/index.js.map +1 -1
  229. package/dist/src/redteam/plugins/intent.d.ts +22 -0
  230. package/dist/src/redteam/plugins/intent.d.ts.map +1 -0
  231. package/dist/src/redteam/plugins/intent.js +100 -0
  232. package/dist/src/redteam/plugins/intent.js.map +1 -0
  233. package/dist/src/redteam/plugins/pii.d.ts.map +1 -1
  234. package/dist/src/redteam/plugins/pii.js +21 -14
  235. package/dist/src/redteam/plugins/pii.js.map +1 -1
  236. package/dist/src/redteam/plugins/rbac.d.ts.map +1 -1
  237. package/dist/src/redteam/plugins/rbac.js +5 -4
  238. package/dist/src/redteam/plugins/rbac.js.map +1 -1
  239. package/dist/src/redteam/plugins/sqlInjection.d.ts.map +1 -1
  240. package/dist/src/redteam/plugins/sqlInjection.js +13 -4
  241. package/dist/src/redteam/plugins/sqlInjection.js.map +1 -1
  242. package/dist/src/redteam/providers/crescendo/index.d.ts.map +1 -1
  243. package/dist/src/redteam/providers/crescendo/index.js +6 -4
  244. package/dist/src/redteam/providers/crescendo/index.js.map +1 -1
  245. package/dist/src/redteam/providers/goat.d.ts +2 -0
  246. package/dist/src/redteam/providers/goat.d.ts.map +1 -1
  247. package/dist/src/redteam/providers/goat.js +10 -3
  248. package/dist/src/redteam/providers/goat.js.map +1 -1
  249. package/dist/src/redteam/providers/iterative.d.ts.map +1 -1
  250. package/dist/src/redteam/providers/iterative.js +4 -2
  251. package/dist/src/redteam/providers/iterative.js.map +1 -1
  252. package/dist/src/redteam/providers/iterativeImage.d.ts.map +1 -1
  253. package/dist/src/redteam/providers/iterativeImage.js +4 -2
  254. package/dist/src/redteam/providers/iterativeImage.js.map +1 -1
  255. package/dist/src/redteam/providers/iterativeTree.d.ts +3 -1
  256. package/dist/src/redteam/providers/iterativeTree.d.ts.map +1 -1
  257. package/dist/src/redteam/providers/iterativeTree.js +5 -3
  258. package/dist/src/redteam/providers/iterativeTree.js.map +1 -1
  259. package/dist/src/redteam/providers/shared.d.ts +2 -2
  260. package/dist/src/redteam/providers/shared.d.ts.map +1 -1
  261. package/dist/src/redteam/providers/shared.js +2 -2
  262. package/dist/src/redteam/providers/shared.js.map +1 -1
  263. package/dist/src/redteam/strategies/mathPrompt.js +1 -1
  264. package/dist/src/redteam/strategies/mathPrompt.js.map +1 -1
  265. package/dist/src/redteam/strategies/multilingual.js +1 -1
  266. package/dist/src/redteam/strategies/multilingual.js.map +1 -1
  267. package/dist/src/redteam/types.d.ts +1 -0
  268. package/dist/src/redteam/types.d.ts.map +1 -1
  269. package/dist/src/remoteGrading.js +1 -1
  270. package/dist/src/remoteGrading.js.map +1 -1
  271. package/dist/src/telemetry.d.ts +2 -2
  272. package/dist/src/types/index.d.ts +4737 -12
  273. package/dist/src/types/index.d.ts.map +1 -1
  274. package/dist/src/types/index.js +23 -26
  275. package/dist/src/types/index.js.map +1 -1
  276. package/dist/src/types/providers.d.ts +12 -0
  277. package/dist/src/types/providers.d.ts.map +1 -1
  278. package/dist/src/types/providers.js.map +1 -1
  279. package/dist/src/util/config/load.d.ts +1 -1
  280. package/dist/src/util/config/load.d.ts.map +1 -1
  281. package/dist/src/util/config/load.js +15 -2
  282. package/dist/src/util/config/load.js.map +1 -1
  283. package/dist/src/util/index.d.ts +34 -0
  284. package/dist/src/util/index.d.ts.map +1 -1
  285. package/dist/src/validators/providers.d.ts +391 -0
  286. package/dist/src/validators/providers.d.ts.map +1 -1
  287. package/dist/src/validators/providers.js +18 -0
  288. package/dist/src/validators/providers.js.map +1 -1
  289. package/dist/src/validators/redteam.d.ts +136 -0
  290. package/dist/src/validators/redteam.d.ts.map +1 -1
  291. package/dist/test/assertions/bleu.test.d.ts +2 -0
  292. package/dist/test/assertions/bleu.test.d.ts.map +1 -0
  293. package/dist/test/assertions/bleu.test.js +162 -0
  294. package/dist/test/assertions/bleu.test.js.map +1 -0
  295. package/dist/test/assertions/index.test.js +276 -297
  296. package/dist/test/assertions/index.test.js.map +1 -1
  297. package/dist/test/assertions/json.test.d.ts +2 -0
  298. package/dist/test/assertions/json.test.d.ts.map +1 -0
  299. package/dist/test/assertions/json.test.js +36 -0
  300. package/dist/test/assertions/json.test.js.map +1 -0
  301. package/dist/test/assertions/sql.test.d.ts +2 -0
  302. package/dist/test/assertions/sql.test.d.ts.map +1 -0
  303. package/dist/test/assertions/sql.test.js +280 -0
  304. package/dist/test/assertions/sql.test.js.map +1 -0
  305. package/dist/test/commands/eval/filterTests.test.js +30 -0
  306. package/dist/test/commands/eval/filterTests.test.js.map +1 -1
  307. package/dist/test/factories/evalFactory.d.ts +155 -1
  308. package/dist/test/factories/evalFactory.d.ts.map +1 -1
  309. package/dist/test/fetch.test.js +17 -0
  310. package/dist/test/fetch.test.js.map +1 -1
  311. package/dist/test/onboarding.test.js +126 -1
  312. package/dist/test/onboarding.test.js.map +1 -1
  313. package/dist/test/providers/anthropic.test.js +120 -0
  314. package/dist/test/providers/anthropic.test.js.map +1 -1
  315. package/dist/test/providers/azure.test.js +22 -25
  316. package/dist/test/providers/azure.test.js.map +1 -1
  317. package/dist/test/providers/bedrock.test.js +178 -55
  318. package/dist/test/providers/bedrock.test.js.map +1 -1
  319. package/dist/test/providers/index.test.js +7 -7
  320. package/dist/test/providers/index.test.js.map +1 -1
  321. package/dist/test/providers/portkey.test.d.ts +2 -0
  322. package/dist/test/providers/portkey.test.d.ts.map +1 -0
  323. package/dist/test/providers/portkey.test.js +46 -0
  324. package/dist/test/providers/portkey.test.js.map +1 -0
  325. package/dist/test/redteam/commands/init.test.d.ts +2 -0
  326. package/dist/test/redteam/commands/init.test.d.ts.map +1 -0
  327. package/dist/test/redteam/commands/init.test.js +109 -0
  328. package/dist/test/redteam/commands/init.test.js.map +1 -0
  329. package/dist/test/redteam/plugins/pluginDocumentation.test.js +4 -1
  330. package/dist/test/redteam/plugins/pluginDocumentation.test.js.map +1 -1
  331. package/dist/test/redteam/providers/goat.test.js +1 -1
  332. package/dist/test/redteam/providers/goat.test.js.map +1 -1
  333. package/dist/test/redteam/providers/iterativeTree.test.js +8 -3
  334. package/dist/test/redteam/providers/iterativeTree.test.js.map +1 -1
  335. package/dist/test/util/config/load.test.js +44 -1
  336. package/dist/test/util/config/load.test.js.map +1 -1
  337. package/dist/tsconfig.tsbuildinfo +1 -1
  338. package/package.json +19 -17
  339. package/dist/src/providers/azureopenai.d.ts.map +0 -1
  340. package/dist/src/providers/azureopenai.js.map +0 -1
  341. package/dist/src/providers/azureopenaiUtil.d.ts.map +0 -1
  342. package/dist/src/providers/azureopenaiUtil.js.map +0 -1
  343. package/dist/src/redteam/eval/excessive-agency/llm_rubric-20240617.json +0 -10
  344. package/dist/src/redteam/eval/excessive-agency/llm_rubric-20240618.json +0 -10
  345. package/dist/src/redteam/eval/harmful/llm_rubric-20240723.json +0 -10
  346. package/dist/src/redteam/eval/harmful/llm_rubric-20240724.json +0 -10
  347. package/dist/test/is-sql-tests/node-sql-parser.test.d.ts +0 -2
  348. package/dist/test/is-sql-tests/node-sql-parser.test.d.ts.map +0 -1
  349. package/dist/test/is-sql-tests/node-sql-parser.test.js +0 -179
  350. package/dist/test/is-sql-tests/node-sql-parser.test.js.map +0 -1
@@ -0,0 +1,280 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ /* eslint-disable jest/no-commented-out-tests */
4
+ const sql_1 = require("../../src/assertions/sql");
5
+ const assertion = {
6
+ type: 'is-sql',
7
+ };
8
+ describe('is-sql assertion', () => {
9
+ // -------------------------------------------------- Basic Tests ------------------------------------------------------ //
10
+ describe('Basic tests', () => {
11
+ it('should pass when the output string is a valid SQL statement', async () => {
12
+ const renderedValue = undefined;
13
+ const outputString = 'SELECT id, name FROM users';
14
+ const result = await (0, sql_1.handleIsSql)({
15
+ assertion,
16
+ renderedValue,
17
+ outputString,
18
+ inverse: false,
19
+ });
20
+ expect(result).toEqual({
21
+ assertion,
22
+ pass: true,
23
+ reason: 'Assertion passed',
24
+ score: 1,
25
+ });
26
+ });
27
+ it('should fail when the SQL statement contains a syntax error in the ORDER BY clause', async () => {
28
+ const renderedValue = undefined;
29
+ const outputString = 'SELECT * FROM orders ORDERY BY order_date';
30
+ const result = await (0, sql_1.handleIsSql)({
31
+ assertion,
32
+ renderedValue,
33
+ outputString,
34
+ inverse: false,
35
+ });
36
+ expect(result).toEqual({
37
+ pass: false,
38
+ score: 0,
39
+ reason: 'SQL statement does not conform to the provided MySQL database syntax.',
40
+ assertion,
41
+ });
42
+ });
43
+ it('should fail when the SQL statement uses a reserved keyword as a table name', async () => {
44
+ const renderedValue = undefined;
45
+ const outputString = 'SELECT * FROM select WHERE id = 1';
46
+ const result = await (0, sql_1.handleIsSql)({
47
+ assertion,
48
+ renderedValue,
49
+ outputString,
50
+ inverse: false,
51
+ });
52
+ expect(result).toEqual({
53
+ pass: false,
54
+ score: 0,
55
+ reason: 'SQL statement does not conform to the provided MySQL database syntax.',
56
+ assertion,
57
+ });
58
+ });
59
+ it('should fail when the SQL statement has an incorrect DELETE syntax', async () => {
60
+ const renderedValue = undefined;
61
+ const outputString = 'DELETE employees WHERE id = 1';
62
+ const result = await (0, sql_1.handleIsSql)({
63
+ assertion,
64
+ renderedValue,
65
+ outputString,
66
+ inverse: false,
67
+ });
68
+ expect(result).toEqual({
69
+ pass: false,
70
+ score: 0,
71
+ reason: 'SQL statement does not conform to the provided MySQL database syntax.',
72
+ assertion,
73
+ });
74
+ });
75
+ /**
76
+ * Catches an incorrect output from node-sql-parser package
77
+ * The parser cannot identify the syntax error: missing comma between column names
78
+ */
79
+ // it('should fail when the output string is an invalid SQL statement', () => {
80
+ // const renderedValue = undefined;
81
+ // const outputString = 'SELECT first_name last_name FROM employees';
82
+ // const result = testFunction(renderedValue, outputString, false);
83
+ // expect(result.pass).toBe(false);
84
+ // expect(result.reason).toBe('SQL statement does not conform to the provided MySQL database syntax.');
85
+ // });
86
+ /**
87
+ * Catches an incorrect output from node-sql-parser package
88
+ * The parser cannot identify the syntax error: misuse of backticks (`)
89
+ */
90
+ // it('should fail when the output string is an invalid SQL statement', () => {
91
+ // const renderedValue = undefined;
92
+ // const outputString = 'SELECT * FROM `employees`';
93
+ // const result = testFunction(renderedValue, outputString, false);
94
+ // expect(result.pass).toBe(false);
95
+ // expect(result.reason).toBe('SQL statement does not conform to the provided MySQL database syntax.');
96
+ // });
97
+ });
98
+ // ------------------------------------------ Database Specific Syntax Tests ------------------------------------------- //
99
+ describe('Database Specific Syntax Tests', () => {
100
+ it('should fail if the output SQL statement conforms to MySQL but not PostgreSQL', async () => {
101
+ const renderedValue = {
102
+ databaseType: 'PostgreSQL',
103
+ };
104
+ const outputString = `SELECT * FROM employees WHERE id = 1 LOCK IN SHARE MODE`;
105
+ const result = await (0, sql_1.handleIsSql)({
106
+ assertion,
107
+ renderedValue,
108
+ outputString,
109
+ inverse: false,
110
+ });
111
+ expect(result).toEqual({
112
+ pass: false,
113
+ score: 0,
114
+ reason: 'SQL statement does not conform to the provided PostgreSQL database syntax.',
115
+ assertion,
116
+ });
117
+ });
118
+ it('should fail if the output SQL statement conforms to PostgreSQL but not MySQL', async () => {
119
+ const renderedValue = {
120
+ databaseType: 'MySQL',
121
+ };
122
+ const outputString = `SELECT first_name, last_name FROM employees WHERE first_name ILIKE 'john%'`;
123
+ const result = await (0, sql_1.handleIsSql)({
124
+ assertion,
125
+ renderedValue,
126
+ outputString,
127
+ inverse: false,
128
+ });
129
+ expect(result).toEqual({
130
+ pass: false,
131
+ score: 0,
132
+ reason: 'SQL statement does not conform to the provided MySQL database syntax.',
133
+ assertion,
134
+ });
135
+ });
136
+ it('should pass if the output SQL statement conforms to PostgreSQL but not MySQL', async () => {
137
+ const renderedValue = {
138
+ databaseType: 'PostgreSQL',
139
+ };
140
+ const outputString = `SELECT first_name, last_name FROM employees WHERE first_name ILIKE 'john%'`;
141
+ const result = await (0, sql_1.handleIsSql)({
142
+ assertion,
143
+ renderedValue,
144
+ outputString,
145
+ inverse: false,
146
+ });
147
+ expect(result).toEqual({
148
+ pass: true,
149
+ score: 1,
150
+ reason: 'Assertion passed',
151
+ assertion,
152
+ });
153
+ });
154
+ /**
155
+ * Catches an incorrect output from node-sql-parser package
156
+ * The parser cannot differentiate certain syntax between MySQL and PostgreSQL
157
+ */
158
+ // it('should fail if the output SQL statement conforms to PostgreSQL but not MySQL', () => {
159
+ // const renderedValue = {
160
+ // databaseType: 'MySQL',
161
+ // };
162
+ // const outputString = `SELECT generate_series(1, 10);`;
163
+ // const result = testFunction(renderedValue, outputString, false);
164
+ // expect(result.pass).toBe(false);
165
+ // expect(result.reason).toBe('SQL statement does not conform to the provided MySQL database syntax.');
166
+ // });
167
+ });
168
+ // ------------------------------------------- White Table/Column List Tests ------------------------------------------- //
169
+ describe('White Table/Column List Tests', () => {
170
+ it('should fail if the output SQL statement violate allowedTables', async () => {
171
+ const renderedValue = {
172
+ databaseType: 'MySQL',
173
+ allowedTables: ['(select|update|insert|delete)::null::departments'],
174
+ };
175
+ const outputString = `SELECT * FROM employees`;
176
+ const result = await (0, sql_1.handleIsSql)({
177
+ assertion,
178
+ renderedValue,
179
+ outputString,
180
+ inverse: false,
181
+ });
182
+ expect(result).toEqual({
183
+ pass: false,
184
+ score: 0,
185
+ reason: `SQL validation failed: authority = 'select::null::employees' is required in table whiteList to execute SQL = 'SELECT * FROM employees'.`,
186
+ assertion,
187
+ });
188
+ });
189
+ it('should pass if the output SQL statement does not violate allowedTables', async () => {
190
+ const renderedValue = {
191
+ databaseType: 'MySQL',
192
+ allowedTables: ['(select|update|insert|delete)::null::departments'],
193
+ };
194
+ const outputString = `SELECT * FROM departments`;
195
+ const result = await (0, sql_1.handleIsSql)({
196
+ assertion,
197
+ renderedValue,
198
+ outputString,
199
+ inverse: false,
200
+ });
201
+ expect(result).toEqual({
202
+ pass: true,
203
+ score: 1,
204
+ reason: 'Assertion passed',
205
+ assertion,
206
+ });
207
+ });
208
+ it('should fail if the output SQL statement violate allowedColumns', async () => {
209
+ const renderedValue = {
210
+ databaseType: 'MySQL',
211
+ allowedColumns: ['select::null::name', 'update::null::id'],
212
+ };
213
+ const outputString = `SELECT id FROM t`;
214
+ const result = await (0, sql_1.handleIsSql)({
215
+ assertion,
216
+ renderedValue,
217
+ outputString,
218
+ inverse: false,
219
+ });
220
+ expect(result).toEqual({
221
+ pass: false,
222
+ score: 0,
223
+ reason: `SQL validation failed: authority = 'select::null::id' is required in column whiteList to execute SQL = 'SELECT id FROM t'.`,
224
+ assertion,
225
+ });
226
+ });
227
+ it('should pass if the output SQL statement does not violate allowedColumns', async () => {
228
+ const renderedValue = {
229
+ databaseType: 'MySQL',
230
+ allowedColumns: ['insert::department::dept_name', 'insert::department::location'],
231
+ };
232
+ const outputString = `INSERT INTO department (dept_name, location) VALUES ('Sales', 'New York')`;
233
+ const result = await (0, sql_1.handleIsSql)({
234
+ assertion,
235
+ renderedValue,
236
+ outputString,
237
+ inverse: false,
238
+ });
239
+ expect(result).toEqual({
240
+ pass: true,
241
+ score: 1,
242
+ reason: 'Assertion passed',
243
+ assertion,
244
+ });
245
+ });
246
+ /**
247
+ * Catches an incorrect output from node-sql-parser package
248
+ * Error message: message: "authority = 'update::null::id' is required in column whiteList to execute SQL = 'UPDATE a SET id = 1'"
249
+ * issue: In this test case, the 'whiteListCheck' function in node-sql-parser requires an explicit 'update::null::id'
250
+ * in the whitelist to allow SQL statement like `UPDATE a SET id = 1`, despite the presence
251
+ * of rule `update::a::id`
252
+ */
253
+ // it('should pass if the output SQL statement does not violate allowedColumns', () => {
254
+ // const renderedValue = {
255
+ // databaseType: 'MySQL',
256
+ // allowedColumns: ['update::a::id'],
257
+ // };
258
+ // const outputString = `UPDATE a SET id = 1`;
259
+ // const result = testFunction(renderedValue, outputString, false);
260
+ // expect(result.pass).toBe(true);
261
+ // expect(result.reason).toBe('Assertion passed');
262
+ // });
263
+ /**
264
+ * Similar issue: the error message is Error: authority = 'select::null::id' is required
265
+ * in column whiteList to execute SQL = 'UPDATE employee SET salary = 50000 WHERE id = 1'
266
+ */
267
+ // it('should pass if the output SQL statement does not violate allowedColumns', () => {
268
+ // const renderedValue = {
269
+ // databaseType: 'MySQL',
270
+ // allowedColumns: ['update::employee::salary','select::employee::id'],
271
+ // };
272
+ // const outputString = `UPDATE employee SET salary = 50000 WHERE id = 1`;
273
+ // const result = testFunction(renderedValue, outputString, false);
274
+ // expect(result.pass).toBe(true);
275
+ // expect(result.reason).toBe('Assertion passed');
276
+ // });
277
+ });
278
+ /* eslint-enable jest/no-commented-out-tests */
279
+ });
280
+ //# sourceMappingURL=sql.test.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"sql.test.js","sourceRoot":"","sources":["../../../test/assertions/sql.test.ts"],"names":[],"mappings":";;AAAA,gDAAgD;AAChD,kDAAuD;AAGvD,MAAM,SAAS,GAAc;IAC3B,IAAI,EAAE,QAAQ;CACf,CAAC;AAEF,QAAQ,CAAC,kBAAkB,EAAE,GAAG,EAAE;IAChC,2HAA2H;IAC3H,QAAQ,CAAC,aAAa,EAAE,GAAG,EAAE;QAC3B,EAAE,CAAC,6DAA6D,EAAE,KAAK,IAAI,EAAE;YAC3E,MAAM,aAAa,GAAG,SAAS,CAAC;YAChC,MAAM,YAAY,GAAG,4BAA4B,CAAC;YAClD,MAAM,MAAM,GAAkB,MAAM,IAAA,iBAAW,EAAC;gBAC9C,SAAS;gBACT,aAAa;gBACb,YAAY;gBACZ,OAAO,EAAE,KAAK;aACI,CAAC,CAAC;YACtB,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;gBACrB,SAAS;gBACT,IAAI,EAAE,IAAI;gBACV,MAAM,EAAE,kBAAkB;gBAC1B,KAAK,EAAE,CAAC;aACT,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,mFAAmF,EAAE,KAAK,IAAI,EAAE;YACjG,MAAM,aAAa,GAAG,SAAS,CAAC;YAChC,MAAM,YAAY,GAAG,2CAA2C,CAAC;YACjE,MAAM,MAAM,GAAkB,MAAM,IAAA,iBAAW,EAAC;gBAC9C,SAAS;gBACT,aAAa;gBACb,YAAY;gBACZ,OAAO,EAAE,KAAK;aACI,CAAC,CAAC;YACtB,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;gBACrB,IAAI,EAAE,KAAK;gBACX,KAAK,EAAE,CAAC;gBACR,MAAM,EAAE,uEAAuE;gBAC/E,SAAS;aACV,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,4EAA4E,EAAE,KAAK,IAAI,EAAE;YAC1F,MAAM,aAAa,GAAG,SAAS,CAAC;YAChC,MAAM,YAAY,GAAG,mCAAmC,CAAC;YACzD,MAAM,MAAM,GAAkB,MAAM,IAAA,iBAAW,EAAC;gBAC9C,SAAS;gBACT,aAAa;gBACb,YAAY;gBACZ,OAAO,EAAE,KAAK;aACI,CAAC,CAAC;YACtB,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;gBACrB,IAAI,EAAE,KAAK;gBACX,KAAK,EAAE,CAAC;gBACR,MAAM,EAAE,uEAAuE;gBAC/E,SAAS;aACV,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,mEAAmE,EAAE,KAAK,IAAI,EAAE;YACjF,MAAM,aAAa,GAAG,SAAS,CAAC;YAChC,MAAM,YAAY,GAAG,+BAA+B,CAAC;YACrD,MAAM,MAAM,GAAkB,MAAM,IAAA,iBAAW,EAAC;gBAC9C,SAAS;gBACT,aAAa;gBACb,YAAY;gBACZ,OAAO,EAAE,KAAK;aACI,CAAC,CAAC;YACtB,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;gBACrB,IAAI,EAAE,KAAK;gBACX,KAAK,EAAE,CAAC;gBACR,MAAM,EAAE,uEAAuE;gBAC/E,SAAS;aACV,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH;;;WAGG;QACH,+EAA+E;QAC/E,qCAAqC;QACrC,uEAAuE;QACvE,qEAAqE;QACrE,qCAAqC;QACrC,yGAAyG;QACzG,MAAM;QAEN;;;WAGG;QACH,+EAA+E;QAC/E,qCAAqC;QACrC,sDAAsD;QACtD,qEAAqE;QACrE,qCAAqC;QACrC,yGAAyG;QACzG,MAAM;IACR,CAAC,CAAC,CAAC;IAEH,2HAA2H;IAC3H,QAAQ,CAAC,gCAAgC,EAAE,GAAG,EAAE;QAC9C,EAAE,CAAC,8EAA8E,EAAE,KAAK,IAAI,EAAE;YAC5F,MAAM,aAAa,GAAG;gBACpB,YAAY,EAAE,YAAY;aAC3B,CAAC;YACF,MAAM,YAAY,GAAG,yDAAyD,CAAC;YAC/E,MAAM,MAAM,GAAkB,MAAM,IAAA,iBAAW,EAAC;gBAC9C,SAAS;gBACT,aAAa;gBACb,YAAY;gBACZ,OAAO,EAAE,KAAK;aACI,CAAC,CAAC;YACtB,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;gBACrB,IAAI,EAAE,KAAK;gBACX,KAAK,EAAE,CAAC;gBACR,MAAM,EAAE,4EAA4E;gBACpF,SAAS;aACV,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,8EAA8E,EAAE,KAAK,IAAI,EAAE;YAC5F,MAAM,aAAa,GAAG;gBACpB,YAAY,EAAE,OAAO;aACtB,CAAC;YACF,MAAM,YAAY,GAAG,4EAA4E,CAAC;YAClG,MAAM,MAAM,GAAkB,MAAM,IAAA,iBAAW,EAAC;gBAC9C,SAAS;gBACT,aAAa;gBACb,YAAY;gBACZ,OAAO,EAAE,KAAK;aACI,CAAC,CAAC;YACtB,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;gBACrB,IAAI,EAAE,KAAK;gBACX,KAAK,EAAE,CAAC;gBACR,MAAM,EAAE,uEAAuE;gBAC/E,SAAS;aACV,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,8EAA8E,EAAE,KAAK,IAAI,EAAE;YAC5F,MAAM,aAAa,GAAG;gBACpB,YAAY,EAAE,YAAY;aAC3B,CAAC;YACF,MAAM,YAAY,GAAG,4EAA4E,CAAC;YAClG,MAAM,MAAM,GAAkB,MAAM,IAAA,iBAAW,EAAC;gBAC9C,SAAS;gBACT,aAAa;gBACb,YAAY;gBACZ,OAAO,EAAE,KAAK;aACI,CAAC,CAAC;YACtB,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;gBACrB,IAAI,EAAE,IAAI;gBACV,KAAK,EAAE,CAAC;gBACR,MAAM,EAAE,kBAAkB;gBAC1B,SAAS;aACV,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH;;;WAGG;QACH,6FAA6F;QAC7F,4BAA4B;QAC5B,6BAA6B;QAC7B,OAAO;QACP,2DAA2D;QAC3D,qEAAqE;QACrE,qCAAqC;QACrC,yGAAyG;QACzG,MAAM;IACR,CAAC,CAAC,CAAC;IAEH,2HAA2H;IAC3H,QAAQ,CAAC,+BAA+B,EAAE,GAAG,EAAE;QAC7C,EAAE,CAAC,+DAA+D,EAAE,KAAK,IAAI,EAAE;YAC7E,MAAM,aAAa,GAAG;gBACpB,YAAY,EAAE,OAAO;gBACrB,aAAa,EAAE,CAAC,kDAAkD,CAAC;aACpE,CAAC;YACF,MAAM,YAAY,GAAG,yBAAyB,CAAC;YAC/C,MAAM,MAAM,GAAkB,MAAM,IAAA,iBAAW,EAAC;gBAC9C,SAAS;gBACT,aAAa;gBACb,YAAY;gBACZ,OAAO,EAAE,KAAK;aACI,CAAC,CAAC;YACtB,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;gBACrB,IAAI,EAAE,KAAK;gBACX,KAAK,EAAE,CAAC;gBACR,MAAM,EAAE,yIAAyI;gBACjJ,SAAS;aACV,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,wEAAwE,EAAE,KAAK,IAAI,EAAE;YACtF,MAAM,aAAa,GAAG;gBACpB,YAAY,EAAE,OAAO;gBACrB,aAAa,EAAE,CAAC,kDAAkD,CAAC;aACpE,CAAC;YACF,MAAM,YAAY,GAAG,2BAA2B,CAAC;YACjD,MAAM,MAAM,GAAkB,MAAM,IAAA,iBAAW,EAAC;gBAC9C,SAAS;gBACT,aAAa;gBACb,YAAY;gBACZ,OAAO,EAAE,KAAK;aACI,CAAC,CAAC;YACtB,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;gBACrB,IAAI,EAAE,IAAI;gBACV,KAAK,EAAE,CAAC;gBACR,MAAM,EAAE,kBAAkB;gBAC1B,SAAS;aACV,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,gEAAgE,EAAE,KAAK,IAAI,EAAE;YAC9E,MAAM,aAAa,GAAG;gBACpB,YAAY,EAAE,OAAO;gBACrB,cAAc,EAAE,CAAC,oBAAoB,EAAE,kBAAkB,CAAC;aAC3D,CAAC;YACF,MAAM,YAAY,GAAG,kBAAkB,CAAC;YACxC,MAAM,MAAM,GAAkB,MAAM,IAAA,iBAAW,EAAC;gBAC9C,SAAS;gBACT,aAAa;gBACb,YAAY;gBACZ,OAAO,EAAE,KAAK;aACI,CAAC,CAAC;YACtB,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;gBACrB,IAAI,EAAE,KAAK;gBACX,KAAK,EAAE,CAAC;gBACR,MAAM,EAAE,4HAA4H;gBACpI,SAAS;aACV,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,yEAAyE,EAAE,KAAK,IAAI,EAAE;YACvF,MAAM,aAAa,GAAG;gBACpB,YAAY,EAAE,OAAO;gBACrB,cAAc,EAAE,CAAC,+BAA+B,EAAE,8BAA8B,CAAC;aAClF,CAAC;YACF,MAAM,YAAY,GAAG,2EAA2E,CAAC;YACjG,MAAM,MAAM,GAAkB,MAAM,IAAA,iBAAW,EAAC;gBAC9C,SAAS;gBACT,aAAa;gBACb,YAAY;gBACZ,OAAO,EAAE,KAAK;aACI,CAAC,CAAC;YACtB,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;gBACrB,IAAI,EAAE,IAAI;gBACV,KAAK,EAAE,CAAC;gBACR,MAAM,EAAE,kBAAkB;gBAC1B,SAAS;aACV,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH;;;;;;WAMG;QACH,wFAAwF;QACxF,4BAA4B;QAC5B,6BAA6B;QAC7B,yCAAyC;QACzC,OAAO;QACP,gDAAgD;QAChD,qEAAqE;QACrE,oCAAoC;QACpC,oDAAoD;QACpD,MAAM;QAEN;;;WAGG;QACH,wFAAwF;QACxF,4BAA4B;QAC5B,6BAA6B;QAC7B,2EAA2E;QAC3E,OAAO;QACP,4EAA4E;QAC5E,qEAAqE;QACrE,oCAAoC;QACpC,oDAAoD;QACpD,MAAM;IACR,CAAC,CAAC,CAAC;IAEH,+CAA+C;AACjD,CAAC,CAAC,CAAC"}
@@ -47,5 +47,35 @@ describe('filterTests', () => {
47
47
  expect(result).not.toBe(tests);
48
48
  });
49
49
  });
50
+ describe('sample', () => {
51
+ it('should return the requested number of random tests', async () => {
52
+ const result = await (0, filterTests_1.filterTests)(testSuite, { sample: 2 });
53
+ expect(result).toHaveLength(2);
54
+ // Each result should be one of the original tests
55
+ result?.forEach((test) => {
56
+ expect(tests).toContain(test);
57
+ });
58
+ });
59
+ it('accepts sample as string', async () => {
60
+ const result = await (0, filterTests_1.filterTests)(testSuite, { sample: '2' });
61
+ expect(result).toHaveLength(2);
62
+ });
63
+ it('throws an exception when sample is not a number', async () => {
64
+ await expect(() => (0, filterTests_1.filterTests)(testSuite, { sample: 'NOPE' })).rejects.toEqual(new Error('sample must be a number, got: NOPE'));
65
+ });
66
+ it('returns all tests when sample size exceeds test count', async () => {
67
+ const result = await (0, filterTests_1.filterTests)(testSuite, { sample: 10 });
68
+ expect(result).toHaveLength(tests.length);
69
+ result?.forEach((test) => {
70
+ expect(tests).toContain(test);
71
+ });
72
+ });
73
+ it('can combine with pattern filter', async () => {
74
+ const result = await (0, filterTests_1.filterTests)(testSuite, { sample: 1, pattern: 'ey$' });
75
+ expect(result).toBeDefined();
76
+ expect(result).toHaveLength(1);
77
+ expect(result?.[0].description).toMatch(/ey$/);
78
+ });
79
+ });
50
80
  });
51
81
  //# sourceMappingURL=filterTests.test.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"filterTests.test.js","sourceRoot":"","sources":["../../../../test/commands/eval/filterTests.test.ts"],"names":[],"mappings":";;AAAA,wEAAqE;AAGrE,QAAQ,CAAC,aAAa,EAAE,GAAG,EAAE;IAC3B,MAAM,iBAAiB,GAAG,EAAE,CAAC;IAC7B,MAAM,QAAQ,GAAG;QACf,WAAW,EAAE,MAAM;KACpB,CAAC;IACF,MAAM,SAAS,GAAG;QAChB,WAAW,EAAE,OAAO;KACrB,CAAC;IACF,MAAM,SAAS,GAAG;QAChB,WAAW,EAAE,OAAO;KACrB,CAAC;IACF,MAAM,KAAK,GAAG,CAAC,QAAQ,EAAE,iBAAiB,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;IAClE,MAAM,SAAS,GAAG;QAChB,KAAK;KACO,CAAC;IAEf,EAAE,CAAC,mCAAmC,EAAE,KAAK,IAAI,EAAE;QACjD,MAAM,MAAM,GAAG,MAAM,IAAA,yBAAW,EAAC,SAAS,EAAE,EAAE,CAAC,CAAC;QAEhD,MAAM,CAAC,MAAM,CAAC,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;IACtC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,kBAAkB,EAAE,KAAK,IAAI,EAAE;QAChC,MAAM,MAAM,GAAG,MAAM,IAAA,yBAAW,EAAC,EAAe,EAAE,EAAE,CAAC,CAAC;QAEtD,MAAM,CAAC,MAAM,CAAC,CAAC,aAAa,EAAE,CAAC;IACjC,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,QAAQ,EAAE,GAAG,EAAE;QACtB,EAAE,CAAC,gCAAgC,EAAE,KAAK,IAAI,EAAE;YAC9C,MAAM,MAAM,GAAG,MAAM,IAAA,yBAAW,EAAC,SAAS,EAAE,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC;YAE7D,MAAM,CAAC,MAAM,CAAC,CAAC,aAAa,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QAClD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,iDAAiD,EAAE,KAAK,IAAI,EAAE;YAC/D,MAAM,MAAM,CAAC,GAAG,EAAE,CAAC,IAAA,yBAAW,EAAC,SAAS,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAC5E,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAChD,CAAC;QACJ,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,SAAS,EAAE,GAAG,EAAE;QACvB,EAAE,CAAC,yDAAyD,EAAE,KAAK,IAAI,EAAE;YACvE,MAAM,MAAM,GAAG,MAAM,IAAA,yBAAW,EAAC,SAAS,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC;YAEhE,MAAM,CAAC,MAAM,CAAC,CAAC,aAAa,CAAC,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC,CAAC;QACtD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,gCAAgC,EAAE,KAAK,IAAI,EAAE;YAC9C,MAAM,MAAM,GAAG,MAAM,IAAA,yBAAW,EAAC,SAAS,EAAE,EAAE,MAAM,EAAE,GAAG,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC;YAE7E,MAAM,CAAC,MAAM,CAAC,CAAC,aAAa,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC;QAC3C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,oCAAoC,EAAE,KAAK,IAAI,EAAE;YAClD,MAAM,MAAM,GAAG,MAAM,IAAA,yBAAW,EAAC,SAAS,EAAE,EAAE,MAAM,EAAE,GAAG,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC;YAE7E,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACjC,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
1
+ {"version":3,"file":"filterTests.test.js","sourceRoot":"","sources":["../../../../test/commands/eval/filterTests.test.ts"],"names":[],"mappings":";;AAAA,wEAAqE;AAGrE,QAAQ,CAAC,aAAa,EAAE,GAAG,EAAE;IAC3B,MAAM,iBAAiB,GAAG,EAAE,CAAC;IAC7B,MAAM,QAAQ,GAAG;QACf,WAAW,EAAE,MAAM;KACpB,CAAC;IACF,MAAM,SAAS,GAAG;QAChB,WAAW,EAAE,OAAO;KACrB,CAAC;IACF,MAAM,SAAS,GAAG;QAChB,WAAW,EAAE,OAAO;KACrB,CAAC;IACF,MAAM,KAAK,GAAG,CAAC,QAAQ,EAAE,iBAAiB,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;IAClE,MAAM,SAAS,GAAG;QAChB,KAAK;KACO,CAAC;IAEf,EAAE,CAAC,mCAAmC,EAAE,KAAK,IAAI,EAAE;QACjD,MAAM,MAAM,GAAG,MAAM,IAAA,yBAAW,EAAC,SAAS,EAAE,EAAE,CAAC,CAAC;QAEhD,MAAM,CAAC,MAAM,CAAC,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;IACtC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,kBAAkB,EAAE,KAAK,IAAI,EAAE;QAChC,MAAM,MAAM,GAAG,MAAM,IAAA,yBAAW,EAAC,EAAe,EAAE,EAAE,CAAC,CAAC;QAEtD,MAAM,CAAC,MAAM,CAAC,CAAC,aAAa,EAAE,CAAC;IACjC,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,QAAQ,EAAE,GAAG,EAAE;QACtB,EAAE,CAAC,gCAAgC,EAAE,KAAK,IAAI,EAAE;YAC9C,MAAM,MAAM,GAAG,MAAM,IAAA,yBAAW,EAAC,SAAS,EAAE,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC;YAE7D,MAAM,CAAC,MAAM,CAAC,CAAC,aAAa,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QAClD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,iDAAiD,EAAE,KAAK,IAAI,EAAE;YAC/D,MAAM,MAAM,CAAC,GAAG,EAAE,CAAC,IAAA,yBAAW,EAAC,SAAS,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAC5E,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAChD,CAAC;QACJ,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,SAAS,EAAE,GAAG,EAAE;QACvB,EAAE,CAAC,yDAAyD,EAAE,KAAK,IAAI,EAAE;YACvE,MAAM,MAAM,GAAG,MAAM,IAAA,yBAAW,EAAC,SAAS,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC;YAEhE,MAAM,CAAC,MAAM,CAAC,CAAC,aAAa,CAAC,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC,CAAC;QACtD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,gCAAgC,EAAE,KAAK,IAAI,EAAE;YAC9C,MAAM,MAAM,GAAG,MAAM,IAAA,yBAAW,EAAC,SAAS,EAAE,EAAE,MAAM,EAAE,GAAG,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC;YAE7E,MAAM,CAAC,MAAM,CAAC,CAAC,aAAa,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC;QAC3C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,oCAAoC,EAAE,KAAK,IAAI,EAAE;YAClD,MAAM,MAAM,GAAG,MAAM,IAAA,yBAAW,EAAC,SAAS,EAAE,EAAE,MAAM,EAAE,GAAG,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC;YAE7E,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACjC,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,QAAQ,EAAE,GAAG,EAAE;QACtB,EAAE,CAAC,oDAAoD,EAAE,KAAK,IAAI,EAAE;YAClE,MAAM,MAAM,GAAG,MAAM,IAAA,yBAAW,EAAC,SAAS,EAAE,EAAE,MAAM,EAAE,CAAC,EAAE,CAAC,CAAC;YAE3D,MAAM,CAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;YAC/B,kDAAkD;YAClD,MAAM,EAAE,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE;gBACvB,MAAM,CAAC,KAAK,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;YAChC,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,0BAA0B,EAAE,KAAK,IAAI,EAAE;YACxC,MAAM,MAAM,GAAG,MAAM,IAAA,yBAAW,EAAC,SAAS,EAAE,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC;YAE7D,MAAM,CAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QACjC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,iDAAiD,EAAE,KAAK,IAAI,EAAE;YAC/D,MAAM,MAAM,CAAC,GAAG,EAAE,CAAC,IAAA,yBAAW,EAAC,SAAS,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAC5E,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAChD,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,uDAAuD,EAAE,KAAK,IAAI,EAAE;YACrE,MAAM,MAAM,GAAG,MAAM,IAAA,yBAAW,EAAC,SAAS,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,CAAC;YAE5D,MAAM,CAAC,MAAM,CAAC,CAAC,YAAY,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;YAC1C,MAAM,EAAE,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE;gBACvB,MAAM,CAAC,KAAK,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;YAChC,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,iCAAiC,EAAE,KAAK,IAAI,EAAE;YAC/C,MAAM,MAAM,GAAG,MAAM,IAAA,yBAAW,EAAC,SAAS,EAAE,EAAE,MAAM,EAAE,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC;YAE3E,MAAM,CAAC,MAAM,CAAC,CAAC,WAAW,EAAE,CAAC;YAC7B,MAAM,CAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;YAC/B,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QACjD,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}