promptfoo 0.77.0 → 0.78.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (176) hide show
  1. package/README.md +5 -1
  2. package/dist/package.json +10 -10
  3. package/dist/src/assertions.js +1 -1
  4. package/dist/src/assertions.js.map +1 -1
  5. package/dist/src/commands/delete.js +1 -1
  6. package/dist/src/commands/delete.js.map +1 -1
  7. package/dist/src/commands/eval.d.ts.map +1 -1
  8. package/dist/src/commands/eval.js +19 -9
  9. package/dist/src/commands/eval.js.map +1 -1
  10. package/dist/src/commands/generate/redteam.d.ts.map +1 -1
  11. package/dist/src/commands/generate/redteam.js +4 -1
  12. package/dist/src/commands/generate/redteam.js.map +1 -1
  13. package/dist/src/commands/redteam.d.ts.map +1 -1
  14. package/dist/src/commands/redteam.js +93 -77
  15. package/dist/src/commands/redteam.js.map +1 -1
  16. package/dist/src/commands/share.js +1 -1
  17. package/dist/src/commands/share.js.map +1 -1
  18. package/dist/src/config.d.ts.map +1 -1
  19. package/dist/src/config.js +12 -0
  20. package/dist/src/config.js.map +1 -1
  21. package/dist/src/database/tables.d.ts +1 -0
  22. package/dist/src/database/tables.d.ts.map +1 -1
  23. package/dist/src/evaluator.d.ts.map +1 -1
  24. package/dist/src/evaluator.js +13 -1
  25. package/dist/src/evaluator.js.map +1 -1
  26. package/dist/src/evaluatorHelpers.d.ts +8 -0
  27. package/dist/src/evaluatorHelpers.d.ts.map +1 -1
  28. package/dist/src/evaluatorHelpers.js +28 -0
  29. package/dist/src/evaluatorHelpers.js.map +1 -1
  30. package/dist/src/integrations/helicone.d.ts +11 -0
  31. package/dist/src/integrations/helicone.d.ts.map +1 -0
  32. package/dist/src/integrations/helicone.js +68 -0
  33. package/dist/src/integrations/helicone.js.map +1 -0
  34. package/dist/src/onboarding.d.ts +1 -1
  35. package/dist/src/onboarding.d.ts.map +1 -1
  36. package/dist/src/onboarding.js +3 -1
  37. package/dist/src/onboarding.js.map +1 -1
  38. package/dist/src/prompts/utils.js +1 -1
  39. package/dist/src/prompts/utils.js.map +1 -1
  40. package/dist/src/providers/anthropic.d.ts.map +1 -1
  41. package/dist/src/providers/anthropic.js +2 -1
  42. package/dist/src/providers/anthropic.js.map +1 -1
  43. package/dist/src/providers/azureopenai.d.ts.map +1 -1
  44. package/dist/src/providers/azureopenai.js +2 -2
  45. package/dist/src/providers/azureopenai.js.map +1 -1
  46. package/dist/src/providers/bedrock.d.ts.map +1 -1
  47. package/dist/src/providers/bedrock.js +3 -2
  48. package/dist/src/providers/bedrock.js.map +1 -1
  49. package/dist/src/providers/openai.d.ts.map +1 -1
  50. package/dist/src/providers/openai.js +4 -3
  51. package/dist/src/providers/openai.js.map +1 -1
  52. package/dist/src/providers/shared.d.ts +0 -1
  53. package/dist/src/providers/shared.d.ts.map +1 -1
  54. package/dist/src/providers/shared.js +0 -53
  55. package/dist/src/providers/shared.js.map +1 -1
  56. package/dist/src/python/pythonUtils.d.ts.map +1 -1
  57. package/dist/src/python/pythonUtils.js +8 -8
  58. package/dist/src/python/pythonUtils.js.map +1 -1
  59. package/dist/src/python/wrapper.py +3 -1
  60. package/dist/src/redteam/index.d.ts +1 -1
  61. package/dist/src/redteam/index.d.ts.map +1 -1
  62. package/dist/src/redteam/index.js +5 -2
  63. package/dist/src/redteam/index.js.map +1 -1
  64. package/dist/src/redteam/plugins/base.d.ts +4 -1
  65. package/dist/src/redteam/plugins/base.d.ts.map +1 -1
  66. package/dist/src/redteam/plugins/base.js +30 -9
  67. package/dist/src/redteam/plugins/base.js.map +1 -1
  68. package/dist/src/redteam/plugins/harmful.d.ts.map +1 -1
  69. package/dist/src/redteam/plugins/harmful.js +3 -3
  70. package/dist/src/redteam/plugins/harmful.js.map +1 -1
  71. package/dist/src/redteam/plugins/index.d.ts.map +1 -1
  72. package/dist/src/redteam/plugins/index.js +15 -19
  73. package/dist/src/redteam/plugins/index.js.map +1 -1
  74. package/dist/src/redteam/providers/constants.d.ts +4 -0
  75. package/dist/src/redteam/providers/constants.d.ts.map +1 -0
  76. package/dist/src/redteam/providers/constants.js +9 -0
  77. package/dist/src/redteam/providers/constants.js.map +1 -0
  78. package/dist/src/redteam/providers/iterative.d.ts +1 -0
  79. package/dist/src/redteam/providers/iterative.d.ts.map +1 -1
  80. package/dist/src/redteam/providers/iterative.js +13 -83
  81. package/dist/src/redteam/providers/iterative.js.map +1 -1
  82. package/dist/src/redteam/providers/iterativeImage.d.ts.map +1 -1
  83. package/dist/src/redteam/providers/iterativeImage.js +3 -26
  84. package/dist/src/redteam/providers/iterativeImage.js.map +1 -1
  85. package/dist/src/redteam/providers/iterativeTree.d.ts +161 -14
  86. package/dist/src/redteam/providers/iterativeTree.d.ts.map +1 -1
  87. package/dist/src/redteam/providers/iterativeTree.js +343 -328
  88. package/dist/src/redteam/providers/iterativeTree.js.map +1 -1
  89. package/dist/src/redteam/providers/prompts.d.ts +4 -0
  90. package/dist/src/redteam/providers/prompts.d.ts.map +1 -0
  91. package/dist/src/redteam/providers/prompts.js +77 -0
  92. package/dist/src/redteam/providers/prompts.js.map +1 -0
  93. package/dist/src/types/index.d.ts +10 -0
  94. package/dist/src/types/index.d.ts.map +1 -1
  95. package/dist/src/types/index.js +4 -0
  96. package/dist/src/types/index.js.map +1 -1
  97. package/dist/src/types/redteam.d.ts +3 -0
  98. package/dist/src/types/redteam.d.ts.map +1 -1
  99. package/dist/src/util/index.d.ts +14 -1
  100. package/dist/src/util/index.d.ts.map +1 -1
  101. package/dist/src/util/index.js +63 -26
  102. package/dist/src/util/index.js.map +1 -1
  103. package/dist/src/util/templates.d.ts +1 -3
  104. package/dist/src/util/templates.d.ts.map +1 -1
  105. package/dist/src/util/templates.js.map +1 -1
  106. package/dist/src/util/transform.d.ts +9 -8
  107. package/dist/src/util/transform.d.ts.map +1 -1
  108. package/dist/src/util/transform.js +66 -16
  109. package/dist/src/util/transform.js.map +1 -1
  110. package/dist/src/validators/redteam.d.ts +12 -5
  111. package/dist/src/validators/redteam.d.ts.map +1 -1
  112. package/dist/src/validators/redteam.js +6 -3
  113. package/dist/src/validators/redteam.js.map +1 -1
  114. package/dist/src/web/nextui/404/index.html +1 -1
  115. package/dist/src/web/nextui/404.html +1 -1
  116. package/dist/src/web/nextui/_next/static/chunks/585-8b9b2c41f03d3ab8.js +1 -0
  117. package/dist/src/web/nextui/_next/static/chunks/app/{layout-3e4716f7ebdd0dab.js → layout-47ed2a4410747d7e.js} +1 -1
  118. package/dist/src/web/nextui/_next/static/chunks/app/report/page-1da24ad6423ea2a5.js +1 -0
  119. package/dist/src/web/nextui/_next/static/chunks/main-app-0dc90092cd7acaaf.js +1 -0
  120. package/dist/src/web/nextui/auth/login/index.html +1 -1
  121. package/dist/src/web/nextui/auth/login/index.txt +2 -2
  122. package/dist/src/web/nextui/auth/signup/index.html +1 -1
  123. package/dist/src/web/nextui/auth/signup/index.txt +2 -2
  124. package/dist/src/web/nextui/datasets/index.html +1 -1
  125. package/dist/src/web/nextui/datasets/index.txt +2 -2
  126. package/dist/src/web/nextui/eval/index.html +1 -1
  127. package/dist/src/web/nextui/eval/index.txt +3 -3
  128. package/dist/src/web/nextui/index.html +1 -1
  129. package/dist/src/web/nextui/index.txt +2 -2
  130. package/dist/src/web/nextui/progress/index.html +1 -1
  131. package/dist/src/web/nextui/progress/index.txt +2 -2
  132. package/dist/src/web/nextui/prompts/index.html +1 -1
  133. package/dist/src/web/nextui/prompts/index.txt +2 -2
  134. package/dist/src/web/nextui/report/index.html +1 -1
  135. package/dist/src/web/nextui/report/index.txt +3 -3
  136. package/dist/src/web/nextui/setup/index.html +1 -1
  137. package/dist/src/web/nextui/setup/index.txt +2 -2
  138. package/dist/src/web/server.js +2 -2
  139. package/dist/src/web/server.js.map +1 -1
  140. package/dist/test/config.test.js +60 -0
  141. package/dist/test/config.test.js.map +1 -1
  142. package/dist/test/evaluator.test.js +57 -115
  143. package/dist/test/evaluator.test.js.map +1 -1
  144. package/dist/test/evaluatorHelpers.test.d.ts +2 -0
  145. package/dist/test/evaluatorHelpers.test.d.ts.map +1 -0
  146. package/dist/test/evaluatorHelpers.test.js +180 -0
  147. package/dist/test/evaluatorHelpers.test.js.map +1 -0
  148. package/dist/test/is-sql-tests/node-sql-parser.test.js +10 -10
  149. package/dist/test/is-sql-tests/node-sql-parser.test.js.map +1 -1
  150. package/dist/test/prompts.utils.test.js +3 -0
  151. package/dist/test/prompts.utils.test.js.map +1 -1
  152. package/dist/test/pythonWrapper.test.js +53 -20
  153. package/dist/test/pythonWrapper.test.js.map +1 -1
  154. package/dist/test/redteam/plugins/base.test.js +16 -31
  155. package/dist/test/redteam/plugins/base.test.js.map +1 -1
  156. package/dist/test/redteam/providers/iterativeTree.test.d.ts +2 -0
  157. package/dist/test/redteam/providers/iterativeTree.test.d.ts.map +1 -0
  158. package/dist/test/redteam/providers/iterativeTree.test.js +315 -0
  159. package/dist/test/redteam/providers/iterativeTree.test.js.map +1 -0
  160. package/dist/test/redteam/validators.test.js +15 -0
  161. package/dist/test/redteam/validators.test.js.map +1 -1
  162. package/dist/test/util.test.js +89 -0
  163. package/dist/test/util.test.js.map +1 -1
  164. package/dist/test/util.transform.test.js +41 -1
  165. package/dist/test/util.transform.test.js.map +1 -1
  166. package/package.json +10 -10
  167. package/dist/src/web/nextui/_next/static/chunks/944-95d6fadc7a5d7fb2.js +0 -1
  168. package/dist/src/web/nextui/_next/static/chunks/app/report/page-c630a8d0cba86664.js +0 -1
  169. package/dist/src/web/nextui/_next/static/chunks/main-app-4398f7af7ad8387f.js +0 -1
  170. package/dist/test/providers.shared.test.d.ts +0 -2
  171. package/dist/test/providers.shared.test.d.ts.map +0 -1
  172. package/dist/test/providers.shared.test.js +0 -112
  173. package/dist/test/providers.shared.test.js.map +0 -1
  174. /package/dist/src/web/nextui/_next/static/{dLlNz9-V4NvRIcKyU_b89 → FBiShPpcI4BVY3fwT1B5q}/_buildManifest.js +0 -0
  175. /package/dist/src/web/nextui/_next/static/{dLlNz9-V4NvRIcKyU_b89 → FBiShPpcI4BVY3fwT1B5q}/_ssgManifest.js +0 -0
  176. /package/dist/src/web/nextui/_next/static/chunks/{258-e4da2950edde8e2a.js → 258-b2e321ee0d0a5055.js} +0 -0
package/dist/test/providers.shared.test.js.map DELETED
@@ -1 +0,0 @@
1
- {"version":3,"file":"providers.shared.test.js","sourceRoot":"","sources":["../../test/providers.shared.test.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,uCAAyB;AACzB,2CAA6B;AAC7B,+DAAuC;AACvC,oDAAoE;AAEpE,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAEhB,QAAQ,CAAC,2BAA2B,EAAE,GAAG,EAAE;IACzC,MAAM,eAAe,GAAG,cAAc,CAAC;IACvC,MAAM,eAAe,GAAG,kBAAkB,CAAC;IAC3C,MAAM,eAAe,GAAG,YAAY,CAAC;IAErC,UAAU,CAAC,GAAG,EAAE;QACd,IAAI,CAAC,aAAa,EAAE,CAAC;QACrB,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,UAAU,CAAC,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC;QACjD,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,YAAY,CAAC,CAAC,eAAe,CAAC,eAAe,CAAC,CAAC;IAChE,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,+CAA+C,EAAE,GAAG,EAAE;QACvD,MAAM,KAAK,GAAG,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC;QAC/B,MAAM,CAAC,IAAA,kCAAyB,EAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACvD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,6DAA6D,EAAE,GAAG,EAAE;QACrE,MAAM,KAAK,GAAG,iBAAiB,CAAC;QAChC,MAAM,CAAC,IAAA,kCAAyB,EAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACvD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,kDAAkD,EAAE,GAAG,EAAE;QAC1D,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,UAAU,CAAC,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;QAClD,MAAM,CAAC,GAAG,EAAE,CAAC,IAAA,kCAAyB,EAAC,wBAAwB,CAAC,CAAC,CAAC,OAAO,CACvE,qBAAqB,CACtB,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,+DAA+D,EAAE,GAAG,EAAE;QACvE,MAAM,CAAC,IAAA,kCAAyB,EAAC,iBAAiB,CAAC,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;IAC7E,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,uDAAuD,EAAE,GAAG,EAAE;QAC/D,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,YAAY,CAAC,CAAC,eAAe,CAAC,eAAe,CAAC,CAAC;QAC9D,MAAM,CAAC,IAAA,kCAAyB,EAAC,kBAAkB,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC,CAAC;IAClF,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,uDAAuD,EAAE,GAAG,EAAE;QAC/D,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,YAAY,CAAC,CAAC,eAAe,CAAC,eAAe,CAAC,CAAC;QAC9D,MAAM,CAAC,IAAA,kCAAyB,EAAC,kBAAkB,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC,CAAC;IAClF,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,sDAAsD,EAAE,GAAG,EAAE;QAC9D,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,YAAY,CAAC,CAAC,eAAe,CAAC,eAAe,CAAC,CAAC;QAC9D,MAAM,CAAC,IAAA,kCAAyB,EAAC,iBAAiB,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC,CAAC;IACjF,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,+CAA+C,EAAE,GAAG,EAAE;QACvD,MAAM,QAAQ,GAAG,YAAY,CAAC;QAC9B,kBAAQ,CAAC,QAAQ,GAAG,QAAQ,CAAC;QAC7B,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,YAAY,CAAC,CAAC,eAAe,CAAC,eAAe,CAAC,CAAC;QAE9D,IAAA,kCAAyB,EAAC,iBAAiB,CAAC,CAAC;QAE7C,MAAM,YAAY,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;QACxD,MAAM,CAAC,EAAE,CAAC,UAAU,CAAC,CAAC,oBAAoB,CAAC,YAAY,CAAC,CAAC;QACzD,MAAM,CAAC,EAAE,CAAC,YAAY,CAAC,CAAC,oBAAoB,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC;QAEnE,kBAAQ,CAAC,QAAQ,GAAG,SAAS,CAAC;IAChC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,wCAAwC,EAAE,GAAG,EAAE;QAChD,MAAM,QAAQ,GAAG,iBAAiB,CAAC;QACnC,kBAAQ,CAAC,QAAQ,GAAG,QAAQ,CAAC;QAC7B,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,YAAY,CAAC,CAAC,eAAe,CAAC,eAAe,CAAC,CAAC;QAE9D,IAAA,kCAAyB,EAAC,iBAAiB,CAAC,CAAC;QAE7C,MAAM,YAAY,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;QACxD,MAAM,CAAC,EAAE,CAAC,UAAU,CAAC,CAAC,oBAAoB,CAAC,YAAY,CAAC,CAAC;QACzD,MAAM,CAAC,EAAE,CAAC,YAAY,CAAC,CAAC,oBAAoB,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC;QAEnE,kBAAQ,CAAC,QAAQ,GAAG,SAAS,CAAC;IAChC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,mDAAmD,EAAE,GAAG,EAAE;QAC3D,MAAM,QAAQ,GAAG,YAAY,CAAC;QAC9B,kBAAQ,CAAC,QAAQ,GAAG,QAAQ,CAAC;QAC7B,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,YAAY,CAAC,CAAC,eAAe,CAAC,eAAe,CAAC,CAAC;QAE9D,IAAA,kCAAyB,EAAC,gCAAgC,CAAC,CAAC;QAE5D,MAAM,YAAY,GAAG,IAAI,CAAC,OAAO,CAAC,yBAAyB,CAAC,CAAC;QAC7D,MAAM,CAAC,EAAE,CAAC,UAAU,CAAC,CAAC,oBAAoB,CAAC,YAAY,CAAC,CAAC;QACzD,MAAM,CAAC,EAAE,CAAC,YAAY,CAAC,CAAC,oBAAoB,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC;QAEnE,kBAAQ,CAAC,QAAQ,GAAG,SAAS,CAAC;IAChC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,6BAA6B,EAAE,GAAG,EAAE;QACrC,MAAM,QAAQ,GAAG,iBAAiB,CAAC;QACnC,kBAAQ,CAAC,QAAQ,GAAG,QAAQ,CAAC;QAC7B,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,YAAY,CAAC,CAAC,eAAe,CAAC,eAAe,CAAC,CAAC;QAE9D,IAAA,kCAAyB,EAAC,CAAC,kBAAkB,EAAE,kBAAkB,EAAE,kBAAkB,CAAC,CAAC,CAAC;QAExF,MAAM,CAAC,EAAE,CAAC,UAAU,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC;QAC/C,MAAM,CAAC,EAAE,CAAC,UAAU,CAAC,CAAC,uBAAuB,CAAC,CAAC,EAAE,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,WAAW,CAAC,CAAC,CAAC;QACtF,MAAM,CAAC,EAAE,CAAC,UAAU,CAAC,CAAC,uBAAuB,CAAC,CAAC,EAAE,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,WAAW,CAAC,CAAC,CAAC;QACtF,MAAM,CAAC,EAAE,CAAC,UAAU,CAAC,CAAC,uBAAuB,CAAC,CAAC,EAAE,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,WAAW,CAAC,CAAC,CAAC;QAEtF,kBAAQ,CAAC,QAAQ,GAAG,SAAS,CAAC;IAChC,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}