promptfoo 0.91.3 → 0.92.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (181) hide show
  1. package/dist/drizzle/0006_harsh_caretaker.sql +42 -0
  2. package/dist/drizzle/0007_cloudy_wong.sql +1 -0
  3. package/dist/drizzle/meta/0006_snapshot.json +721 -0
  4. package/dist/drizzle/meta/0007_snapshot.json +723 -0
  5. package/dist/drizzle/meta/_journal.json +14 -0
  6. package/dist/package.json +10 -8
  7. package/dist/src/app/assets/{index-C6z1nbLN.js → index-BpjzEMiv.js} +243 -241
  8. package/dist/src/app/assets/{index.es-oqbvfIxR.js → index.es-ihzvEu35.js} +1 -1
  9. package/dist/src/app/assets/{sync-D2s75VlC.js → sync-BosjlpGJ.js} +1 -1
  10. package/dist/src/app/index.html +3 -3
  11. package/dist/src/assertions.js +2 -2
  12. package/dist/src/assertions.js.map +1 -1
  13. package/dist/src/commands/cache.d.ts.map +1 -1
  14. package/dist/src/commands/cache.js +0 -2
  15. package/dist/src/commands/cache.js.map +1 -1
  16. package/dist/src/commands/eval.d.ts.map +1 -1
  17. package/dist/src/commands/eval.js +19 -16
  18. package/dist/src/commands/eval.js.map +1 -1
  19. package/dist/src/commands/export.d.ts.map +1 -1
  20. package/dist/src/commands/export.js +8 -31
  21. package/dist/src/commands/export.js.map +1 -1
  22. package/dist/src/commands/import.d.ts.map +1 -1
  23. package/dist/src/commands/import.js +52 -13
  24. package/dist/src/commands/import.js.map +1 -1
  25. package/dist/src/commands/list.d.ts.map +1 -1
  26. package/dist/src/commands/list.js +35 -7
  27. package/dist/src/commands/list.js.map +1 -1
  28. package/dist/src/commands/share.d.ts +2 -2
  29. package/dist/src/commands/share.d.ts.map +1 -1
  30. package/dist/src/commands/share.js +12 -13
  31. package/dist/src/commands/share.js.map +1 -1
  32. package/dist/src/commands/show.d.ts.map +1 -1
  33. package/dist/src/commands/show.js +10 -6
  34. package/dist/src/commands/show.js.map +1 -1
  35. package/dist/src/constants.d.ts +1 -0
  36. package/dist/src/constants.d.ts.map +1 -1
  37. package/dist/src/constants.js +2 -1
  38. package/dist/src/constants.js.map +1 -1
  39. package/dist/src/database/index.js +1 -1
  40. package/dist/src/database/index.js.map +1 -1
  41. package/dist/src/database/tables.d.ts +609 -11
  42. package/dist/src/database/tables.d.ts.map +1 -1
  43. package/dist/src/database/tables.js +111 -52
  44. package/dist/src/database/tables.js.map +1 -1
  45. package/dist/src/database/types.d.ts +3 -3
  46. package/dist/src/database/types.d.ts.map +1 -1
  47. package/dist/src/evaluator.d.ts +3 -2
  48. package/dist/src/evaluator.d.ts.map +1 -1
  49. package/dist/src/evaluator.js +75 -104
  50. package/dist/src/evaluator.js.map +1 -1
  51. package/dist/src/evaluatorHelpers.d.ts.map +1 -1
  52. package/dist/src/evaluatorHelpers.js +2 -1
  53. package/dist/src/evaluatorHelpers.js.map +1 -1
  54. package/dist/src/index.d.ts +2 -1
  55. package/dist/src/index.d.ts.map +1 -1
  56. package/dist/src/index.js +18 -10
  57. package/dist/src/index.js.map +1 -1
  58. package/dist/src/models/eval.d.ts +95 -0
  59. package/dist/src/models/eval.d.ts.map +1 -0
  60. package/dist/src/models/eval.js +390 -0
  61. package/dist/src/models/eval.js.map +1 -0
  62. package/dist/src/models/evalResult.d.ts +50 -0
  63. package/dist/src/models/evalResult.d.ts.map +1 -0
  64. package/dist/src/models/evalResult.js +122 -0
  65. package/dist/src/models/evalResult.js.map +1 -0
  66. package/dist/src/models/provider.d.ts +9 -0
  67. package/dist/src/models/provider.d.ts.map +1 -0
  68. package/dist/src/models/provider.js +47 -0
  69. package/dist/src/models/provider.js.map +1 -0
  70. package/dist/src/prompts/index.d.ts.map +1 -1
  71. package/dist/src/prompts/index.js +2 -1
  72. package/dist/src/prompts/index.js.map +1 -1
  73. package/dist/src/prompts/utils.d.ts +1 -0
  74. package/dist/src/prompts/utils.d.ts.map +1 -1
  75. package/dist/src/prompts/utils.js +7 -0
  76. package/dist/src/prompts/utils.js.map +1 -1
  77. package/dist/src/providers/fal.d.ts +2 -2
  78. package/dist/src/providers/fal.d.ts.map +1 -1
  79. package/dist/src/providers/fal.js +2 -1
  80. package/dist/src/providers/fal.js.map +1 -1
  81. package/dist/src/providers/http.js +2 -2
  82. package/dist/src/providers/http.js.map +1 -1
  83. package/dist/src/providers/palm.d.ts +4 -3
  84. package/dist/src/providers/palm.d.ts.map +1 -1
  85. package/dist/src/providers/palm.js +13 -3
  86. package/dist/src/providers/palm.js.map +1 -1
  87. package/dist/src/providers.js +5 -5
  88. package/dist/src/providers.js.map +1 -1
  89. package/dist/src/redteam/eval/excessive-agency/llm_rubric-20240617.json +10 -0
  90. package/dist/src/redteam/eval/excessive-agency/llm_rubric-20240618.json +10 -0
  91. package/dist/src/redteam/eval/harmful/llm_rubric-20240723.json +10 -0
  92. package/dist/src/redteam/eval/harmful/llm_rubric-20240724.json +10 -0
  93. package/dist/src/server/server.d.ts +1 -0
  94. package/dist/src/server/server.d.ts.map +1 -1
  95. package/dist/src/server/server.js +70 -31
  96. package/dist/src/server/server.js.map +1 -1
  97. package/dist/src/share.d.ts +2 -2
  98. package/dist/src/share.d.ts.map +1 -1
  99. package/dist/src/share.js +93 -34
  100. package/dist/src/share.js.map +1 -1
  101. package/dist/src/table.d.ts +2 -2
  102. package/dist/src/table.d.ts.map +1 -1
  103. package/dist/src/table.js +3 -3
  104. package/dist/src/table.js.map +1 -1
  105. package/dist/src/types/index.d.ts +163 -11
  106. package/dist/src/types/index.d.ts.map +1 -1
  107. package/dist/src/types/index.js +21 -1
  108. package/dist/src/types/index.js.map +1 -1
  109. package/dist/src/util/config/load.d.ts.map +1 -1
  110. package/dist/src/util/config/load.js +2 -1
  111. package/dist/src/util/config/load.js.map +1 -1
  112. package/dist/src/util/config/manage.d.ts.map +1 -1
  113. package/dist/src/util/config/manage.js.map +1 -1
  114. package/dist/src/util/convertEvalResultsToTable.d.ts +16 -0
  115. package/dist/src/util/convertEvalResultsToTable.d.ts.map +1 -0
  116. package/dist/src/util/convertEvalResultsToTable.js +136 -0
  117. package/dist/src/util/convertEvalResultsToTable.js.map +1 -0
  118. package/dist/src/util/createHash.d.ts +1 -0
  119. package/dist/src/util/createHash.d.ts.map +1 -1
  120. package/dist/src/util/createHash.js +9 -0
  121. package/dist/src/util/createHash.js.map +1 -1
  122. package/dist/src/util/file.d.ts +8 -0
  123. package/dist/src/util/file.d.ts.map +1 -0
  124. package/dist/src/util/file.js +13 -0
  125. package/dist/src/util/file.js.map +1 -0
  126. package/dist/src/util/index.d.ts +9 -14
  127. package/dist/src/util/index.d.ts.map +1 -1
  128. package/dist/src/util/index.js +132 -268
  129. package/dist/src/util/index.js.map +1 -1
  130. package/dist/src/util/time.d.ts +2 -0
  131. package/dist/src/util/time.d.ts.map +1 -0
  132. package/dist/src/util/time.js +7 -0
  133. package/dist/src/util/time.js.map +1 -0
  134. package/dist/src/util/transform.js +2 -2
  135. package/dist/src/util/transform.js.map +1 -1
  136. package/dist/src/validators/providers.d.ts +6 -0
  137. package/dist/src/validators/providers.d.ts.map +1 -1
  138. package/dist/src/validators/providers.js +1 -0
  139. package/dist/src/validators/providers.js.map +1 -1
  140. package/dist/src/validators/redteam.d.ts +6 -0
  141. package/dist/src/validators/redteam.d.ts.map +1 -1
  142. package/dist/test/commands/eval/filterFailingTests.test.js +24 -2
  143. package/dist/test/commands/eval/filterFailingTests.test.js.map +1 -1
  144. package/dist/test/evaluator.test.js +152 -74
  145. package/dist/test/evaluator.test.js.map +1 -1
  146. package/dist/test/factories/data/eval/database_records.d.ts +142 -0
  147. package/dist/test/factories/data/eval/database_records.d.ts.map +1 -0
  148. package/dist/test/factories/data/eval/database_records.js +251 -0
  149. package/dist/test/factories/data/eval/database_records.js.map +1 -0
  150. package/dist/test/factories/evalFactory.d.ts +768 -0
  151. package/dist/test/factories/evalFactory.d.ts.map +1 -0
  152. package/dist/test/factories/evalFactory.js +121 -0
  153. package/dist/test/factories/evalFactory.js.map +1 -0
  154. package/dist/test/index.test.js +20 -35
  155. package/dist/test/index.test.js.map +1 -1
  156. package/dist/test/models/eval.test.d.ts +2 -0
  157. package/dist/test/models/eval.test.d.ts.map +1 -0
  158. package/dist/test/models/eval.test.js +34 -0
  159. package/dist/test/models/eval.test.js.map +1 -0
  160. package/dist/test/providers.test.js +3 -3
  161. package/dist/test/providers.test.js.map +1 -1
  162. package/dist/test/server/share.test.d.ts +2 -0
  163. package/dist/test/server/share.test.d.ts.map +1 -0
  164. package/dist/test/server/share.test.js +36 -0
  165. package/dist/test/server/share.test.js.map +1 -0
  166. package/dist/test/server/v3evalToShare.json +507 -0
  167. package/dist/test/server/v4evalToShare.json +421 -0
  168. package/dist/test/types.test.js +56 -3
  169. package/dist/test/types.test.js.map +1 -1
  170. package/dist/test/util.file.test.d.ts +2 -0
  171. package/dist/test/util.file.test.d.ts.map +1 -0
  172. package/dist/test/util.file.test.js +32 -0
  173. package/dist/test/util.file.test.js.map +1 -0
  174. package/dist/test/util.listPrevious.test.d.ts +2 -0
  175. package/dist/test/util.listPrevious.test.d.ts.map +1 -0
  176. package/dist/test/util.listPrevious.test.js +37 -0
  177. package/dist/test/util.listPrevious.test.js.map +1 -0
  178. package/dist/test/util.test.js +38 -311
  179. package/dist/test/util.test.js.map +1 -1
  180. package/dist/tsconfig.tsbuildinfo +1 -0
  181. package/package.json +10 -8
package/dist/src/table.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"table.js","sourceRoot":"","sources":["../../src/table.ts"],"names":[],"mappings":";;;;;AAYA,sCAsCC;AAED,8BAeC;AAnED,kDAA0B;AAC1B,4DAA+B;AAC/B,2CAAiD;AAGjD,SAAS,SAAS,CAAC,GAAW,EAAE,MAAc;IAC5C,IAAI,GAAG,CAAC,MAAM,GAAG,MAAM,EAAE,CAAC;QACxB,OAAO,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,MAAM,GAAG,CAAC,CAAC,GAAG,KAAK,CAAC;IAC1C,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,SAAgB,aAAa,CAAC,OAAwB,EAAE,kBAAkB,GAAG,GAAG,EAAE,OAAO,GAAG,EAAE;IAC5F,MAAM,IAAI,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC;IAChC,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC;IAC1D,MAAM,KAAK,GAAG,IAAI,oBAAK,CAAC;QACtB,IAAI,EAAE;YACJ,GAAG,IAAI,CAAC,IAAI;YACZ,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI,MAAM,CAAC,QAAQ,KAAK,MAAM,CAAC,KAAK,EAAE,CAAC;SACxE,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,SAAS,CAAC,CAAC,EAAE,kBAAkB,CAAC,CAAC;QAC9C,SAAS,EAAE,KAAK,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,8BAAkB,GAAG,UAAU,CAAC,CAAC;QAC9E,QAAQ,EAAE,IAAI;QACd,kBAAkB,EAAE,IAAI,EAAE,8BAA8B;QACxD,KAAK,EAAE;YACL,IAAI,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC;SACvB;KACF,CAAC,CAAC;IACH,4DAA4D;IAC5D,KAAK,MAAM,GAAG,IAAI,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,EAAE,CAAC;QACvD,KAAK,CAAC,IAAI,CAAC;YACT,GAAG,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,SAAS,CAAC,CAAC,EAAE,kBAAkB,CAAC,CAAC;YACxD,GAAG,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE,EAAE;gBAC3C,IAAI,GAAG,SAAS,CAAC,IAAI,EAAE,kBAAkB,CAAC,CAAC;gBAC3C,IAAI,IAAI,EAAE,CAAC;oBACT,OAAO,eAAK,CAAC,KAAK,CAAC,SAAS,CAAC,GAAG,IAAI,CAAC;gBACvC,CAAC;qBAAM,IAAI,CAAC,IAAI,EAAE,CAAC;oBACjB,sCAAsC;oBACtC,OAAO,CACL,eAAK,CAAC,GAAG,CAAC,SAAS,CAAC;wBACpB,IAAI;6BACD,KAAK,CAAC,KAAK,CAAC;6BACZ,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,eAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;6BACpD,IAAI,CAAC,KAAK,CAAC,CACf,CAAC;gBACJ,CAAC;gBACD,OAAO,IAAI,CAAC;YACd,CAAC,CAAC;SACH,CAAC,CAAC;IACL,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAgB,SAAS,CAAC,IAAuC;IAC/D,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACtB,OAAO,oBAAoB,CAAC;IAC9B,CAAC;IACD,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IAClC,MAAM,KAAK,GAAG,IAAI,oBAAK,CAAC;QACtB,IAAI;QACJ,SAAS,EAAE,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,8BAAkB,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC;QAChF,QAAQ,EAAE,IAAI;QACd,kBAAkB,EAAE,IAAI;KACzB,CAAC,CAAC;IACH,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;IACjC,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC"}
1
+ {"version":3,"file":"table.js","sourceRoot":"","sources":["../../src/table.ts"],"names":[],"mappings":";;;;;AAYA,sCA0CC;AAED,8BAeC;AAvED,kDAA0B;AAC1B,4DAA+B;AAC/B,2CAAiD;AAGjD,SAAS,SAAS,CAAC,GAAW,EAAE,MAAc;IAC5C,IAAI,GAAG,CAAC,MAAM,GAAG,MAAM,EAAE,CAAC;QACxB,OAAO,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,MAAM,GAAG,CAAC,CAAC,GAAG,KAAK,CAAC;IAC1C,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,SAAgB,aAAa,CAC3B,aAA4B,EAC5B,kBAAkB,GAAG,GAAG,EACxB,OAAO,GAAG,EAAE;IAEZ,MAAM,IAAI,GAAG,aAAa,CAAC,IAAI,CAAC;IAChC,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC;IAC1D,MAAM,KAAK,GAAG,IAAI,oBAAK,CAAC;QACtB,IAAI,EAAE;YACJ,GAAG,IAAI,CAAC,IAAI;YACZ,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI,MAAM,CAAC,QAAQ,KAAK,MAAM,CAAC,KAAK,EAAE,CAAC;SACxE,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,SAAS,CAAC,CAAC,EAAE,kBAAkB,CAAC,CAAC;QAC9C,SAAS,EAAE,KAAK,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,8BAAkB,GAAG,UAAU,CAAC,CAAC;QAC9E,QAAQ,EAAE,IAAI;QACd,kBAAkB,EAAE,IAAI,EAAE,8BAA8B;QACxD,KAAK,EAAE;YACL,IAAI,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC;SACvB;KACF,CAAC,CAAC;IACH,4DAA4D;IAC5D,KAAK,MAAM,GAAG,IAAI,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,EAAE,CAAC;QACvD,KAAK,CAAC,IAAI,CAAC;YACT,GAAG,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,SAAS,CAAC,CAAC,EAAE,kBAAkB,CAAC,CAAC;YACxD,GAAG,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE,EAAE;gBAC3C,IAAI,GAAG,SAAS,CAAC,IAAI,EAAE,kBAAkB,CAAC,CAAC;gBAC3C,IAAI,IAAI,EAAE,CAAC;oBACT,OAAO,eAAK,CAAC,KAAK,CAAC,SAAS,CAAC,GAAG,IAAI,CAAC;gBACvC,CAAC;qBAAM,IAAI,CAAC,IAAI,EAAE,CAAC;oBACjB,sCAAsC;oBACtC,OAAO,CACL,eAAK,CAAC,GAAG,CAAC,SAAS,CAAC;wBACpB,IAAI;6BACD,KAAK,CAAC,KAAK,CAAC;6BACZ,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,eAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;6BACpD,IAAI,CAAC,KAAK,CAAC,CACf,CAAC;gBACJ,CAAC;gBACD,OAAO,IAAI,CAAC;YACd,CAAC,CAAC;SACH,CAAC,CAAC;IACL,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAgB,SAAS,CAAC,IAAuC;IAC/D,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACtB,OAAO,oBAAoB,CAAC;IAC9B,CAAC;IACD,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IAClC,MAAM,KAAK,GAAG,IAAI,oBAAK,CAAC;QACtB,IAAI;QACJ,SAAS,EAAE,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,8BAAkB,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC;QAChF,QAAQ,EAAE,IAAI;QACd,kBAAkB,EAAE,IAAI;KACzB,CAAC,CAAC;IACH,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;IACjC,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC"}