promptfoo 0.112.3 → 0.112.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (182) hide show
  1. package/dist/package.json +39 -33
  2. package/dist/src/app/assets/index-CtILcypo.js +985 -0
  3. package/dist/src/app/assets/{index.es-BGDEHdxg.js → index.es-D1K8VDHD.js} +1 -1
  4. package/dist/src/app/assets/{source-map-support-BR7TbsEY.js → source-map-support-C0aSbjFf.js} +1 -1
  5. package/dist/src/app/assets/{sync-C4JKkKoZ.js → sync-BE-MyvTb.js} +1 -1
  6. package/dist/src/app/index.html +1 -1
  7. package/dist/src/assertions/index.d.ts +1 -1
  8. package/dist/src/assertions/index.d.ts.map +1 -1
  9. package/dist/src/assertions/index.js +2 -4
  10. package/dist/src/assertions/index.js.map +1 -1
  11. package/dist/src/assertions/pi.d.ts +3 -0
  12. package/dist/src/assertions/pi.d.ts.map +1 -0
  13. package/dist/src/assertions/pi.js +15 -0
  14. package/dist/src/assertions/pi.js.map +1 -0
  15. package/dist/src/commands/eval.d.ts.map +1 -1
  16. package/dist/src/commands/eval.js +4 -6
  17. package/dist/src/commands/eval.js.map +1 -1
  18. package/dist/src/commands/init.d.ts.map +1 -1
  19. package/dist/src/commands/init.js +0 -2
  20. package/dist/src/commands/init.js.map +1 -1
  21. package/dist/src/commands/modelScan.d.ts.map +1 -1
  22. package/dist/src/commands/modelScan.js +0 -4
  23. package/dist/src/commands/modelScan.js.map +1 -1
  24. package/dist/src/commands/share.d.ts.map +1 -1
  25. package/dist/src/commands/share.js +3 -6
  26. package/dist/src/commands/share.js.map +1 -1
  27. package/dist/src/commands/view.js +1 -1
  28. package/dist/src/commands/view.js.map +1 -1
  29. package/dist/src/constants.d.ts +5 -4
  30. package/dist/src/constants.d.ts.map +1 -1
  31. package/dist/src/constants.js +26 -12
  32. package/dist/src/constants.js.map +1 -1
  33. package/dist/src/database/tables.d.ts +48 -24
  34. package/dist/src/database/tables.d.ts.map +1 -1
  35. package/dist/src/envars.d.ts +11 -1
  36. package/dist/src/envars.d.ts.map +1 -1
  37. package/dist/src/envars.js +10 -0
  38. package/dist/src/envars.js.map +1 -1
  39. package/dist/src/evaluator.d.ts +1 -1
  40. package/dist/src/evaluator.d.ts.map +1 -1
  41. package/dist/src/evaluator.js +114 -5
  42. package/dist/src/evaluator.js.map +1 -1
  43. package/dist/src/fetch.d.ts +0 -1
  44. package/dist/src/fetch.d.ts.map +1 -1
  45. package/dist/src/fetch.js +17 -13
  46. package/dist/src/fetch.js.map +1 -1
  47. package/dist/src/globalConfig/cloud.d.ts.map +1 -1
  48. package/dist/src/globalConfig/cloud.js +33 -21
  49. package/dist/src/globalConfig/cloud.js.map +1 -1
  50. package/dist/src/guardrails.js +1 -1
  51. package/dist/src/guardrails.js.map +1 -1
  52. package/dist/src/main.d.ts +5 -1
  53. package/dist/src/main.d.ts.map +1 -1
  54. package/dist/src/main.js +65 -4
  55. package/dist/src/main.js.map +1 -1
  56. package/dist/src/matchers.d.ts +4 -1
  57. package/dist/src/matchers.d.ts.map +1 -1
  58. package/dist/src/matchers.js +26 -1
  59. package/dist/src/matchers.js.map +1 -1
  60. package/dist/src/models/eval.d.ts.map +1 -1
  61. package/dist/src/models/eval.js +22 -33
  62. package/dist/src/models/eval.js.map +1 -1
  63. package/dist/src/providers/bedrock.d.ts +4 -3
  64. package/dist/src/providers/bedrock.d.ts.map +1 -1
  65. package/dist/src/providers/bedrock.js +109 -70
  66. package/dist/src/providers/bedrock.js.map +1 -1
  67. package/dist/src/providers/google/ai.studio.d.ts +10 -1
  68. package/dist/src/providers/google/ai.studio.d.ts.map +1 -1
  69. package/dist/src/providers/google/ai.studio.js +63 -1
  70. package/dist/src/providers/google/ai.studio.js.map +1 -1
  71. package/dist/src/providers/google/shared.d.ts.map +1 -1
  72. package/dist/src/providers/google/shared.js +14 -3
  73. package/dist/src/providers/google/shared.js.map +1 -1
  74. package/dist/src/providers/http.d.ts.map +1 -1
  75. package/dist/src/providers/http.js +4 -4
  76. package/dist/src/providers/http.js.map +1 -1
  77. package/dist/src/providers/mcp/client.d.ts.map +1 -1
  78. package/dist/src/providers/mcp/client.js +7 -3
  79. package/dist/src/providers/mcp/client.js.map +1 -1
  80. package/dist/src/providers/promptfoo.d.ts.map +1 -1
  81. package/dist/src/providers/promptfoo.js +2 -1
  82. package/dist/src/providers/promptfoo.js.map +1 -1
  83. package/dist/src/providers/sagemaker.d.ts +1 -5
  84. package/dist/src/providers/sagemaker.d.ts.map +1 -1
  85. package/dist/src/python/pythonUtils.js +1 -1
  86. package/dist/src/python/pythonUtils.js.map +1 -1
  87. package/dist/src/redteam/commands/generate.d.ts.map +1 -1
  88. package/dist/src/redteam/commands/generate.js +1 -6
  89. package/dist/src/redteam/commands/generate.js.map +1 -1
  90. package/dist/src/redteam/commands/init.js +1 -1
  91. package/dist/src/redteam/commands/init.js.map +1 -1
  92. package/dist/src/redteam/commands/poison.d.ts.map +1 -1
  93. package/dist/src/redteam/commands/poison.js +1 -5
  94. package/dist/src/redteam/commands/poison.js.map +1 -1
  95. package/dist/src/redteam/commands/report.js +1 -1
  96. package/dist/src/redteam/commands/report.js.map +1 -1
  97. package/dist/src/redteam/commands/run.d.ts.map +1 -1
  98. package/dist/src/redteam/commands/run.js +1 -39
  99. package/dist/src/redteam/commands/run.js.map +1 -1
  100. package/dist/src/redteam/commands/setup.js +1 -1
  101. package/dist/src/redteam/commands/setup.js.map +1 -1
  102. package/dist/src/redteam/constants.d.ts +5 -5
  103. package/dist/src/redteam/constants.d.ts.map +1 -1
  104. package/dist/src/redteam/constants.js +20 -12
  105. package/dist/src/redteam/constants.js.map +1 -1
  106. package/dist/src/redteam/plugins/index.d.ts.map +1 -1
  107. package/dist/src/redteam/plugins/index.js +5 -1
  108. package/dist/src/redteam/plugins/index.js.map +1 -1
  109. package/dist/src/redteam/plugins/pii.d.ts.map +1 -1
  110. package/dist/src/redteam/plugins/pii.js +5 -0
  111. package/dist/src/redteam/plugins/pii.js.map +1 -1
  112. package/dist/src/redteam/plugins/shellInjection.d.ts.map +1 -1
  113. package/dist/src/redteam/plugins/shellInjection.js +4 -2
  114. package/dist/src/redteam/plugins/shellInjection.js.map +1 -1
  115. package/dist/src/redteam/providers/crescendo/index.d.ts +1 -1
  116. package/dist/src/redteam/providers/crescendo/index.d.ts.map +1 -1
  117. package/dist/src/redteam/providers/crescendo/index.js +1 -1
  118. package/dist/src/redteam/providers/crescendo/index.js.map +1 -1
  119. package/dist/src/redteam/providers/iterative.d.ts.map +1 -1
  120. package/dist/src/redteam/providers/iterative.js +5 -2
  121. package/dist/src/redteam/providers/iterative.js.map +1 -1
  122. package/dist/src/redteam/providers/iterativeTree.d.ts +1 -1
  123. package/dist/src/redteam/providers/iterativeTree.d.ts.map +1 -1
  124. package/dist/src/redteam/providers/iterativeTree.js +3 -3
  125. package/dist/src/redteam/providers/iterativeTree.js.map +1 -1
  126. package/dist/src/redteam/providers/prompts.d.ts.map +1 -1
  127. package/dist/src/redteam/providers/prompts.js +20 -2
  128. package/dist/src/redteam/providers/prompts.js.map +1 -1
  129. package/dist/src/redteam/strategies/index.d.ts.map +1 -1
  130. package/dist/src/redteam/strategies/index.js +12 -2
  131. package/dist/src/redteam/strategies/index.js.map +1 -1
  132. package/dist/src/redteam/strategies/simpleVideo.d.ts +20 -0
  133. package/dist/src/redteam/strategies/simpleVideo.d.ts.map +1 -0
  134. package/dist/src/redteam/strategies/simpleVideo.js +243 -0
  135. package/dist/src/redteam/strategies/simpleVideo.js.map +1 -0
  136. package/dist/src/remoteScoring.d.ts +14 -0
  137. package/dist/src/remoteScoring.d.ts.map +1 -0
  138. package/dist/src/remoteScoring.js +52 -0
  139. package/dist/src/remoteScoring.js.map +1 -0
  140. package/dist/src/server/index.js +1 -1
  141. package/dist/src/server/index.js.map +1 -1
  142. package/dist/src/server/server.d.ts +1 -0
  143. package/dist/src/server/server.d.ts.map +1 -1
  144. package/dist/src/server/server.js +2 -3
  145. package/dist/src/server/server.js.map +1 -1
  146. package/dist/src/share.js +6 -6
  147. package/dist/src/share.js.map +1 -1
  148. package/dist/src/telemetry.d.ts.map +1 -1
  149. package/dist/src/telemetry.js +4 -2
  150. package/dist/src/telemetry.js.map +1 -1
  151. package/dist/src/types/env.d.ts +3 -0
  152. package/dist/src/types/env.d.ts.map +1 -1
  153. package/dist/src/types/env.js +1 -0
  154. package/dist/src/types/env.js.map +1 -1
  155. package/dist/src/types/index.d.ts +712 -1483
  156. package/dist/src/types/index.d.ts.map +1 -1
  157. package/dist/src/types/index.js +5 -0
  158. package/dist/src/types/index.js.map +1 -1
  159. package/dist/src/types/providers.d.ts +10 -1
  160. package/dist/src/types/providers.d.ts.map +1 -1
  161. package/dist/src/types/providers.js.map +1 -1
  162. package/dist/src/types/shared.d.ts +3 -3
  163. package/dist/src/util/cloud.d.ts.map +1 -1
  164. package/dist/src/util/cloud.js +18 -5
  165. package/dist/src/util/cloud.js.map +1 -1
  166. package/dist/src/util/config/load.d.ts.map +1 -1
  167. package/dist/src/util/config/load.js +2 -1
  168. package/dist/src/util/config/load.js.map +1 -1
  169. package/dist/src/util/database.d.ts +6 -4
  170. package/dist/src/util/database.d.ts.map +1 -1
  171. package/dist/src/util/database.js +12 -12
  172. package/dist/src/util/database.js.map +1 -1
  173. package/dist/src/util/server.d.ts.map +1 -1
  174. package/dist/src/util/server.js +2 -2
  175. package/dist/src/util/server.js.map +1 -1
  176. package/dist/src/validators/providers.d.ts +23 -3
  177. package/dist/src/validators/providers.d.ts.map +1 -1
  178. package/dist/src/validators/redteam.d.ts +8 -0
  179. package/dist/src/validators/redteam.d.ts.map +1 -1
  180. package/dist/tsconfig.tsbuildinfo +1 -1
  181. package/package.json +39 -33
  182. package/dist/src/app/assets/index-CHWpe2P4.js +0 -976
package/dist/src/validators/redteam.d.ts CHANGED
@@ -249,6 +249,7 @@ export declare const RedteamConfigSchema: z.ZodEffects<z.ZodObject<{
249
249
  AWS_SAGEMAKER_TEMPERATURE: z.ZodOptional<z.ZodString>;
250
250
  AWS_SAGEMAKER_TOP_P: z.ZodOptional<z.ZodString>;
251
251
  AWS_SAGEMAKER_MAX_RETRIES: z.ZodOptional<z.ZodString>;
252
+ PROMPTFOO_EVAL_TIMEOUT_MS: z.ZodOptional<z.ZodString>;
252
253
  }, "strip", z.ZodTypeAny, {
253
254
  AI21_API_BASE_URL?: string | undefined;
254
255
  AI21_API_KEY?: string | undefined;
@@ -333,6 +334,7 @@ export declare const RedteamConfigSchema: z.ZodEffects<z.ZodObject<{
333
334
  AWS_SAGEMAKER_TEMPERATURE?: string | undefined;
334
335
  AWS_SAGEMAKER_TOP_P?: string | undefined;
335
336
  AWS_SAGEMAKER_MAX_RETRIES?: string | undefined;
337
+ PROMPTFOO_EVAL_TIMEOUT_MS?: string | undefined;
336
338
  }, {
337
339
  AI21_API_BASE_URL?: string | undefined;
338
340
  AI21_API_KEY?: string | undefined;
@@ -417,6 +419,7 @@ export declare const RedteamConfigSchema: z.ZodEffects<z.ZodObject<{
417
419
  AWS_SAGEMAKER_TEMPERATURE?: string | undefined;
418
420
  AWS_SAGEMAKER_TOP_P?: string | undefined;
419
421
  AWS_SAGEMAKER_MAX_RETRIES?: string | undefined;
422
+ PROMPTFOO_EVAL_TIMEOUT_MS?: string | undefined;
420
423
  }>>;
421
424
  }, "strict", z.ZodTypeAny, {
422
425
  id?: string | undefined;
@@ -509,6 +512,7 @@ export declare const RedteamConfigSchema: z.ZodEffects<z.ZodObject<{
509
512
  AWS_SAGEMAKER_TEMPERATURE?: string | undefined;
510
513
  AWS_SAGEMAKER_TOP_P?: string | undefined;
511
514
  AWS_SAGEMAKER_MAX_RETRIES?: string | undefined;
515
+ PROMPTFOO_EVAL_TIMEOUT_MS?: string | undefined;
512
516
  } | undefined;
513
517
  }, {
514
518
  id?: string | undefined;
@@ -601,6 +605,7 @@ export declare const RedteamConfigSchema: z.ZodEffects<z.ZodObject<{
601
605
  AWS_SAGEMAKER_TEMPERATURE?: string | undefined;
602
606
  AWS_SAGEMAKER_TOP_P?: string | undefined;
603
607
  AWS_SAGEMAKER_MAX_RETRIES?: string | undefined;
608
+ PROMPTFOO_EVAL_TIMEOUT_MS?: string | undefined;
604
609
  } | undefined;
605
610
  }>, z.ZodObject<{
606
611
  id: z.ZodFunction<z.ZodTuple<[], z.ZodUnknown>, z.ZodString>;
@@ -763,6 +768,7 @@ export declare const RedteamConfigSchema: z.ZodEffects<z.ZodObject<{
763
768
  AWS_SAGEMAKER_TEMPERATURE?: string | undefined;
764
769
  AWS_SAGEMAKER_TOP_P?: string | undefined;
765
770
  AWS_SAGEMAKER_MAX_RETRIES?: string | undefined;
771
+ PROMPTFOO_EVAL_TIMEOUT_MS?: string | undefined;
766
772
  } | undefined;
767
773
  } | {
768
774
  callApi: import("..").CallApiFunction;
@@ -879,6 +885,7 @@ export declare const RedteamConfigSchema: z.ZodEffects<z.ZodObject<{
879
885
  AWS_SAGEMAKER_TEMPERATURE?: string | undefined;
880
886
  AWS_SAGEMAKER_TOP_P?: string | undefined;
881
887
  AWS_SAGEMAKER_MAX_RETRIES?: string | undefined;
888
+ PROMPTFOO_EVAL_TIMEOUT_MS?: string | undefined;
882
889
  } | undefined;
883
890
  } | {
884
891
  callApi: import("..").CallApiFunction;
@@ -999,6 +1006,7 @@ export declare const RedteamConfigSchema: z.ZodEffects<z.ZodObject<{
999
1006
  AWS_SAGEMAKER_TEMPERATURE?: string | undefined;
1000
1007
  AWS_SAGEMAKER_TOP_P?: string | undefined;
1001
1008
  AWS_SAGEMAKER_MAX_RETRIES?: string | undefined;
1009
+ PROMPTFOO_EVAL_TIMEOUT_MS?: string | undefined;
1002
1010
  } | undefined;
1003
1011
  } | {
1004
1012
  callApi: import("..").CallApiFunction;
package/dist/src/validators/redteam.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"redteam.d.ts","sourceRoot":"","sources":["../../../src/validators/redteam.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAkBxB,OAAO,KAAK,EAAE,iBAAiB,EAAwC,MAAM,kBAAkB,CAAC;AAIhG,eAAO,MAAM,aAAa,EAAE,MAAM,EAI1B,CAAC;AACT;;GAEG;AACH,eAAO,MAAM,yBAAyB;;;;;;;;;;;;EAyBpC,CAAC;AAEH;;GAEG;AACH,eAAO,MAAM,mBAAmB;;;;;;;;;;;;IAmB9B,CAAC;AAEH,eAAO,MAAM,gBAAgB,yHAqBM,CAAC;AACpC;;GAEG;AACH,eAAO,MAAM,qBAAqB;;;;;;;;;IAMhC,CAAC;AAEH;;GAEG;AAEH,eAAO,MAAM,4BAA4B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAsCvC,CAAC;AAEH;;GAEG;AACH,eAAO,MAAM,mBAAmB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAyL5B,CAAC"}
1
+ {"version":3,"file":"redteam.d.ts","sourceRoot":"","sources":["../../../src/validators/redteam.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAkBxB,OAAO,KAAK,EAAE,iBAAiB,EAAwC,MAAM,kBAAkB,CAAC;AAIhG,eAAO,MAAM,aAAa,EAAE,MAAM,EAI1B,CAAC;AACT;;GAEG;AACH,eAAO,MAAM,yBAAyB;;;;;;;;;;;;EAyBpC,CAAC;AAEH;;GAEG;AACH,eAAO,MAAM,mBAAmB;;;;;;;;;;;;IAmB9B,CAAC;AAEH,eAAO,MAAM,gBAAgB,yHAqBM,CAAC;AACpC;;GAEG;AACH,eAAO,MAAM,qBAAqB;;;;;;;;;IAMhC,CAAC;AAEH;;GAEG;AAEH,eAAO,MAAM,4BAA4B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAsCvC,CAAC;AAEH;;GAEG;AACH,eAAO,MAAM,mBAAmB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAyL5B,CAAC"}