promptfoo 0.83.1 → 0.84.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (115) hide show
  1. package/dist/package.json +1 -1
  2. package/dist/src/cliState.d.ts +1 -0
  3. package/dist/src/cliState.d.ts.map +1 -1
  4. package/dist/src/cliState.js.map +1 -1
  5. package/dist/src/commands/eval.d.ts.map +1 -1
  6. package/dist/src/commands/eval.js +0 -2
  7. package/dist/src/commands/eval.js.map +1 -1
  8. package/dist/src/commands/redteam/generate.d.ts.map +1 -1
  9. package/dist/src/commands/redteam/generate.js +17 -5
  10. package/dist/src/commands/redteam/generate.js.map +1 -1
  11. package/dist/src/config.d.ts.map +1 -1
  12. package/dist/src/config.js +1 -0
  13. package/dist/src/config.js.map +1 -1
  14. package/dist/src/database/tables.d.ts +168 -144
  15. package/dist/src/database/tables.d.ts.map +1 -1
  16. package/dist/src/index.d.ts +22 -0
  17. package/dist/src/index.d.ts.map +1 -1
  18. package/dist/src/index.js +12 -0
  19. package/dist/src/index.js.map +1 -1
  20. package/dist/src/matchers.d.ts.map +1 -1
  21. package/dist/src/matchers.js +70 -0
  22. package/dist/src/matchers.js.map +1 -1
  23. package/dist/src/providers/ollama.d.ts.map +1 -1
  24. package/dist/src/providers/ollama.js +1 -0
  25. package/dist/src/providers/ollama.js.map +1 -1
  26. package/dist/src/providers/openai.js +2 -2
  27. package/dist/src/providers/openai.js.map +1 -1
  28. package/dist/src/providers/vertex.d.ts.map +1 -1
  29. package/dist/src/providers/vertex.js +19 -1
  30. package/dist/src/providers/vertex.js.map +1 -1
  31. package/dist/src/providers/vertexUtil.d.ts +12 -1
  32. package/dist/src/providers/vertexUtil.d.ts.map +1 -1
  33. package/dist/src/providers/vertexUtil.js.map +1 -1
  34. package/dist/src/redteam/plugins/base.d.ts.map +1 -1
  35. package/dist/src/redteam/plugins/base.js +6 -3
  36. package/dist/src/redteam/plugins/base.js.map +1 -1
  37. package/dist/src/redteam/plugins/index.d.ts.map +1 -1
  38. package/dist/src/redteam/plugins/index.js +16 -10
  39. package/dist/src/redteam/plugins/index.js.map +1 -1
  40. package/dist/src/redteam/plugins/pii.d.ts.map +1 -1
  41. package/dist/src/redteam/plugins/pii.js +5 -2
  42. package/dist/src/redteam/plugins/pii.js.map +1 -1
  43. package/dist/src/redteam/providers/crescendo/index.d.ts.map +1 -1
  44. package/dist/src/redteam/providers/crescendo/index.js +5 -4
  45. package/dist/src/redteam/providers/crescendo/index.js.map +1 -1
  46. package/dist/src/redteam/providers/iterative.d.ts.map +1 -1
  47. package/dist/src/redteam/providers/iterative.js +5 -3
  48. package/dist/src/redteam/providers/iterative.js.map +1 -1
  49. package/dist/src/redteam/providers/iterativeImage.d.ts.map +1 -1
  50. package/dist/src/redteam/providers/iterativeImage.js +4 -3
  51. package/dist/src/redteam/providers/iterativeImage.js.map +1 -1
  52. package/dist/src/redteam/providers/iterativeTree.d.ts.map +1 -1
  53. package/dist/src/redteam/providers/iterativeTree.js +4 -3
  54. package/dist/src/redteam/providers/iterativeTree.js.map +1 -1
  55. package/dist/src/redteam/strategies/index.d.ts +1 -2
  56. package/dist/src/redteam/strategies/index.d.ts.map +1 -1
  57. package/dist/src/redteam/strategies/multilingual.d.ts.map +1 -1
  58. package/dist/src/redteam/strategies/multilingual.js +2 -8
  59. package/dist/src/redteam/strategies/multilingual.js.map +1 -1
  60. package/dist/src/redteam/util.d.ts.map +1 -1
  61. package/dist/src/redteam/util.js +7 -1
  62. package/dist/src/redteam/util.js.map +1 -1
  63. package/dist/src/types/index.d.ts +4481 -3848
  64. package/dist/src/types/index.d.ts.map +1 -1
  65. package/dist/src/types/providers.d.ts +19 -19
  66. package/dist/src/types/providers.d.ts.map +1 -1
  67. package/dist/src/types/providers.js +4 -1
  68. package/dist/src/types/providers.js.map +1 -1
  69. package/dist/src/types/redteam.d.ts +1 -0
  70. package/dist/src/types/redteam.d.ts.map +1 -1
  71. package/dist/src/util/index.d.ts +47 -36
  72. package/dist/src/util/index.d.ts.map +1 -1
  73. package/dist/src/util/index.js +11 -1
  74. package/dist/src/util/index.js.map +1 -1
  75. package/dist/src/util/json.d.ts +1 -0
  76. package/dist/src/util/json.d.ts.map +1 -1
  77. package/dist/src/util/json.js +10 -0
  78. package/dist/src/util/json.js.map +1 -1
  79. package/dist/src/validators/providers.d.ts +484 -415
  80. package/dist/src/validators/providers.d.ts.map +1 -1
  81. package/dist/src/validators/providers.js +21 -18
  82. package/dist/src/validators/providers.js.map +1 -1
  83. package/dist/src/validators/redteam.d.ts +168 -144
  84. package/dist/src/validators/redteam.d.ts.map +1 -1
  85. package/dist/src/web/nextui/404/index.html +1 -1
  86. package/dist/src/web/nextui/404.html +1 -1
  87. package/dist/src/web/nextui/_next/static/chunks/app/{layout-840b64134bca448d.js → layout-cd2ab2a02e6ca1f4.js} +1 -1
  88. package/dist/src/web/nextui/auth/login/index.html +1 -1
  89. package/dist/src/web/nextui/auth/login/index.txt +2 -2
  90. package/dist/src/web/nextui/auth/signup/index.html +1 -1
  91. package/dist/src/web/nextui/auth/signup/index.txt +2 -2
  92. package/dist/src/web/nextui/datasets/index.html +1 -1
  93. package/dist/src/web/nextui/datasets/index.txt +2 -2
  94. package/dist/src/web/nextui/eval/index.html +1 -1
  95. package/dist/src/web/nextui/eval/index.txt +3 -3
  96. package/dist/src/web/nextui/index.html +1 -1
  97. package/dist/src/web/nextui/index.txt +2 -2
  98. package/dist/src/web/nextui/progress/index.html +1 -1
  99. package/dist/src/web/nextui/progress/index.txt +2 -2
  100. package/dist/src/web/nextui/prompts/index.html +1 -1
  101. package/dist/src/web/nextui/prompts/index.txt +2 -2
  102. package/dist/src/web/nextui/report/index.html +1 -1
  103. package/dist/src/web/nextui/report/index.txt +2 -2
  104. package/dist/src/web/nextui/setup/index.html +1 -1
  105. package/dist/src/web/nextui/setup/index.txt +2 -2
  106. package/dist/test/index.test.d.ts +2 -0
  107. package/dist/test/index.test.d.ts.map +1 -0
  108. package/dist/test/index.test.js +98 -0
  109. package/dist/test/index.test.js.map +1 -0
  110. package/dist/test/redteam/plugins/base.test.js +0 -5
  111. package/dist/test/redteam/plugins/base.test.js.map +1 -1
  112. package/package.json +1 -1
  113. /package/dist/src/web/nextui/_next/static/{48DSwiWE8aGl1g_5NqL3u → CvuPvd-3y8Il_b6bK8bfx}/_buildManifest.js +0 -0
  114. /package/dist/src/web/nextui/_next/static/{48DSwiWE8aGl1g_5NqL3u → CvuPvd-3y8Il_b6bK8bfx}/_ssgManifest.js +0 -0
  115. /package/dist/src/web/nextui/_next/static/chunks/{258-bb444f6e7eb2432f.js → 258-5aac745397a7d5f4.js} +0 -0
package/dist/src/util/json.js CHANGED
@@ -1,9 +1,14 @@
1
1
  "use strict";
2
+ var __importDefault = (this && this.__importDefault) || function (mod) {
3
+ return (mod && mod.__esModule) ? mod : { "default": mod };
4
+ };
2
5
  Object.defineProperty(exports, "__esModule", { value: true });
3
6
  exports.isValidJson = isValidJson;
4
7
  exports.safeJsonStringify = safeJsonStringify;
5
8
  exports.extractJsonObjects = extractJsonObjects;
9
+ exports.extractFirstJsonObject = extractFirstJsonObject;
6
10
  exports.orderKeys = orderKeys;
11
+ const tiny_invariant_1 = __importDefault(require("tiny-invariant"));
7
12
  function isValidJson(str) {
8
13
  try {
9
14
  JSON.parse(str);
@@ -56,6 +61,11 @@ function extractJsonObjects(str) {
56
61
  }
57
62
  return jsonObjects;
58
63
  }
64
+ function extractFirstJsonObject(str) {
65
+ const jsonObjects = extractJsonObjects(str);
66
+ (0, tiny_invariant_1.default)(jsonObjects.length >= 1, `Expected a JSON object, but got ${str}`);
67
+ return jsonObjects[0];
68
+ }
59
69
  /**
60
70
  * Reorders the keys of an object based on a specified order, preserving any unspecified keys.
61
71
  * Symbol keys are preserved and added at the end.
package/dist/src/util/json.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"json.js","sourceRoot":"","sources":["../../../src/util/json.ts"],"names":[],"mappings":";;AAAA,kCAOC;AAED,8CAiBC;AAED,gDA4BC;AAeD,8BA0BC;AAjGD,SAAgB,WAAW,CAAC,GAAW;IACrC,IAAI,CAAC;QACH,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAChB,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED,SAAgB,iBAAiB,CAAC,KAAU,EAAE,cAAuB,KAAK;IACxE,8BAA8B;IAC9B,MAAM,KAAK,GAAG,IAAI,GAAG,EAAE,CAAC;IACxB,MAAM,KAAK,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAC1C,OAAO,IAAI,CAAC,SAAS,CACnB,KAAK,EACL,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE;QACX,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,GAAG,KAAK,IAAI,EAAE,CAAC;YAC5C,IAAI,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;gBACnB,OAAO;YACT,CAAC;YACD,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACjB,CAAC;QACD,OAAO,GAAG,CAAC;IACb,CAAC,EACD,KAAK,CACN,CAAC;AACJ,CAAC;AAED,SAAgB,kBAAkB,CAAC,GAAW;IAC5C,mDAAmD;IAEnD,MAAM,WAAW,GAAG,EAAE,CAAC;IACvB,IAAI,WAAW,GAAG,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IACnC,IAAI,YAAY,GAAG,GAAG,CAAC,OAAO,CAAC,GAAG,EAAE,WAAW,CAAC,CAAC;IACjD,kEAAkE;IAClE,+DAA+D;IAC/D,OAAO,WAAW,KAAK,CAAC,CAAC,EAAE,CAAC;QAC1B,MAAM,OAAO,GAAG,GAAG,CAAC,KAAK,CAAC,WAAW,EAAE,YAAY,GAAG,CAAC,CAAC,CAAC;QACzD,IAAI,CAAC;YACH,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC;YACtC,oDAAoD;YACpD,oDAAoD;YACpD,WAAW,GAAG,GAAG,CAAC,OAAO,CAAC,GAAG,EAAE,YAAY,GAAG,CAAC,CAAC,CAAC;YACjD,YAAY,GAAG,GAAG,CAAC,OAAO,CAAC,GAAG,EAAE,WAAW,CAAC,CAAC;QAC/C,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,0DAA0D;YAC1D,YAAY,GAAG,GAAG,CAAC,OAAO,CAAC,GAAG,EAAE,YAAY,GAAG,CAAC,CAAC,CAAC;YAClD,OAAO,YAAY,KAAK,CAAC,CAAC,EAAE,CAAC;gBAC3B,mDAAmD;gBACnD,8CAA8C;gBAC9C,WAAW,GAAG,GAAG,CAAC,OAAO,CAAC,GAAG,EAAE,WAAW,GAAG,CAAC,CAAC,CAAC;gBAChD,YAAY,GAAG,GAAG,CAAC,OAAO,CAAC,GAAG,EAAE,WAAW,CAAC,CAAC;YAC/C,CAAC;QACH,CAAC;IACH,CAAC;IACD,OAAO,WAAW,CAAC;AACrB,CAAC;AAED;;;;;;;;;;;;GAYG;AACH,SAAgB,SAAS,CAAmB,GAAM,EAAE,KAAkB;IACpE,MAAM,MAAM,GAAM,EAAO,CAAC;IAE1B,gDAAgD;IAChD,KAAK,MAAM,GAAG,IAAI,KAAK,EAAE,CAAC;QACxB,IAAI,GAAG,IAAI,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,KAAK,SAAS,EAAE,CAAC;YACzC,MAAM,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC;QACzB,CAAC;IACH,CAAC;IAED,kDAAkD;IAClD,KAAK,MAAM,GAAG,IAAI,GAAG,EAAE,CAAC;QACtB,IAAI,CAAC,CAAC,GAAG,IAAI,MAAM,CAAC,IAAI,GAAG,CAAC,GAAG,CAAC,KAAK,SAAS,EAAE,CAAC;YAC/C,MAAM,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC;QACzB,CAAC;IACH,CAAC;IAED,+CAA+C;IAC/C,MAAM,UAAU,GAAG,MAAM,CAAC,qBAAqB,CAAC,GAAG,CAAC,CAAC;IACrD,KAAK,MAAM,GAAG,IAAI,UAAU,EAAE,CAAC;QAC7B,IAAI,GAAG,CAAC,GAAc,CAAC,KAAK,SAAS,EAAE,CAAC;YACtC,MAAM,CAAC,GAAc,CAAC,GAAG,GAAG,CAAC,GAAc,CAAC,CAAC;QAC/C,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC"}
1
+ {"version":3,"file":"json.js","sourceRoot":"","sources":["../../../src/util/json.ts"],"names":[],"mappings":";;;;;AAEA,kCAOC;AAED,8CAiBC;AAED,gDA4BC;AAED,wDAIC;AAeD,8BA0BC;AAzGD,oEAAuC;AAEvC,SAAgB,WAAW,CAAC,GAAW;IACrC,IAAI,CAAC;QACH,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAChB,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED,SAAgB,iBAAiB,CAAC,KAAU,EAAE,cAAuB,KAAK;IACxE,8BAA8B;IAC9B,MAAM,KAAK,GAAG,IAAI,GAAG,EAAE,CAAC;IACxB,MAAM,KAAK,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAC1C,OAAO,IAAI,CAAC,SAAS,CACnB,KAAK,EACL,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE;QACX,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,GAAG,KAAK,IAAI,EAAE,CAAC;YAC5C,IAAI,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;gBACnB,OAAO;YACT,CAAC;YACD,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACjB,CAAC;QACD,OAAO,GAAG,CAAC;IACb,CAAC,EACD,KAAK,CACN,CAAC;AACJ,CAAC;AAED,SAAgB,kBAAkB,CAAC,GAAW;IAC5C,mDAAmD;IAEnD,MAAM,WAAW,GAAG,EAAE,CAAC;IACvB,IAAI,WAAW,GAAG,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IACnC,IAAI,YAAY,GAAG,GAAG,CAAC,OAAO,CAAC,GAAG,EAAE,WAAW,CAAC,CAAC;IACjD,kEAAkE;IAClE,+DAA+D;IAC/D,OAAO,WAAW,KAAK,CAAC,CAAC,EAAE,CAAC;QAC1B,MAAM,OAAO,GAAG,GAAG,CAAC,KAAK,CAAC,WAAW,EAAE,YAAY,GAAG,CAAC,CAAC,CAAC;QACzD,IAAI,CAAC;YACH,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC;YACtC,oDAAoD;YACpD,oDAAoD;YACpD,WAAW,GAAG,GAAG,CAAC,OAAO,CAAC,GAAG,EAAE,YAAY,GAAG,CAAC,CAAC,CAAC;YACjD,YAAY,GAAG,GAAG,CAAC,OAAO,CAAC,GAAG,EAAE,WAAW,CAAC,CAAC;QAC/C,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,0DAA0D;YAC1D,YAAY,GAAG,GAAG,CAAC,OAAO,CAAC,GAAG,EAAE,YAAY,GAAG,CAAC,CAAC,CAAC;YAClD,OAAO,YAAY,KAAK,CAAC,CAAC,EAAE,CAAC;gBAC3B,mDAAmD;gBACnD,8CAA8C;gBAC9C,WAAW,GAAG,GAAG,CAAC,OAAO,CAAC,GAAG,EAAE,WAAW,GAAG,CAAC,CAAC,CAAC;gBAChD,YAAY,GAAG,GAAG,CAAC,OAAO,CAAC,GAAG,EAAE,WAAW,CAAC,CAAC;YAC/C,CAAC;QACH,CAAC;IACH,CAAC;IACD,OAAO,WAAW,CAAC;AACrB,CAAC;AAED,SAAgB,sBAAsB,CAAI,GAAW;IACnD,MAAM,WAAW,GAAG,kBAAkB,CAAC,GAAG,CAAC,CAAC;IAC5C,IAAA,wBAAS,EAAC,WAAW,CAAC,MAAM,IAAI,CAAC,EAAE,mCAAmC,GAAG,EAAE,CAAC,CAAC;IAC7E,OAAO,WAAW,CAAC,CAAC,CAAM,CAAC;AAC7B,CAAC;AAED;;;;;;;;;;;;GAYG;AACH,SAAgB,SAAS,CAAmB,GAAM,EAAE,KAAkB;IACpE,MAAM,MAAM,GAAM,EAAO,CAAC;IAE1B,gDAAgD;IAChD,KAAK,MAAM,GAAG,IAAI,KAAK,EAAE,CAAC;QACxB,IAAI,GAAG,IAAI,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,KAAK,SAAS,EAAE,CAAC;YACzC,MAAM,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC;QACzB,CAAC;IACH,CAAC;IAED,kDAAkD;IAClD,KAAK,MAAM,GAAG,IAAI,GAAG,EAAE,CAAC;QACtB,IAAI,CAAC,CAAC,GAAG,IAAI,MAAM,CAAC,IAAI,GAAG,CAAC,GAAG,CAAC,KAAK,SAAS,EAAE,CAAC;YAC/C,MAAM,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC;QACzB,CAAC;IACH,CAAC;IAED,+CAA+C;IAC/C,MAAM,UAAU,GAAG,MAAM,CAAC,qBAAqB,CAAC,GAAG,CAAC,CAAC;IACrD,KAAK,MAAM,GAAG,IAAI,UAAU,EAAE,CAAC;QAC7B,IAAI,GAAG,CAAC,GAAc,CAAC,KAAK,SAAS,EAAE,CAAC;YACtC,MAAM,CAAC,GAAc,CAAC,GAAG,GAAG,CAAC,GAAc,CAAC,CAAC;QAC/C,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC"}