promptfoo 0.92.1 → 0.92.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (78) hide show
  1. package/dist/package.json +12 -11
  2. package/dist/src/app/assets/{index-DdvQdOby.css → index-D9XEsKus.css} +1 -1
  3. package/dist/src/app/assets/{index-BpjzEMiv.js → index-qAwIwe8f.js} +264 -265
  4. package/dist/src/app/assets/{index.es-ihzvEu35.js → index.es-BAabtj1c.js} +1 -1
  5. package/dist/src/app/assets/{sync-BosjlpGJ.js → sync-uXDZgY7U.js} +1 -1
  6. package/dist/src/app/index.html +2 -2
  7. package/dist/src/assertions.js +1 -1
  8. package/dist/src/assertions.js.map +1 -1
  9. package/dist/src/commands/eval.d.ts.map +1 -1
  10. package/dist/src/commands/eval.js +5 -3
  11. package/dist/src/commands/eval.js.map +1 -1
  12. package/dist/src/database/index.d.ts.map +1 -1
  13. package/dist/src/database/index.js +9 -1
  14. package/dist/src/database/index.js.map +1 -1
  15. package/dist/src/database/tables.d.ts +1 -0
  16. package/dist/src/database/tables.d.ts.map +1 -1
  17. package/dist/src/evaluator.d.ts.map +1 -1
  18. package/dist/src/evaluator.js +3 -1
  19. package/dist/src/evaluator.js.map +1 -1
  20. package/dist/src/evaluatorHelpers.d.ts.map +1 -1
  21. package/dist/src/evaluatorHelpers.js +1 -0
  22. package/dist/src/evaluatorHelpers.js.map +1 -1
  23. package/dist/src/index.d.ts.map +1 -1
  24. package/dist/src/index.js +2 -1
  25. package/dist/src/index.js.map +1 -1
  26. package/dist/src/models/provider.d.ts +3 -1
  27. package/dist/src/models/provider.d.ts.map +1 -1
  28. package/dist/src/models/provider.js +22 -17
  29. package/dist/src/models/provider.js.map +1 -1
  30. package/dist/src/prompts/grading.js +2 -2
  31. package/dist/src/providers/fal.d.ts +1 -0
  32. package/dist/src/providers/fal.d.ts.map +1 -1
  33. package/dist/src/providers/fal.js +10 -3
  34. package/dist/src/providers/fal.js.map +1 -1
  35. package/dist/src/redteam/commands/generate.d.ts.map +1 -1
  36. package/dist/src/redteam/commands/generate.js +39 -8
  37. package/dist/src/redteam/commands/generate.js.map +1 -1
  38. package/dist/src/redteam/commands/run.d.ts.map +1 -1
  39. package/dist/src/redteam/commands/run.js +13 -35
  40. package/dist/src/redteam/commands/run.js.map +1 -1
  41. package/dist/src/redteam/index.d.ts.map +1 -1
  42. package/dist/src/redteam/index.js +17 -16
  43. package/dist/src/redteam/index.js.map +1 -1
  44. package/dist/src/redteam/plugins/custom.d.ts +15 -4
  45. package/dist/src/redteam/plugins/custom.d.ts.map +1 -1
  46. package/dist/src/redteam/plugins/custom.js +37 -3
  47. package/dist/src/redteam/plugins/custom.js.map +1 -1
  48. package/dist/src/redteam/plugins/policy.d.ts.map +1 -1
  49. package/dist/src/redteam/plugins/policy.js +0 -3
  50. package/dist/src/redteam/plugins/policy.js.map +1 -1
  51. package/dist/src/redteam/types.d.ts +1 -0
  52. package/dist/src/redteam/types.d.ts.map +1 -1
  53. package/dist/src/share.d.ts.map +1 -1
  54. package/dist/src/share.js +14 -5
  55. package/dist/src/share.js.map +1 -1
  56. package/dist/src/types/index.d.ts +8 -0
  57. package/dist/src/types/index.d.ts.map +1 -1
  58. package/dist/src/types/index.js +1 -0
  59. package/dist/src/types/index.js.map +1 -1
  60. package/dist/test/assertions.test.js +14 -14
  61. package/dist/test/assertions.test.js.map +1 -1
  62. package/dist/test/factories/evalFactory.d.ts +1 -0
  63. package/dist/test/factories/evalFactory.d.ts.map +1 -1
  64. package/dist/test/index.test.js +1 -0
  65. package/dist/test/index.test.js.map +1 -1
  66. package/dist/test/python/python.integration.test.d.ts +2 -0
  67. package/dist/test/python/python.integration.test.d.ts.map +1 -0
  68. package/dist/test/python/{integration.test.js → python.integration.test.js} +1 -1
  69. package/dist/test/python/python.integration.test.js.map +1 -0
  70. package/dist/test/redteam/plugins/custom.test.d.ts +2 -0
  71. package/dist/test/redteam/plugins/custom.test.d.ts.map +1 -0
  72. package/dist/test/redteam/plugins/custom.test.js +87 -0
  73. package/dist/test/redteam/plugins/custom.test.js.map +1 -0
  74. package/dist/tsconfig.tsbuildinfo +1 -1
  75. package/package.json +12 -11
  76. package/dist/test/python/integration.test.d.ts +0 -2
  77. package/dist/test/python/integration.test.d.ts.map +0 -1
  78. package/dist/test/python/integration.test.js.map +0 -1
package/package.json CHANGED
@@ -2,7 +2,7 @@
2
2
  "name": "promptfoo",
3
3
  "description": "LLM eval & testing toolkit",
4
4
  "author": "Ian Webster",
5
- "version": "0.92.1",
5
+ "version": "0.92.3",
6
6
  "license": "MIT",
7
7
  "type": "commonjs",
8
8
  "repository": {
@@ -45,6 +45,7 @@
45
45
  "build:watch": "tsc --watch",
46
46
  "build": "tsc && shx cp src/*.html dist/src && shx cp src/python/wrapper.py dist/src/python && shx mkdir -p dist/src/golang && shx cp src/golang/wrapper.go dist/src/golang && shx rm -rf dist/drizzle && shx cp -r drizzle dist/drizzle && npm run build:app && shx chmod +x dist/src/main.js",
47
47
  "test": "jest",
48
+ "test:integration": "jest --config jest.integration.config.ts integration",
48
49
  "test:watch": "jest --watch",
49
50
  "test:app": "npm run test --prefix src/app",
50
51
  "format": "prettier -w .",
@@ -80,12 +81,12 @@
80
81
  "puppeteer-extra-plugin-stealth": "^2.11.2"
81
82
  },
82
83
  "devDependencies": {
83
- "@aws-sdk/client-bedrock-runtime": "^3.665.0",
84
+ "@aws-sdk/client-bedrock-runtime": "^3.670.0",
84
85
  "@azure/identity": "^4.4.1",
85
86
  "@eslint/js": "^9.12.0",
86
87
  "@libsql/client": "^0.14.0",
87
88
  "@swc/cli": "^0.4.0",
88
- "@swc/core": "^1.7.28",
89
+ "@swc/core": "^1.7.35",
89
90
  "@swc/jest": "^0.2.36",
90
91
  "@trivago/prettier-plugin-sort-imports": "^4.3.0",
91
92
  "@types/async": "^3.2.24",
@@ -128,17 +129,17 @@
128
129
  "supertest": "^7.0.0",
129
130
  "ts-node": "^10.9.2",
130
131
  "tsconfig-paths": "^4.2.0",
131
- "typescript": "^5.6.2",
132
- "typescript-eslint": "^8.8.0",
132
+ "typescript": "^5.6.3",
133
+ "typescript-eslint": "^8.8.1",
133
134
  "zod-to-json-schema": "^3.23.3"
134
135
  },
135
136
  "dependencies": {
136
- "@anthropic-ai/sdk": "^0.28.0",
137
- "@apidevtools/json-schema-ref-parser": "^11.7.0",
137
+ "@anthropic-ai/sdk": "^0.29.0",
138
+ "@apidevtools/json-schema-ref-parser": "^11.7.2",
138
139
  "@emotion/react": "^11.13.3",
139
140
  "@emotion/styled": "^11.13.0",
140
141
  "@googleapis/sheets": "^9.3.1",
141
- "@mui/material": "^6.1.2",
142
+ "@mui/material": "^6.1.3",
142
143
  "ajv": "^8.17.1",
143
144
  "ajv-formats": "^3.0.1",
144
145
  "async": "^3.2.6",
@@ -158,7 +159,7 @@
158
159
  "dedent": "^1.5.3",
159
160
  "dotenv": "^16.4.5",
160
161
  "drizzle-orm": "^0.33.0",
161
- "express": "^4.21.0",
162
+ "express": "^4.21.1",
162
163
  "fast-deep-equal": "^3.1.3",
163
164
  "fast-xml-parser": "^4.5.0",
164
165
  "fastest-levenshtein": "^1.0.16",
@@ -172,7 +173,7 @@
172
173
  "node-cache": "^5.1.2",
173
174
  "node-fetch": "^2.6.7",
174
175
  "nunjucks": "^3.2.4",
175
- "openai": "^4.67.1",
176
+ "openai": "^4.67.3",
176
177
  "opener": "^1.5.2",
177
178
  "proxy-agent": "^6.4.0",
178
179
  "python-shell": "^5.0.0",
@@ -182,7 +183,7 @@
182
183
  "socket.io": "^4.8.0",
183
184
  "tiny-invariant": "^1.3.3",
184
185
  "uuid": "^10.0.0",
185
- "winston": "^3.14.2",
186
+ "winston": "^3.15.0",
186
187
  "ws": "^8.18.0",
187
188
  "zod": "^3.23.8",
188
189
  "zod-validation-error": "^3.4.0"
package/dist/test/python/integration.test.d.ts DELETED
@@ -1,2 +0,0 @@
1
- export {};
2
- //# sourceMappingURL=integration.test.d.ts.map
package/dist/test/python/integration.test.d.ts.map DELETED
@@ -1 +0,0 @@
1
- {"version":3,"file":"integration.test.d.ts","sourceRoot":"","sources":["../../../test/python/integration.test.ts"],"names":[],"mappings":""}
package/dist/test/python/integration.test.js.map DELETED
@@ -1 +0,0 @@
1
- {"version":3,"file":"integration.test.js","sourceRoot":"","sources":["../../../test/python/integration.test.ts"],"names":[],"mappings":";;;;;AAAA,iDAAqC;AACrC,oDAA4B;AAC5B,4CAAoB;AACpB,gDAAwB;AACxB,gDAAwB;AACxB,8DAAsC;AACtC,8DAAyD;AAEzD,MAAM,WAAW,GAAG,cAAI,CAAC,SAAS,CAAC,oBAAI,CAAC,CAAC;AAEzC,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;AAE9B,QAAQ,CAAC,+BAA+B,EAAE,GAAG,EAAE;IAC7C,MAAM,UAAU,GAAG,cAAI,CAAC,IAAI,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;IAEnD,SAAS,CAAC,GAAG,EAAE;QACb,IAAI,CAAC,YAAE,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;YAC/B,YAAE,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;QAC3B,CAAC;QAED,YAAE,CAAC,aAAa,CACd,cAAI,CAAC,IAAI,CAAC,UAAU,EAAE,WAAW,CAAC,EAClC,IAAA,gBAAM,EAAA;;;;;;;;;;;;;;;;;;;;;;;SAuBH,CACJ,CAAC;QAEF,YAAE,CAAC,aAAa,CACd,cAAI,CAAC,IAAI,CAAC,UAAU,EAAE,iBAAiB,CAAC,EACxC,IAAA,gBAAM,EAAA;;;;;;;;;SASH,CACJ,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,GAAG,EAAE;QACZ,YAAE,CAAC,MAAM,CAAC,cAAI,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IACrE,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,wCAAwC,EAAE,KAAK,IAAI,EAAE;QACtD,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,WAAW,CAAC,kBAAkB,CAAC,CAAC;QACzD,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;IACrC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,gDAAgD,EAAE,KAAK,IAAI,EAAE;QAC9D,MAAM,MAAM,GAAG,MAAM,IAAA,uBAAS,EAAC,cAAI,CAAC,IAAI,CAAC,UAAU,EAAE,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC,eAAe,CAAC,CAAC,CAAC;QAC9F,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;YACrB,OAAO,EAAE,eAAe;YACxB,OAAO,EAAE,IAAI;SACd,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,kCAAkC,EAAE,KAAK,IAAI,EAAE;QAChD,MAAM,MAAM,GAAG,MAAM,IAAA,uBAAS,EAAC,cAAI,CAAC,IAAI,CAAC,UAAU,EAAE,WAAW,CAAC,EAAE,MAAM,EAAE;YACzE,UAAU;YACV,WAAW;SACZ,CAAC,CAAC;QACH,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;YACrB,OAAO,EAAE,oBAAoB;YAC7B,OAAO,EAAE,IAAI;SACd,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,qCAAqC,EAAE,KAAK,IAAI,EAAE;QACnD,MAAM,MAAM,GAAG,MAAM,IAAA,uBAAS,EAAC,cAAI,CAAC,IAAI,CAAC,UAAU,EAAE,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;QACjF,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;YACrB,OAAO,EAAE,EAAE;YACX,OAAO,EAAE,IAAI;SACd,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,mCAAmC,EAAE,KAAK,IAAI,EAAE;QACjD,MAAM,MAAM,GAAG,MAAM,IAAA,uBAAS,EAAC,cAAI,CAAC,IAAI,CAAC,UAAU,EAAE,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;QAClF,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;YACrB,OAAO,EAAE,KAAK;YACd,OAAO,EAAE,IAAI;SACd,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,+CAA+C,EAAE,KAAK,IAAI,EAAE;QAC7D,MAAM,eAAe,GAAG,cAAI,CAAC,IAAI,CAAC,UAAU,EAAE,iBAAiB,CAAC,CAAC;QACjE,MAAM,MAAM,CAAC,IAAA,uBAAS,EAAC,eAAe,EAAE,MAAM,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC;IAChG,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,mDAAmD,EAAE,KAAK,IAAI,EAAE;QACjE,MAAM,MAAM,GAAG,MAAM,IAAA,uBAAS,EAAC,cAAI,CAAC,IAAI,CAAC,UAAU,EAAE,WAAW,CAAC,EAAE,iBAAiB,EAAE;YACpF,iBAAiB;SAClB,CAAC,CAAC;QACH,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;YACrB,OAAO,EAAE,iBAAiB;YAC1B,OAAO,EAAE,IAAI;SACd,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,6BAA6B,EAAE,KAAK,IAAI,EAAE;QAC3C,MAAM,MAAM,GAAG,MAAM,IAAA,uBAAS,EAAC,cAAI,CAAC,IAAI,CAAC,UAAU,EAAE,WAAW,CAAC,EAAE,wBAAwB,EAAE;YAC3F,cAAc;SACf,CAAC,CAAC;QACH,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;YACrB,OAAO,EAAE,cAAc;YACvB,OAAO,EAAE,IAAI;SACd,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,+BAA+B,EAAE,KAAK,IAAI,EAAE;QAC7C,MAAM,MAAM,GAAG,MAAM,IAAA,uBAAS,EAAC,cAAI,CAAC,IAAI,CAAC,UAAU,EAAE,WAAW,CAAC,EAAE,gBAAgB,EAAE;YACnF,gBAAgB;SACjB,CAAC,CAAC;QACH,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;YACrB,OAAO,EAAE,gBAAgB;YACzB,OAAO,EAAE,IAAI;SACd,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,oCAAoC,EAAE,KAAK,IAAI,EAAE;QAClD,MAAM,MAAM,GAAG,MAAM,IAAA,uBAAS,EAC5B,cAAI,CAAC,IAAI,CAAC,UAAU,EAAE,iBAAiB,CAAC,EACxC,kBAAkB,EAClB,EAAE,CACH,CAAC;QACF,MAAM,CAAC,MAAM,CAAC,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC;QACrC,MAAM,CAAC,MAAM,CAAC,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC;QACtC,MAAM,CAAE,MAAc,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAC5C,MAAM,CAAC,IAAI,IAAI,CAAE,MAAc,CAAC,IAAI,CAAC,CAAC,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;IAC9D,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,kDAAkD,EAAE,KAAK,IAAI,EAAE;QAChE,OAAO,CAAC,GAAG,CAAC,QAAQ,GAAG,YAAY,CAAC;QACpC,MAAM,MAAM,GAAG,MAAM,IAAA,uBAAS,EAC5B,cAAI,CAAC,IAAI,CAAC,UAAU,EAAE,iBAAiB,CAAC,EACxC,kBAAkB,EAClB,EAAE,CACH,CAAC;QACF,MAAM,CAAE,MAAc,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;QAC/C,OAAO,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC;IAC9B,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,2BAA2B,EAAE,KAAK,IAAI,EAAE;QACzC,IAAI,CAAC,aAAa,EAAE,CAAC;QAErB,MAAM,MAAM,GAAG,MAAM,IAAA,uBAAS,EAAC,cAAI,CAAC,IAAI,CAAC,UAAU,EAAE,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC,YAAY,CAAC,CAAC,CAAC;QAE3F,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;YACrB,OAAO,EAAE,YAAY;YACrB,OAAO,EAAE,IAAI;SACd,CAAC,CAAC;QACH,MAAM,CAAC,gBAAM,CAAC,KAAK,CAAC,CAAC,oBAAoB,CACvC,MAAM,CAAC,gBAAgB,CAAC,kCAAkC,CAAC,CAC5D,CAAC;QACF,MAAM,CAAC,gBAAM,CAAC,KAAK,CAAC,CAAC,oBAAoB,CAAC,MAAM,CAAC,gBAAgB,CAAC,eAAe,CAAC,CAAC,CAAC;IACtF,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}