promptfoo 0.119.13 → 0.119.14

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (131) hide show
  1. package/dist/package.json +28 -26
  2. package/dist/src/app/assets/index-eJ2lMe94.js +51 -0
  3. package/dist/src/app/assets/{source-map-support-Bnh0UQ2S.js → source-map-support-1v4oeb7P.js} +1 -1
  4. package/dist/src/app/assets/sync-CtLQRuC1.js +1 -0
  5. package/dist/src/app/assets/{vendor-charts-T60Uk0Z3.js → vendor-charts-DnVv66VV.js} +1 -1
  6. package/dist/src/app/assets/{vendor-markdown-DLig-KJh.js → vendor-markdown-DCpQIyMA.js} +1 -1
  7. package/dist/src/app/assets/{vendor-mui-core-5BLaiG3c.js → vendor-mui-core-Boqnpf9f.js} +1 -1
  8. package/dist/src/app/assets/{vendor-mui-icons-fn39Fu2e.js → vendor-mui-icons-B8MqoVbj.js} +1 -1
  9. package/dist/src/app/assets/vendor-mui-x-CGSS6QHF.js +45 -0
  10. package/dist/src/app/assets/{vendor-utils-DYBMEuwX.js → vendor-utils-DdfHIEy8.js} +1 -1
  11. package/dist/src/app/index.html +7 -7
  12. package/dist/src/assertions/guardrails.d.ts +1 -1
  13. package/dist/src/assertions/guardrails.js +18 -9
  14. package/dist/src/assertions/index.d.ts +1 -1
  15. package/dist/src/assertions/index.js +9 -3
  16. package/dist/src/assertions/searchRubric.d.ts +3 -0
  17. package/dist/src/assertions/searchRubric.js +18 -0
  18. package/dist/src/commands/eval.js +1 -1
  19. package/dist/src/commands/modelScan.d.ts +7 -1
  20. package/dist/src/commands/modelScan.js +121 -59
  21. package/dist/src/database/index.d.ts +6 -0
  22. package/dist/src/database/index.js +11 -0
  23. package/dist/src/database/tables.d.ts +46 -24
  24. package/dist/src/envars.d.ts +17 -0
  25. package/dist/src/generated/constants.js +1 -1
  26. package/dist/src/logger.d.ts +5 -0
  27. package/dist/src/logger.js +28 -0
  28. package/dist/src/main.js +17 -6
  29. package/dist/src/matchers.d.ts +1 -0
  30. package/dist/src/matchers.js +80 -0
  31. package/dist/src/models/eval.d.ts +2 -1
  32. package/dist/src/models/eval.js +44 -2
  33. package/dist/src/prompts/grading.d.ts +1 -0
  34. package/dist/src/prompts/grading.js +26 -1
  35. package/dist/src/prompts/index.d.ts +1 -0
  36. package/dist/src/prompts/index.js +4 -1
  37. package/dist/src/providers/adaline.gateway.js +2 -2
  38. package/dist/src/providers/anthropic/defaults.d.ts +1 -1
  39. package/dist/src/providers/anthropic/defaults.js +15 -0
  40. package/dist/src/providers/azure/chat.d.ts +3 -1
  41. package/dist/src/providers/azure/chat.js +16 -3
  42. package/dist/src/providers/azure/defaults.js +660 -141
  43. package/dist/src/providers/azure/responses.d.ts +5 -0
  44. package/dist/src/providers/azure/responses.js +33 -4
  45. package/dist/src/providers/azure/types.d.ts +4 -0
  46. package/dist/src/providers/bedrock/agents.d.ts +1 -1
  47. package/dist/src/providers/bedrock/agents.js +2 -2
  48. package/dist/src/providers/bedrock/base.d.ts +40 -0
  49. package/dist/src/providers/bedrock/base.js +171 -0
  50. package/dist/src/providers/bedrock/converse.d.ts +146 -0
  51. package/dist/src/providers/bedrock/converse.js +1044 -0
  52. package/dist/src/providers/bedrock/index.d.ts +1 -34
  53. package/dist/src/providers/bedrock/index.js +4 -159
  54. package/dist/src/providers/bedrock/knowledgeBase.d.ts +1 -1
  55. package/dist/src/providers/bedrock/knowledgeBase.js +2 -2
  56. package/dist/src/providers/bedrock/nova-sonic.d.ts +2 -1
  57. package/dist/src/providers/bedrock/nova-sonic.js +2 -2
  58. package/dist/src/providers/claude-agent-sdk.d.ts +58 -1
  59. package/dist/src/providers/claude-agent-sdk.js +22 -1
  60. package/dist/src/providers/defaults.js +4 -0
  61. package/dist/src/providers/github/defaults.js +6 -6
  62. package/dist/src/providers/google/types.d.ts +25 -0
  63. package/dist/src/providers/google/util.d.ts +2 -0
  64. package/dist/src/providers/google/vertex.js +78 -22
  65. package/dist/src/providers/{groq.d.ts → groq/chat.d.ts} +26 -20
  66. package/dist/src/providers/groq/chat.js +79 -0
  67. package/dist/src/providers/groq/index.d.ts +5 -0
  68. package/dist/src/providers/groq/index.js +24 -0
  69. package/dist/src/providers/groq/responses.d.ts +106 -0
  70. package/dist/src/providers/groq/responses.js +64 -0
  71. package/dist/src/providers/groq/types.d.ts +44 -0
  72. package/dist/src/providers/groq/types.js +3 -0
  73. package/dist/src/providers/groq/util.d.ts +15 -0
  74. package/dist/src/providers/groq/util.js +28 -0
  75. package/dist/src/providers/mcp/client.d.ts +8 -0
  76. package/dist/src/providers/mcp/client.js +60 -10
  77. package/dist/src/providers/mcp/types.d.ts +21 -0
  78. package/dist/src/providers/openai/chatkit-pool.d.ts +114 -0
  79. package/dist/src/providers/openai/chatkit-pool.js +548 -0
  80. package/dist/src/providers/openai/chatkit-types.d.ts +73 -0
  81. package/dist/src/providers/openai/chatkit-types.js +3 -0
  82. package/dist/src/providers/openai/chatkit.d.ts +76 -0
  83. package/dist/src/providers/openai/chatkit.js +879 -0
  84. package/dist/src/providers/openai/codex-sdk.d.ts +109 -0
  85. package/dist/src/providers/openai/codex-sdk.js +346 -0
  86. package/dist/src/providers/openai/defaults.d.ts +2 -0
  87. package/dist/src/providers/openai/defaults.js +10 -4
  88. package/dist/src/providers/registry.js +48 -9
  89. package/dist/src/providers/responses/types.d.ts +1 -1
  90. package/dist/src/providers/sagemaker.d.ts +2 -2
  91. package/dist/src/providers/webSearchUtils.d.ts +17 -0
  92. package/dist/src/providers/webSearchUtils.js +169 -0
  93. package/dist/src/providers/xai/chat.d.ts +61 -0
  94. package/dist/src/providers/xai/chat.js +68 -3
  95. package/dist/src/providers/xai/responses.d.ts +189 -0
  96. package/dist/src/providers/xai/responses.js +268 -0
  97. package/dist/src/redteam/constants/plugins.d.ts +1 -1
  98. package/dist/src/redteam/constants/plugins.js +1 -1
  99. package/dist/src/redteam/constants/strategies.d.ts +1 -1
  100. package/dist/src/redteam/constants/strategies.js +1 -0
  101. package/dist/src/redteam/plugins/vlguard.d.ts +53 -4
  102. package/dist/src/redteam/plugins/vlguard.js +362 -46
  103. package/dist/src/redteam/providers/constants.d.ts +2 -2
  104. package/dist/src/redteam/providers/constants.js +2 -2
  105. package/dist/src/redteam/providers/crescendo/index.d.ts +1 -1
  106. package/dist/src/redteam/providers/crescendo/index.js +5 -3
  107. package/dist/src/redteam/providers/hydra/index.js +1 -1
  108. package/dist/src/server/routes/modelAudit.js +4 -4
  109. package/dist/src/share.js +4 -2
  110. package/dist/src/telemetry.js +44 -8
  111. package/dist/src/types/env.d.ts +3 -0
  112. package/dist/src/types/env.js +1 -0
  113. package/dist/src/types/index.d.ts +896 -615
  114. package/dist/src/types/index.js +1 -0
  115. package/dist/src/types/providers.d.ts +1 -0
  116. package/dist/src/types/tracing.d.ts +3 -0
  117. package/dist/src/util/database.d.ts +6 -4
  118. package/dist/src/util/file.js +6 -4
  119. package/dist/src/util/modelAuditCliParser.d.ts +4 -4
  120. package/dist/src/util/xlsx.js +52 -26
  121. package/dist/src/validators/providers.d.ts +142 -122
  122. package/dist/src/validators/providers.js +4 -6
  123. package/dist/src/validators/redteam.d.ts +36 -28
  124. package/dist/src/validators/redteam.js +9 -3
  125. package/dist/tsconfig.tsbuildinfo +1 -1
  126. package/package.json +28 -26
  127. package/dist/drizzle/CLAUDE.md +0 -65
  128. package/dist/src/app/assets/index-DifT6VGT.js +0 -51
  129. package/dist/src/app/assets/sync-Oo-W_Rbj.js +0 -1
  130. package/dist/src/app/assets/vendor-mui-x-C2xF-yiO.js +0 -45
  131. package/dist/src/providers/groq.js +0 -48
package/dist/package.json CHANGED
@@ -2,7 +2,7 @@
2
2
  "name": "promptfoo",
3
3
  "description": "LLM eval & testing toolkit",
4
4
  "author": "Ian Webster",
5
- "version": "0.119.13",
5
+ "version": "0.119.14",
6
6
  "license": "MIT",
7
7
  "type": "commonjs",
8
8
  "repository": {
@@ -42,7 +42,7 @@
42
42
  "build:app": "npm run build --prefix src/app",
43
43
  "build:clean": "shx rm -rf dist",
44
44
  "build:watch": "tsc --watch",
45
- "build": "npm run generate-constants && tsc && shx cp src/*.html dist/src && shx cp src/python/wrapper.py dist/src/python && shx cp src/python/persistent_wrapper.py dist/src/python && shx mkdir -p dist/src/golang && shx cp src/golang/wrapper.go dist/src/golang && shx mkdir -p dist/src/ruby && shx cp src/ruby/wrapper.rb dist/src/ruby && shx cp src/dynamic-import.cjs dist/src && shx rm -rf dist/drizzle && shx cp -r drizzle dist/drizzle && npm run build:app && shx chmod +x dist/src/main.js",
45
+ "build": "npm run generate-constants && tsc && shx cp src/*.html dist/src && shx cp src/python/wrapper.py dist/src/python && shx cp src/python/persistent_wrapper.py dist/src/python && shx mkdir -p dist/src/golang && shx cp src/golang/wrapper.go dist/src/golang && shx mkdir -p dist/src/ruby && shx cp src/ruby/wrapper.rb dist/src/ruby && shx cp src/dynamic-import.cjs dist/src && shx rm -rf dist/drizzle && shx cp -r drizzle dist/drizzle && shx rm -f dist/drizzle/CLAUDE.md dist/drizzle/AGENTS.md && npm run build:app && shx chmod +x dist/src/main.js",
46
46
  "citation:generate": "ts-node scripts/generateCitation.ts",
47
47
  "db:generate": "npx drizzle-kit generate",
48
48
  "db:migrate": "node --require tsx/cjs src/migrate.ts",
@@ -103,7 +103,7 @@
103
103
  }
104
104
  },
105
105
  "devDependencies": {
106
- "@adaline/anthropic": "1.8.0",
106
+ "@adaline/anthropic": "1.9.0",
107
107
  "@adaline/azure": "1.7.0",
108
108
  "@adaline/gateway": "1.10.0",
109
109
  "@adaline/google": "1.10.0",
@@ -114,19 +114,20 @@
114
114
  "@adaline/together-ai": "1.6.0",
115
115
  "@adaline/types": "1.9.1",
116
116
  "@adaline/vertex": "1.9.0",
117
- "@anthropic-ai/claude-agent-sdk": "^0.1.47",
118
- "@aws-sdk/client-bedrock-agent-runtime": "^3.936.0",
119
- "@aws-sdk/client-bedrock-runtime": "^3.936.0",
120
- "@aws-sdk/client-sagemaker-runtime": "^3.936.0",
121
- "@aws-sdk/credential-provider-sso": "^3.936.0",
117
+ "@anthropic-ai/claude-agent-sdk": "^0.1.53",
118
+ "@aws-sdk/client-bedrock-agent-runtime": "^3.940.0",
119
+ "@aws-sdk/client-bedrock-runtime": "^3.941.0",
120
+ "@aws-sdk/client-sagemaker-runtime": "^3.940.0",
121
+ "@aws-sdk/credential-provider-sso": "^3.940.0",
122
122
  "@aws-sdk/types": "^3.936.0",
123
123
  "@azure/identity": "^4.13.0",
124
124
  "@azure/openai-assistants": "^1.0.0-beta.6",
125
125
  "@biomejs/biome": "^2.3.7",
126
126
  "@fal-ai/client": "^1.7.2",
127
- "@ibm-cloud/watsonx-ai": "^1.7.3",
127
+ "@ibm-cloud/watsonx-ai": "^1.7.4",
128
128
  "@ibm-generative-ai/node-sdk": "^3.2.4",
129
129
  "@jest/globals": "^30.2.0",
130
+ "@smithy/types": "^4.9.0",
130
131
  "@swc/core": "^1.15.3",
131
132
  "@swc/jest": "^0.2.39",
132
133
  "@types/async": "^3.2.25",
@@ -155,18 +156,18 @@
155
156
  "drizzle-kit": "^0.31.7",
156
157
  "ibm-cloud-sdk-core": "^5.4.4",
157
158
  "jest": "^30.2.0",
158
- "vitest": "^4.0.0",
159
- "knip": "^5.70.1",
159
+ "knip": "^5.70.2",
160
160
  "langfuse": "^3.38.6",
161
161
  "madge": "^8.0.0",
162
162
  "nock": "^14.0.10",
163
163
  "node-sql-parser": "^5.3.13",
164
164
  "nodemon": "^3.1.11",
165
165
  "pdf-parse": "^1.1.4",
166
- "playwright": "^1.56.1",
166
+ "playwright": "^1.57.0",
167
167
  "playwright-extra": "^4.3.6",
168
168
  "prettier": "3.6.2",
169
169
  "puppeteer-extra-plugin-stealth": "^2.11.2",
170
+ "read-excel-file": "^6.0.1",
170
171
  "sharp": "^0.34.5",
171
172
  "shx": "^0.4.0",
172
173
  "source-map-support": "^0.5.21",
@@ -174,12 +175,12 @@
174
175
  "ts-dedent": "^2.2.0",
175
176
  "ts-node": "^10.9.2",
176
177
  "typescript": "^5.9.3",
178
+ "vitest": "^4.0.14",
177
179
  "winston-transport": "^4.9.0",
178
- "xlsx": "^0.18.5",
179
180
  "zod-to-json-schema": "^3.25.0"
180
181
  },
181
182
  "optionalDependencies": {
182
- "@adaline/anthropic": "1.8.0",
183
+ "@adaline/anthropic": "1.9.0",
183
184
  "@adaline/azure": "1.7.0",
184
185
  "@adaline/gateway": "1.10.0",
185
186
  "@adaline/google": "1.10.0",
@@ -190,11 +191,11 @@
190
191
  "@adaline/together-ai": "1.6.0",
191
192
  "@adaline/types": "1.9.1",
192
193
  "@adaline/vertex": "1.9.0",
193
- "@anthropic-ai/claude-agent-sdk": "^0.1.47",
194
- "@aws-sdk/client-bedrock-agent-runtime": "^3.936.0",
195
- "@aws-sdk/client-bedrock-runtime": "^3.936.0",
196
- "@aws-sdk/client-sagemaker-runtime": "^3.936.0",
197
- "@aws-sdk/credential-provider-sso": "^3.936.0",
194
+ "@anthropic-ai/claude-agent-sdk": "^0.1.53",
195
+ "@aws-sdk/client-bedrock-agent-runtime": "^3.940.0",
196
+ "@aws-sdk/client-bedrock-runtime": "^3.941.0",
197
+ "@aws-sdk/client-sagemaker-runtime": "^3.940.0",
198
+ "@aws-sdk/credential-provider-sso": "^3.940.0",
198
199
  "@azure/ai-projects": "^1.0.1",
199
200
  "@azure/identity": "^4.13.0",
200
201
  "@azure/msal-node": "^3.8.3",
@@ -205,9 +206,10 @@
205
206
  "@biomejs/cli-linux-x64-musl": "2.3.7",
206
207
  "@biomejs/cli-win32-x64": "2.3.7",
207
208
  "@fal-ai/client": "^1.7.2",
208
- "@ibm-cloud/watsonx-ai": "^1.7.3",
209
+ "@ibm-cloud/watsonx-ai": "^1.7.4",
209
210
  "@ibm-generative-ai/node-sdk": "^3.2.4",
210
- "@playwright/browser-chromium": "^1.56.1",
211
+ "@openai/codex-sdk": "^0.63.0",
212
+ "@playwright/browser-chromium": "^1.57.0",
211
213
  "@rollup/rollup-linux-x64-gnu": "^4.53.3",
212
214
  "@slack/web-api": "^7.12.0",
213
215
  "@smithy/node-http-handler": "^4.4.5",
@@ -223,11 +225,11 @@
223
225
  "natural": "^8.1.0",
224
226
  "node-sql-parser": "^5.3.13",
225
227
  "pdf-parse": "^1.1.4",
226
- "playwright": "^1.56.1",
228
+ "playwright": "^1.57.0",
227
229
  "playwright-extra": "^4.3.6",
228
230
  "puppeteer-extra-plugin-stealth": "^2.11.2",
229
- "sharp": "^0.34.5",
230
- "xlsx": "^0.18.5"
231
+ "read-excel-file": "^6.0.1",
232
+ "sharp": "^0.34.5"
231
233
  },
232
234
  "dependencies": {
233
235
  "@anthropic-ai/sdk": "^0.71.0",
@@ -239,8 +241,8 @@
239
241
  "@inquirer/editor": "^4.2.23",
240
242
  "@inquirer/input": "^4.3.1",
241
243
  "@inquirer/select": "^4.4.2",
242
- "@modelcontextprotocol/sdk": "^1.22.0",
243
- "@openai/agents": "^0.3.2",
244
+ "@modelcontextprotocol/sdk": "1.22.0",
245
+ "@openai/agents": "^0.3.3",
244
246
  "@types/ws": "^8.18.1",
245
247
  "ajv": "^8.17.1",
246
248
  "ajv-formats": "^3.0.1",