promptfoo 0.97.0 → 0.99.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/package.json +19 -17
- package/dist/src/app/assets/index-CJGQrbZb.js +787 -0
- package/dist/src/app/assets/{index-YwjS5Vq2.css → index-DatcZSwS.css} +1 -1
- package/dist/src/app/assets/{index.es-Cc9N3uII.js → index.es-BCwhwRu5.js} +1 -1
- package/dist/src/app/assets/{sync-BoVzjiAe.js → sync-hJWHXawF.js} +1 -1
- package/dist/src/app/index.html +2 -2
- package/dist/src/assertions/index.d.ts.map +1 -1
- package/dist/src/assertions/index.js +9 -1
- package/dist/src/assertions/index.js.map +1 -1
- package/dist/src/cache.d.ts +1 -0
- package/dist/src/cache.d.ts.map +1 -1
- package/dist/src/cache.js +5 -0
- package/dist/src/cache.js.map +1 -1
- package/dist/src/commands/debug.d.ts +4 -0
- package/dist/src/commands/debug.d.ts.map +1 -0
- package/dist/src/commands/debug.js +84 -0
- package/dist/src/commands/debug.js.map +1 -0
- package/dist/src/commands/view.d.ts.map +1 -1
- package/dist/src/commands/view.js +6 -4
- package/dist/src/commands/view.js.map +1 -1
- package/dist/src/constants.d.ts +1 -0
- package/dist/src/constants.d.ts.map +1 -1
- package/dist/src/constants.js +2 -1
- package/dist/src/constants.js.map +1 -1
- package/dist/src/database/tables.d.ts +176 -176
- package/dist/src/envars.d.ts +1 -1
- package/dist/src/envars.d.ts.map +1 -1
- package/dist/src/evaluator.d.ts.map +1 -1
- package/dist/src/evaluator.js +7 -3
- package/dist/src/evaluator.js.map +1 -1
- package/dist/src/evaluatorHelpers.d.ts +1 -0
- package/dist/src/evaluatorHelpers.d.ts.map +1 -1
- package/dist/src/evaluatorHelpers.js +19 -0
- package/dist/src/evaluatorHelpers.js.map +1 -1
- package/dist/src/logger.js +10 -10
- package/dist/src/logger.js.map +1 -1
- package/dist/src/main.js +17 -1
- package/dist/src/main.js.map +1 -1
- package/dist/src/models/eval.d.ts +1 -0
- package/dist/src/models/eval.d.ts.map +1 -1
- package/dist/src/models/eval.js +2 -0
- package/dist/src/models/eval.js.map +1 -1
- package/dist/src/prompts/processors/yaml.d.ts.map +1 -1
- package/dist/src/prompts/processors/yaml.js +7 -2
- package/dist/src/prompts/processors/yaml.js.map +1 -1
- package/dist/src/providers/adaline.gateway.d.ts +2 -1
- package/dist/src/providers/adaline.gateway.d.ts.map +1 -1
- package/dist/src/providers/ai21.d.ts +2 -1
- package/dist/src/providers/ai21.d.ts.map +1 -1
- package/dist/src/providers/ai21.js.map +1 -1
- package/dist/src/providers/anthropic.d.ts +2 -1
- package/dist/src/providers/anthropic.d.ts.map +1 -1
- package/dist/src/providers/anthropic.js.map +1 -1
- package/dist/src/providers/azure.d.ts +2 -1
- package/dist/src/providers/azure.d.ts.map +1 -1
- package/dist/src/providers/bam.d.ts +2 -1
- package/dist/src/providers/bam.d.ts.map +1 -1
- package/dist/src/providers/bam.js.map +1 -1
- package/dist/src/providers/bedrock.d.ts +2 -1
- package/dist/src/providers/bedrock.d.ts.map +1 -1
- package/dist/src/providers/cloudflare-ai.d.ts +2 -1
- package/dist/src/providers/cloudflare-ai.d.ts.map +1 -1
- package/dist/src/providers/cohere.d.ts +2 -1
- package/dist/src/providers/cohere.d.ts.map +1 -1
- package/dist/src/providers/defaults.d.ts +2 -1
- package/dist/src/providers/defaults.d.ts.map +1 -1
- package/dist/src/providers/defaults.js.map +1 -1
- package/dist/src/providers/fal.d.ts +2 -1
- package/dist/src/providers/fal.d.ts.map +1 -1
- package/dist/src/providers/fal.js.map +1 -1
- package/dist/src/providers/groq.d.ts +2 -1
- package/dist/src/providers/groq.d.ts.map +1 -1
- package/dist/src/providers/http.d.ts +5 -1
- package/dist/src/providers/http.d.ts.map +1 -1
- package/dist/src/providers/http.js +67 -10
- package/dist/src/providers/http.js.map +1 -1
- package/dist/src/providers/localai.d.ts +2 -1
- package/dist/src/providers/localai.d.ts.map +1 -1
- package/dist/src/providers/localai.js.map +1 -1
- package/dist/src/providers/mistral.d.ts +2 -1
- package/dist/src/providers/mistral.d.ts.map +1 -1
- package/dist/src/providers/openai.d.ts +2 -1
- package/dist/src/providers/openai.d.ts.map +1 -1
- package/dist/src/providers/openai.js +1 -1
- package/dist/src/providers/openai.js.map +1 -1
- package/dist/src/providers/palm.d.ts +2 -1
- package/dist/src/providers/palm.d.ts.map +1 -1
- package/dist/src/providers/palm.js.map +1 -1
- package/dist/src/providers/promptfoo.d.ts +2 -1
- package/dist/src/providers/promptfoo.d.ts.map +1 -1
- package/dist/src/providers/replicate.d.ts +2 -1
- package/dist/src/providers/replicate.d.ts.map +1 -1
- package/dist/src/providers/sequence.d.ts +11 -0
- package/dist/src/providers/sequence.d.ts.map +1 -0
- package/dist/src/providers/sequence.js +66 -0
- package/dist/src/providers/sequence.js.map +1 -0
- package/dist/src/providers/togetherai.d.ts +2 -1
- package/dist/src/providers/togetherai.d.ts.map +1 -1
- package/dist/src/providers/togetherai.js.map +1 -1
- package/dist/src/providers/vertex.d.ts +2 -1
- package/dist/src/providers/vertex.d.ts.map +1 -1
- package/dist/src/providers/watsonx.d.ts +2 -1
- package/dist/src/providers/watsonx.d.ts.map +1 -1
- package/dist/src/providers/watsonx.js.map +1 -1
- package/dist/src/providers/xai.d.ts +2 -1
- package/dist/src/providers/xai.d.ts.map +1 -1
- package/dist/src/providers/xai.js.map +1 -1
- package/dist/src/providers.d.ts +2 -1
- package/dist/src/providers.d.ts.map +1 -1
- package/dist/src/providers.js +4 -0
- package/dist/src/providers.js.map +1 -1
- package/dist/src/redteam/commands/generate.d.ts.map +1 -1
- package/dist/src/redteam/commands/generate.js +25 -4
- package/dist/src/redteam/commands/generate.js.map +1 -1
- package/dist/src/redteam/commands/init.d.ts.map +1 -1
- package/dist/src/redteam/commands/init.js +79 -64
- package/dist/src/redteam/commands/init.js.map +1 -1
- package/dist/src/redteam/commands/report.d.ts.map +1 -1
- package/dist/src/redteam/commands/report.js +11 -3
- package/dist/src/redteam/commands/report.js.map +1 -1
- package/dist/src/redteam/commands/setup.d.ts +3 -0
- package/dist/src/redteam/commands/setup.d.ts.map +1 -0
- package/dist/src/redteam/commands/setup.js +33 -0
- package/dist/src/redteam/commands/setup.js.map +1 -0
- package/dist/src/redteam/constants.d.ts +4 -4
- package/dist/src/redteam/constants.d.ts.map +1 -1
- package/dist/src/redteam/constants.js +12 -2
- package/dist/src/redteam/constants.js.map +1 -1
- package/dist/src/redteam/index.d.ts.map +1 -1
- package/dist/src/redteam/index.js +95 -37
- package/dist/src/redteam/index.js.map +1 -1
- package/dist/src/redteam/plugins/base.d.ts.map +1 -1
- package/dist/src/redteam/plugins/base.js +2 -0
- package/dist/src/redteam/plugins/base.js.map +1 -1
- package/dist/src/redteam/plugins/competitors.d.ts.map +1 -1
- package/dist/src/redteam/plugins/competitors.js +23 -1
- package/dist/src/redteam/plugins/competitors.js.map +1 -1
- package/dist/src/redteam/plugins/debugAccess.d.ts.map +1 -1
- package/dist/src/redteam/plugins/debugAccess.js +18 -0
- package/dist/src/redteam/plugins/debugAccess.js.map +1 -1
- package/dist/src/redteam/plugins/harmful/constants.d.ts.map +1 -1
- package/dist/src/redteam/plugins/harmful/constants.js +32 -8
- package/dist/src/redteam/plugins/harmful/constants.js.map +1 -1
- package/dist/src/redteam/plugins/harmful/unaligned.js +1 -1
- package/dist/src/redteam/plugins/harmful/unaligned.js.map +1 -1
- package/dist/src/redteam/plugins/intent.d.ts.map +1 -1
- package/dist/src/redteam/plugins/intent.js +6 -0
- package/dist/src/redteam/plugins/intent.js.map +1 -1
- package/dist/src/redteam/providers/crescendo/index.d.ts +3 -0
- package/dist/src/redteam/providers/crescendo/index.d.ts.map +1 -1
- package/dist/src/redteam/providers/crescendo/index.js +66 -37
- package/dist/src/redteam/providers/crescendo/index.js.map +1 -1
- package/dist/src/redteam/providers/goat.d.ts.map +1 -1
- package/dist/src/redteam/providers/goat.js +56 -41
- package/dist/src/redteam/providers/goat.js.map +1 -1
- package/dist/src/redteam/providers/iterative.d.ts +1 -0
- package/dist/src/redteam/providers/iterative.d.ts.map +1 -1
- package/dist/src/redteam/providers/iterative.js +14 -9
- package/dist/src/redteam/providers/iterative.js.map +1 -1
- package/dist/src/redteam/providers/iterativeImage.d.ts +3 -1
- package/dist/src/redteam/providers/iterativeImage.d.ts.map +1 -1
- package/dist/src/redteam/providers/iterativeImage.js +14 -8
- package/dist/src/redteam/providers/iterativeImage.js.map +1 -1
- package/dist/src/redteam/providers/iterativeTree.d.ts.map +1 -1
- package/dist/src/redteam/providers/iterativeTree.js +26 -19
- package/dist/src/redteam/providers/iterativeTree.js.map +1 -1
- package/dist/src/redteam/providers/shared.d.ts +9 -6
- package/dist/src/redteam/providers/shared.d.ts.map +1 -1
- package/dist/src/redteam/providers/shared.js +9 -16
- package/dist/src/redteam/providers/shared.js.map +1 -1
- package/dist/src/redteam/strategies/citation.d.ts +3 -0
- package/dist/src/redteam/strategies/citation.d.ts.map +1 -0
- package/dist/src/redteam/strategies/citation.js +96 -0
- package/dist/src/redteam/strategies/citation.js.map +1 -0
- package/dist/src/redteam/strategies/index.d.ts +3 -2
- package/dist/src/redteam/strategies/index.d.ts.map +1 -1
- package/dist/src/redteam/strategies/index.js +68 -13
- package/dist/src/redteam/strategies/index.js.map +1 -1
- package/dist/src/redteam/strategies/singleTurnComposite.d.ts +3 -0
- package/dist/src/redteam/strategies/singleTurnComposite.d.ts.map +1 -0
- package/dist/src/redteam/strategies/singleTurnComposite.js +89 -0
- package/dist/src/redteam/strategies/singleTurnComposite.js.map +1 -0
- package/dist/src/redteam/types.d.ts +5 -1
- package/dist/src/redteam/types.d.ts.map +1 -1
- package/dist/src/server/index.js +4 -2
- package/dist/src/server/index.js.map +1 -1
- package/dist/src/server/routes/providers.js +1 -0
- package/dist/src/server/routes/providers.js.map +1 -1
- package/dist/src/server/server.d.ts +1 -6
- package/dist/src/server/server.d.ts.map +1 -1
- package/dist/src/server/server.js +13 -43
- package/dist/src/server/server.js.map +1 -1
- package/dist/src/testCases.js +1 -1
- package/dist/src/testCases.js.map +1 -1
- package/dist/src/types/env.d.ts +54 -0
- package/dist/src/types/env.d.ts.map +1 -0
- package/dist/src/types/env.js +3 -0
- package/dist/src/types/env.js.map +1 -0
- package/dist/src/types/index.d.ts +8395 -6838
- package/dist/src/types/index.d.ts.map +1 -1
- package/dist/src/types/index.js +2 -1
- package/dist/src/types/index.js.map +1 -1
- package/dist/src/types/providers.d.ts +3 -53
- package/dist/src/types/providers.d.ts.map +1 -1
- package/dist/src/types/providers.js.map +1 -1
- package/dist/src/util/config/load.js +1 -1
- package/dist/src/util/config/load.js.map +1 -1
- package/dist/src/util/index.d.ts +32 -32
- package/dist/src/util/server.d.ts +10 -0
- package/dist/src/util/server.d.ts.map +1 -0
- package/dist/src/util/server.js +74 -0
- package/dist/src/util/server.js.map +1 -0
- package/dist/src/util/transform.d.ts +1 -0
- package/dist/src/util/transform.d.ts.map +1 -1
- package/dist/src/util/transform.js.map +1 -1
- package/dist/src/validators/providers.d.ts +296 -296
- package/dist/src/validators/redteam.d.ts +130 -126
- package/dist/src/validators/redteam.d.ts.map +1 -1
- package/dist/src/validators/redteam.js +18 -2
- package/dist/src/validators/redteam.js.map +1 -1
- package/dist/test/cache.test.js +47 -7
- package/dist/test/cache.test.js.map +1 -1
- package/dist/test/commands/init.test.js +14 -0
- package/dist/test/commands/init.test.js.map +1 -1
- package/dist/test/evaluator.test.js +50 -0
- package/dist/test/evaluator.test.js.map +1 -1
- package/dist/test/factories/evalFactory.d.ts +144 -144
- package/dist/test/prompts/index.test.js +38 -25
- package/dist/test/prompts/index.test.js.map +1 -1
- package/dist/test/prompts/processors/yaml.test.js +50 -4
- package/dist/test/prompts/processors/yaml.test.js.map +1 -1
- package/dist/test/providers/http.test.js +0 -17
- package/dist/test/providers/http.test.js.map +1 -1
- package/dist/test/providers/index.test.js +1 -0
- package/dist/test/providers/index.test.js.map +1 -1
- package/dist/test/providers/togetherai.test.js.map +1 -1
- package/dist/test/redteam/commands/generate.test.js +58 -0
- package/dist/test/redteam/commands/generate.test.js.map +1 -1
- package/dist/test/redteam/index.test.js +2 -2
- package/dist/test/redteam/index.test.js.map +1 -1
- package/dist/test/redteam/plugins/base.test.js +8 -0
- package/dist/test/redteam/plugins/base.test.js.map +1 -1
- package/dist/test/redteam/providers/iterativeTree.test.js +11 -3
- package/dist/test/redteam/providers/iterativeTree.test.js.map +1 -1
- package/dist/test/testCases.test.js +21 -0
- package/dist/test/testCases.test.js.map +1 -1
- package/dist/test/types.test.js +17 -4
- package/dist/test/types.test.js.map +1 -1
- package/dist/test/util/server.test.d.ts +2 -0
- package/dist/test/util/server.test.d.ts.map +1 -0
- package/dist/test/util/server.test.js +114 -0
- package/dist/test/util/server.test.js.map +1 -0
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/package.json +19 -17
- package/dist/src/app/assets/index-CbxDbiAR.js +0 -769
package/README.md
CHANGED
|
@@ -332,7 +332,7 @@ See the full example [here](https://github.com/promptfoo/promptfoo/tree/main/exa
|
|
|
332
332
|
## Configuration
|
|
333
333
|
|
|
334
334
|
- **[Main guide](https://www.promptfoo.dev/docs/configuration/guide)**: Learn about how to configure your YAML file, setup prompt files, etc.
|
|
335
|
-
- **[Configuring test cases](https://www.promptfoo.dev/docs/configuration/expected-outputs)**: Learn more about how to configure
|
|
335
|
+
- **[Configuring test cases](https://www.promptfoo.dev/docs/configuration/expected-outputs)**: Learn more about how to configure assertions and metrics.
|
|
336
336
|
|
|
337
337
|
## Installation
|
|
338
338
|
|
package/dist/package.json
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
"name": "promptfoo",
|
|
3
3
|
"description": "LLM eval & testing toolkit",
|
|
4
4
|
"author": "Ian Webster",
|
|
5
|
-
"version": "0.
|
|
5
|
+
"version": "0.99.0",
|
|
6
6
|
"license": "MIT",
|
|
7
7
|
"type": "commonjs",
|
|
8
8
|
"repository": {
|
|
@@ -77,22 +77,23 @@
|
|
|
77
77
|
"ibm-cloud-sdk-core": "^5.0.2",
|
|
78
78
|
"langfuse": "^3.7.0",
|
|
79
79
|
"node-sql-parser": "^5.2.0",
|
|
80
|
+
"pdf-parse": "^1.1.1",
|
|
80
81
|
"playwright": "^1.47.2",
|
|
81
82
|
"playwright-extra": "^4.3.6",
|
|
82
83
|
"puppeteer-extra-plugin-stealth": "^2.11.2"
|
|
83
84
|
},
|
|
84
85
|
"devDependencies": {
|
|
85
|
-
"@aws-sdk/client-bedrock-runtime": "^3.
|
|
86
|
-
"@aws-sdk/credential-provider-sso": "^3.
|
|
86
|
+
"@aws-sdk/client-bedrock-runtime": "^3.699.0",
|
|
87
|
+
"@aws-sdk/credential-provider-sso": "^3.699.0",
|
|
87
88
|
"@azure/identity": "^4.5.0",
|
|
88
|
-
"@eslint/js": "^9.
|
|
89
|
+
"@eslint/js": "^9.15.0",
|
|
89
90
|
"@libsql/client": "^0.14.0",
|
|
90
|
-
"@swc/cli": "^0.5.
|
|
91
|
-
"@swc/core": "^1.9.
|
|
91
|
+
"@swc/cli": "^0.5.1",
|
|
92
|
+
"@swc/core": "^1.9.3",
|
|
92
93
|
"@swc/jest": "^0.2.37",
|
|
93
94
|
"@trivago/prettier-plugin-sort-imports": "^4.3.0",
|
|
94
95
|
"@types/async": "^3.2.24",
|
|
95
|
-
"@types/better-sqlite3": "^7.6.
|
|
96
|
+
"@types/better-sqlite3": "^7.6.12",
|
|
96
97
|
"@types/cache-manager": "^4.0.6",
|
|
97
98
|
"@types/cache-manager-fs-hash": "^0.0.4",
|
|
98
99
|
"@types/cli-progress": "^3.11.6",
|
|
@@ -107,6 +108,7 @@
|
|
|
107
108
|
"@types/js-yaml": "^4.0.9",
|
|
108
109
|
"@types/nunjucks": "^3.2.6",
|
|
109
110
|
"@types/opener": "^1.4.3",
|
|
111
|
+
"@types/pdf-parse": "^1.1.4",
|
|
110
112
|
"@types/semver": "^7.5.8",
|
|
111
113
|
"@types/supertest": "^6.0.2",
|
|
112
114
|
"@types/uuid": "^10.0.0",
|
|
@@ -115,11 +117,11 @@
|
|
|
115
117
|
"concurrently": "^9.1.0",
|
|
116
118
|
"drizzle-kit": "^0.26.2",
|
|
117
119
|
"esbuild": "^0.24.0",
|
|
118
|
-
"eslint": "^9.
|
|
120
|
+
"eslint": "^9.15.0",
|
|
119
121
|
"eslint-plugin-jest": "^28.9.0",
|
|
120
122
|
"eslint-plugin-react-hooks": "^5.1.0-rc.0",
|
|
121
123
|
"eslint-plugin-react-refresh": "^0.4.14",
|
|
122
|
-
"eslint-plugin-unicorn": "^56.0.
|
|
124
|
+
"eslint-plugin-unicorn": "^56.0.1",
|
|
123
125
|
"eslint-plugin-unused-imports": "^4.1.4",
|
|
124
126
|
"jest": "^29.7.0",
|
|
125
127
|
"jest-watch-typeahead": "^2.2.2",
|
|
@@ -132,7 +134,7 @@
|
|
|
132
134
|
"ts-node": "^10.9.2",
|
|
133
135
|
"tsconfig-paths": "^4.2.0",
|
|
134
136
|
"typescript": "^5.6.3",
|
|
135
|
-
"typescript-eslint": "^8.
|
|
137
|
+
"typescript-eslint": "^8.15.0",
|
|
136
138
|
"zod-to-json-schema": "^3.23.5"
|
|
137
139
|
},
|
|
138
140
|
"dependencies": {
|
|
@@ -149,10 +151,10 @@
|
|
|
149
151
|
"@adaline/vertex": "0.8.1",
|
|
150
152
|
"@anthropic-ai/sdk": "^0.32.1",
|
|
151
153
|
"@apidevtools/json-schema-ref-parser": "^11.7.2",
|
|
152
|
-
"@emotion/react": "^11.13.
|
|
153
|
-
"@emotion/styled": "^11.13.
|
|
154
|
+
"@emotion/react": "^11.13.5",
|
|
155
|
+
"@emotion/styled": "^11.13.5",
|
|
154
156
|
"@googleapis/sheets": "^9.3.1",
|
|
155
|
-
"@mui/material": "^6.1.
|
|
157
|
+
"@mui/material": "^6.1.8",
|
|
156
158
|
"ajv": "^8.17.1",
|
|
157
159
|
"ajv-formats": "^3.0.1",
|
|
158
160
|
"async": "^3.2.6",
|
|
@@ -167,7 +169,7 @@
|
|
|
167
169
|
"compression": "^1.7.5",
|
|
168
170
|
"cors": "^2.8.5",
|
|
169
171
|
"csv-parse": "^5.5.6",
|
|
170
|
-
"csv-stringify": "^6.5.
|
|
172
|
+
"csv-stringify": "^6.5.2",
|
|
171
173
|
"debounce": "^1.2.1",
|
|
172
174
|
"dedent": "^1.5.3",
|
|
173
175
|
"dotenv": "^16.4.5",
|
|
@@ -182,10 +184,10 @@
|
|
|
182
184
|
"inquirer": "^11.1.0",
|
|
183
185
|
"js-rouge": "3.0.0",
|
|
184
186
|
"js-yaml": "^4.1.0",
|
|
185
|
-
"mathjs": "^13.2.
|
|
187
|
+
"mathjs": "^13.2.2",
|
|
186
188
|
"node-cache": "^5.1.2",
|
|
187
189
|
"nunjucks": "^3.2.4",
|
|
188
|
-
"openai": "^4.
|
|
190
|
+
"openai": "^4.73.0",
|
|
189
191
|
"opener": "^1.5.2",
|
|
190
192
|
"proxy-agent": "^6.4.0",
|
|
191
193
|
"python-shell": "^5.0.0",
|
|
@@ -195,7 +197,7 @@
|
|
|
195
197
|
"socket.io": "^4.8.1",
|
|
196
198
|
"tiny-invariant": "^1.3.3",
|
|
197
199
|
"uuid": "^10.0.0",
|
|
198
|
-
"winston": "^3.
|
|
200
|
+
"winston": "^3.17.0",
|
|
199
201
|
"ws": "^8.18.0",
|
|
200
202
|
"zod": "^3.23.8",
|
|
201
203
|
"zod-validation-error": "^3.4.0"
|