promptfoo 0.107.3 → 0.107.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +31 -7
- package/dist/package.json +16 -14
- package/dist/src/app/assets/{index-UtyMLXoC.js → index-CI9xCApR.js} +230 -229
- package/dist/src/app/assets/{index.es-D9No4kw2.js → index.es-C6a8OoRa.js} +2 -2
- package/dist/src/app/assets/source-map-support-BSnVhx0V.js +4 -0
- package/dist/src/app/assets/{sync-CaGQPGW1.js → sync-cEIexnNv.js} +1 -1
- package/dist/src/app/index.html +1 -1
- package/dist/src/assertions/answerRelevance.js +1 -1
- package/dist/src/assertions/contextFaithfulness.js +1 -1
- package/dist/src/assertions/contextRecall.js +1 -1
- package/dist/src/assertions/contextRelevance.js +1 -1
- package/dist/src/assertions/geval.js +1 -1
- package/dist/src/assertions/levenshtein.d.ts.map +1 -1
- package/dist/src/assertions/levenshtein.js +3 -2
- package/dist/src/assertions/levenshtein.js.map +1 -1
- package/dist/src/assertions/rouge.d.ts.map +1 -1
- package/dist/src/assertions/rouge.js +4 -3
- package/dist/src/assertions/rouge.js.map +1 -1
- package/dist/src/assertions/similar.d.ts.map +1 -1
- package/dist/src/assertions/similar.js +3 -2
- package/dist/src/assertions/similar.js.map +1 -1
- package/dist/src/commands/eval/filterTests.d.ts.map +1 -1
- package/dist/src/commands/eval/filterTests.js +11 -2
- package/dist/src/commands/eval/filterTests.js.map +1 -1
- package/dist/src/csv.d.ts.map +1 -1
- package/dist/src/csv.js +29 -3
- package/dist/src/csv.js.map +1 -1
- package/dist/src/golang/wrapper.go +1 -8
- package/dist/src/logger.d.ts +9 -0
- package/dist/src/logger.d.ts.map +1 -1
- package/dist/src/logger.js +114 -9
- package/dist/src/logger.js.map +1 -1
- package/dist/src/providers/azure/assistant.d.ts +19 -0
- package/dist/src/providers/azure/assistant.d.ts.map +1 -1
- package/dist/src/providers/azure/assistant.js +345 -95
- package/dist/src/providers/azure/assistant.js.map +1 -1
- package/dist/src/providers/azure/types.d.ts +0 -4
- package/dist/src/providers/azure/types.d.ts.map +1 -1
- package/dist/src/providers/bedrock.js +1 -1
- package/dist/src/providers/bedrock.js.map +1 -1
- package/dist/src/providers/golangCompletion.d.ts.map +1 -1
- package/dist/src/providers/golangCompletion.js +1 -10
- package/dist/src/providers/golangCompletion.js.map +1 -1
- package/dist/src/providers/googleShared.d.ts.map +1 -1
- package/dist/src/providers/googleShared.js +1 -0
- package/dist/src/providers/googleShared.js.map +1 -1
- package/dist/src/providers/groq.d.ts +9 -0
- package/dist/src/providers/groq.d.ts.map +1 -1
- package/dist/src/providers/openai/responses.d.ts +21 -0
- package/dist/src/providers/openai/responses.d.ts.map +1 -0
- package/dist/src/providers/openai/responses.js +245 -0
- package/dist/src/providers/openai/responses.js.map +1 -0
- package/dist/src/providers/openai/types.d.ts +9 -0
- package/dist/src/providers/openai/types.d.ts.map +1 -1
- package/dist/src/providers/openai/util.d.ts.map +1 -1
- package/dist/src/providers/openai/util.js +80 -22
- package/dist/src/providers/openai/util.js.map +1 -1
- package/dist/src/providers/registry.d.ts.map +1 -1
- package/dist/src/providers/registry.js +7 -0
- package/dist/src/providers/registry.js.map +1 -1
- package/dist/src/providers/simulatedUser.d.ts.map +1 -1
- package/dist/src/providers/simulatedUser.js +7 -6
- package/dist/src/providers/simulatedUser.js.map +1 -1
- package/dist/src/providers/togetherai.d.ts +6 -0
- package/dist/src/providers/togetherai.d.ts.map +1 -1
- package/dist/src/providers/togetherai.js +10 -1
- package/dist/src/providers/togetherai.js.map +1 -1
- package/dist/src/redteam/constants.d.ts +2 -2
- package/dist/src/redteam/constants.d.ts.map +1 -1
- package/dist/src/redteam/constants.js +12 -3
- package/dist/src/redteam/constants.js.map +1 -1
- package/dist/src/redteam/extraction/entities.d.ts.map +1 -1
- package/dist/src/redteam/extraction/entities.js +19 -5
- package/dist/src/redteam/extraction/entities.js.map +1 -1
- package/dist/src/redteam/graders.d.ts.map +1 -1
- package/dist/src/redteam/graders.js +2 -0
- package/dist/src/redteam/graders.js.map +1 -1
- package/dist/src/redteam/plugins/cyberseceval.d.ts.map +1 -1
- package/dist/src/redteam/plugins/cyberseceval.js +1 -0
- package/dist/src/redteam/plugins/cyberseceval.js.map +1 -1
- package/dist/src/redteam/plugins/index.d.ts.map +1 -1
- package/dist/src/redteam/plugins/index.js +1 -0
- package/dist/src/redteam/plugins/index.js.map +1 -1
- package/dist/src/redteam/plugins/pliny.d.ts +9 -1
- package/dist/src/redteam/plugins/pliny.d.ts.map +1 -1
- package/dist/src/redteam/plugins/pliny.js +60 -2
- package/dist/src/redteam/plugins/pliny.js.map +1 -1
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/package.json +16 -14
package/README.md
CHANGED
|
@@ -1,12 +1,24 @@
|
|
|
1
1
|
# Promptfoo: LLM evals & red teaming
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
3
|
+
<p align="center">
|
|
4
|
+
<a href="https://npmjs.com/package/promptfoo"><img src="https://img.shields.io/npm/v/promptfoo" alt="npm"></a>
|
|
5
|
+
<a href="https://npmjs.com/package/promptfoo"><img src="https://img.shields.io/npm/dm/promptfoo" alt="npm"></a>
|
|
6
|
+
<a href="https://github.com/promptfoo/promptfoo/actions/workflows/main.yml"><img src="https://img.shields.io/github/actions/workflow/status/promptfoo/promptfoo/main.yml" alt="GitHub Workflow Status"></a>
|
|
7
|
+
<a href="https://github.com/promptfoo/promptfoo/blob/main/LICENSE"><img src="https://img.shields.io/github/license/promptfoo/promptfoo" alt="MIT license"></a>
|
|
8
|
+
<a href="https://discord.gg/promptfoo"><img src="https://github.com/user-attachments/assets/2092591a-ccc5-42a7-aeb6-24a2808950fd" alt="Discord"></a>
|
|
9
|
+
</p>
|
|
10
|
+
|
|
11
|
+
<p align="center">
|
|
12
|
+
<code>promptfoo</code> is a developer-friendly local tool for testing LLM applications. Stop the trial-and-error approach - start shipping secure, reliable AI apps.
|
|
13
|
+
</p>
|
|
14
|
+
|
|
15
|
+
<p align="center">
|
|
16
|
+
<a href="https://www.promptfoo.dev">Website</a> ·
|
|
17
|
+
<a href="https://www.promptfoo.dev/docs/getting-started/">Getting Started</a> ·
|
|
18
|
+
<a href="https://www.promptfoo.dev/docs/red-team/">Red Teaming</a> ·
|
|
19
|
+
<a href="https://www.promptfoo.dev/docs/">Documentation</a> ·
|
|
20
|
+
<a href="https://discord.gg/promptfoo">Discord</a>
|
|
21
|
+
</p>
|
|
10
22
|
|
|
11
23
|
## Quick Start
|
|
12
24
|
|
|
@@ -49,6 +61,14 @@ It also can generate [security vulnerability reports](https://www.promptfoo.dev/
|
|
|
49
61
|
- 📊 **Data-driven**: Make decisions based on metrics, not gut feel
|
|
50
62
|
- 🤝 **Open source**: MIT licensed, with an active community
|
|
51
63
|
|
|
64
|
+
## Star the Project ⭐
|
|
65
|
+
|
|
66
|
+
If you find promptfoo useful, please star it on GitHub! Stars help the project grow and ensure you stay updated on new releases and features.
|
|
67
|
+
|
|
68
|
+
<p align="center">
|
|
69
|
+
<img src="site/static/img/github/star-animation.gif" alt="Star us on GitHub!" width="800" />
|
|
70
|
+
</p>
|
|
71
|
+
|
|
52
72
|
## Learn More
|
|
53
73
|
|
|
54
74
|
- 📚 [Full Documentation](https://www.promptfoo.dev/docs/intro/)
|
|
@@ -63,3 +83,7 @@ It also can generate [security vulnerability reports](https://www.promptfoo.dev/
|
|
|
63
83
|
We welcome contributions! Check out our [contributing guide](https://www.promptfoo.dev/docs/contributing/) to get started.
|
|
64
84
|
|
|
65
85
|
Join our [Discord community](https://discord.gg/promptfoo) for help and discussion.
|
|
86
|
+
|
|
87
|
+
<a href="https://github.com/promptfoo/promptfoo/graphs/contributors">
|
|
88
|
+
<img src="https://contrib.rocks/image?repo=promptfoo/promptfoo" />
|
|
89
|
+
</a>
|
package/dist/package.json
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
"name": "promptfoo",
|
|
3
3
|
"description": "LLM eval & testing toolkit",
|
|
4
4
|
"author": "Ian Webster",
|
|
5
|
-
"version": "0.107.
|
|
5
|
+
"version": "0.107.5",
|
|
6
6
|
"license": "MIT",
|
|
7
7
|
"type": "commonjs",
|
|
8
8
|
"repository": {
|
|
@@ -57,7 +57,7 @@
|
|
|
57
57
|
"local:web": "npm run dev --prefix src/app",
|
|
58
58
|
"local": "ts-node --cwdMode --transpileOnly src/main.ts",
|
|
59
59
|
"preversion": "[ \"$(git rev-parse --abbrev-ref HEAD)\" = \"main\" ] || (echo \"Error: Must be on main branch to version\" && exit 1) && git pull origin main && git checkout -b \"chore/bump-version-$(date +%s)\"",
|
|
60
|
-
"postversion": "npm run citation:generate && git add CITATION.cff && git commit --amend --no-edit && gh pr create --title \"chore: bump version $npm_package_version\" --body \"\"",
|
|
60
|
+
"postversion": "npm run citation:generate && git add CITATION.cff && git commit --amend --no-edit && gh pr create --repo promptfoo/promptfoo --title \"chore: bump version $npm_package_version\" --body \"\"",
|
|
61
61
|
"prepublishOnly": "npm run build:clean && npm run build",
|
|
62
62
|
"test:app": "npm run test --prefix src/app",
|
|
63
63
|
"test:integration": "jest --config jest.integration.config.ts integration",
|
|
@@ -108,13 +108,13 @@
|
|
|
108
108
|
"sharp": "^0.33.5"
|
|
109
109
|
},
|
|
110
110
|
"devDependencies": {
|
|
111
|
-
"@aws-sdk/client-bedrock-runtime": "^3.
|
|
112
|
-
"@aws-sdk/credential-provider-sso": "^3.
|
|
111
|
+
"@aws-sdk/client-bedrock-runtime": "^3.772.0",
|
|
112
|
+
"@aws-sdk/credential-provider-sso": "^3.772.0",
|
|
113
113
|
"@azure/identity": "^4.8.0",
|
|
114
114
|
"@eslint/js": "^9.22.0",
|
|
115
115
|
"@libsql/client": "^0.14.0",
|
|
116
116
|
"@swc/cli": "^0.6.0",
|
|
117
|
-
"@swc/core": "^1.11.
|
|
117
|
+
"@swc/core": "^1.11.11",
|
|
118
118
|
"@swc/jest": "^0.2.37",
|
|
119
119
|
"@trivago/prettier-plugin-sort-imports": "^5.2.2",
|
|
120
120
|
"@types/async": "^3.2.24",
|
|
@@ -125,7 +125,7 @@
|
|
|
125
125
|
"@types/compression": "^1.7.5",
|
|
126
126
|
"@types/cors": "^2.8.17",
|
|
127
127
|
"@types/debounce": "^1.2.4",
|
|
128
|
-
"@types/express": "^5.0.
|
|
128
|
+
"@types/express": "^5.0.1",
|
|
129
129
|
"@types/glob": "^8.1.0",
|
|
130
130
|
"@types/inquirer": "^9.0.7",
|
|
131
131
|
"@types/jest": "^29.5.14",
|
|
@@ -134,10 +134,11 @@
|
|
|
134
134
|
"@types/opener": "^1.4.3",
|
|
135
135
|
"@types/pdf-parse": "^1.1.4",
|
|
136
136
|
"@types/semver": "^7.5.8",
|
|
137
|
+
"@types/source-map-support": "^0.5.10",
|
|
137
138
|
"@types/supertest": "^6.0.2",
|
|
138
139
|
"@types/uuid": "^10.0.0",
|
|
139
|
-
"@typescript-eslint/eslint-plugin": "^8.
|
|
140
|
-
"@typescript-eslint/parser": "^8.
|
|
140
|
+
"@typescript-eslint/eslint-plugin": "^8.27.0",
|
|
141
|
+
"@typescript-eslint/parser": "^8.27.0",
|
|
141
142
|
"check-dependency-version-consistency": "^5.0.0",
|
|
142
143
|
"cloudflare": "3.4.0",
|
|
143
144
|
"concurrently": "^9.1.2",
|
|
@@ -153,17 +154,18 @@
|
|
|
153
154
|
"nock": "^13.5.6",
|
|
154
155
|
"nodemon": "^3.1.9",
|
|
155
156
|
"prettier": "^3.5.3",
|
|
156
|
-
"shx": "^0.
|
|
157
|
-
"
|
|
157
|
+
"shx": "^0.4.0",
|
|
158
|
+
"source-map-support": "^0.5.21",
|
|
159
|
+
"supertest": "^7.1.0",
|
|
158
160
|
"ts-node": "^10.9.2",
|
|
159
161
|
"typescript": "^5.8.2",
|
|
160
|
-
"typescript-eslint": "^8.
|
|
161
|
-
"zod-to-json-schema": "^3.24.
|
|
162
|
+
"typescript-eslint": "^8.27.0",
|
|
163
|
+
"zod-to-json-schema": "^3.24.5"
|
|
162
164
|
},
|
|
163
165
|
"dependencies": {
|
|
164
166
|
"@anthropic-ai/sdk": "^0.39.0",
|
|
165
167
|
"@apidevtools/json-schema-ref-parser": "^11.9.3",
|
|
166
|
-
"@googleapis/sheets": "^9.
|
|
168
|
+
"@googleapis/sheets": "^9.6.0",
|
|
167
169
|
"@types/ws": "^8.18.0",
|
|
168
170
|
"ajv": "^8.17.1",
|
|
169
171
|
"ajv-formats": "^3.0.1",
|
|
@@ -196,7 +198,7 @@
|
|
|
196
198
|
"mathjs": "^14.3.1",
|
|
197
199
|
"node-cache": "^5.1.2",
|
|
198
200
|
"nunjucks": "^3.2.4",
|
|
199
|
-
"openai": "^4.
|
|
201
|
+
"openai": "^4.89.0",
|
|
200
202
|
"opener": "^1.5.2",
|
|
201
203
|
"proxy-agent": "^6.5.0",
|
|
202
204
|
"python-shell": "^5.0.0",
|