promptfoo 0.93.2 → 0.94.0

package/dist/package.json

@@ -2,7 +2,7 @@
     "name": "promptfoo",
     "description": "LLM eval & testing toolkit",
     "author": "Ian Webster",
-    "version": "0.93.2",
+    "version": "0.94.0",
     "license": "MIT",
     "type": "commonjs",
     "repository": {
@@ -51,6 +51,7 @@
         "format": "prettier -w .",
         "format:check": "prettier --check .",
         "db:migrate": "npx tsx src/migrate.ts",
+        "citation:generate": "ts-node scripts/generateCitation.ts",
         "db:generate": "npx drizzle-kit generate",
         "jsonSchema:generate": "npx -y tsx scripts/generateJsonSchema.ts > site/static/config-schema.json",
         "audit:fix": "npm audit fix && npm audit fix --prefix src/app && npm audit fix --prefix site",
@@ -60,10 +61,7 @@
         "dev": "concurrently \"npm run dev:server\" \"npm run dev:app\""
     },
     "overrides": {
-        "uri-js": "npm:uri-js-replace",
-        "node-fetch": {
-            "whatwg-url": "14.x"
-        }
+        "uri-js": "npm:uri-js-replace"
     },
     "peerDependencies": {
         "@aws-sdk/client-bedrock-runtime": "^3.602.0",
@@ -83,12 +81,12 @@
         "puppeteer-extra-plugin-stealth": "^2.11.2"
     },
     "devDependencies": {
-        "@aws-sdk/client-bedrock-runtime": "^3.670.0",
-        "@azure/identity": "^4.4.1",
-        "@eslint/js": "^9.12.0",
+        "@aws-sdk/client-bedrock-runtime": "^3.677.0",
+        "@azure/identity": "^4.5.0",
+        "@eslint/js": "^9.13.0",
         "@libsql/client": "^0.14.0",
         "@swc/cli": "^0.4.0",
-        "@swc/core": "^1.7.35",
+        "@swc/core": "^1.7.39",
         "@swc/jest": "^0.2.36",
         "@trivago/prettier-plugin-sort-imports": "^4.3.0",
         "@types/async": "^3.2.24",
@@ -105,7 +103,6 @@
         "@types/inquirer": "^9.0.7",
         "@types/jest": "^29.5.13",
         "@types/js-yaml": "^4.0.9",
-        "@types/node-fetch": "^2.6.11",
         "@types/nunjucks": "^3.2.6",
         "@types/opener": "^1.4.3",
         "@types/semver": "^7.5.8",
@@ -114,11 +111,12 @@
         "check-dependency-version-consistency": "^4.1.0",
         "cloudflare": "3.4.0",
         "concurrently": "^9.0.1",
-        "drizzle-kit": "^0.25.0",
+        "drizzle-kit": "^0.26.2",
         "esbuild": "^0.24.0",
-        "eslint": "^9.12.0",
+        "eslint": "^9.13.0",
         "eslint-plugin-jest": "^28.8.3",
         "eslint-plugin-react-hooks": "^5.1.0-rc.0",
+        "eslint-plugin-react-refresh": "^0.4.13",
         "eslint-plugin-unicorn": "^56.0.0",
         "eslint-plugin-unused-imports": "^4.1.4",
         "jest": "^29.7.0",
@@ -132,20 +130,20 @@
         "ts-node": "^10.9.2",
         "tsconfig-paths": "^4.2.0",
         "typescript": "^5.6.3",
-        "typescript-eslint": "^8.9.0",
+        "typescript-eslint": "^8.11.0",
         "zod-to-json-schema": "^3.23.3"
     },
     "dependencies": {
-        "@anthropic-ai/sdk": "^0.29.0",
+        "@anthropic-ai/sdk": "^0.30.0",
         "@apidevtools/json-schema-ref-parser": "^11.7.2",
         "@emotion/react": "^11.13.3",
         "@emotion/styled": "^11.13.0",
         "@googleapis/sheets": "^9.3.1",
-        "@mui/material": "^6.1.3",
+        "@mui/material": "^6.1.4",
         "ajv": "^8.17.1",
         "ajv-formats": "^3.0.1",
         "async": "^3.2.6",
-        "better-sqlite3": "^11.3.0",
+        "better-sqlite3": "^11.4.0",
         "cache-manager": "^4.1.0",
         "cache-manager-fs-hash": "^1.0.0",
         "chalk": "^4.1.2",
@@ -160,7 +158,7 @@
         "debounce": "^1.2.1",
         "dedent": "^1.5.3",
         "dotenv": "^16.4.5",
-        "drizzle-orm": "^0.34.1",
+        "drizzle-orm": "^0.35.2",
         "express": "^4.21.1",
         "fast-deep-equal": "^3.1.3",
         "fast-xml-parser": "^4.5.0",
@@ -173,9 +171,8 @@
         "js-yaml": "^4.1.0",
         "mathjs": "^13.2.0",
         "node-cache": "^5.1.2",
-        "node-fetch": "^2.6.7",
         "nunjucks": "^3.2.4",
-        "openai": "^4.67.3",
+        "openai": "^4.68.2",
         "opener": "^1.5.2",
         "proxy-agent": "^6.4.0",
         "python-shell": "^5.0.0",

package/dist/src/app/assets/index-CKeaKcHn.css

@@ -0,0 +1 @@
+.nav{padding:.25rem 0 .25rem 1rem;gap:1rem;background-color:#333;margin-bottom:1rem;box-shadow:0 4px 6px #0000001a}[data-theme=dark] .nav{background-color:#121212}.nav a{text-decoration:none;align-self:center}@media (max-width: 760px){.nav{font-size:.75rem}}.nav .active{font-weight:700}.nav a{color:#f0f0f0}.nav .right-aligned{display:flex;align-items:center;gap:1rem;margin-left:auto;margin-right:.5rem}.nav a:hover{text-decoration:underline}.nav a:hover{color:#ddd}.prompt-var-highlight{background-color:var(--prompt-highlight-color);padding:.25rem;border-radius:4px}.glowing-border{border:1px solid #5cb3ff;box-shadow:0 0 8px 2px #5cb3ff}code[class*=language-],pre[class*=language-]{color:#000;background:none;text-shadow:0 1px white;font-family:Consolas,Monaco,Andale Mono,Ubuntu Mono,monospace;font-size:1em;text-align:left;white-space:pre;word-spacing:normal;word-break:normal;word-wrap:normal;line-height:1.5;-moz-tab-size:4;-o-tab-size:4;tab-size:4;-webkit-hyphens:none;-moz-hyphens:none;-ms-hyphens:none;hyphens:none}pre[class*=language-]::-moz-selection,pre[class*=language-] ::-moz-selection,code[class*=language-]::-moz-selection,code[class*=language-] ::-moz-selection{text-shadow:none;background:#b3d4fc}pre[class*=language-]::selection,pre[class*=language-] ::selection,code[class*=language-]::selection,code[class*=language-] ::selection{text-shadow:none;background:#b3d4fc}@media print{code[class*=language-],pre[class*=language-]{text-shadow:none}}pre[class*=language-]{padding:1em;margin:.5em 0;overflow:auto}:not(pre)>code[class*=language-],pre[class*=language-]{background:#f5f2f0}:not(pre)>code[class*=language-]{padding:.1em;border-radius:.3em;white-space:normal}.token.comment,.token.prolog,.token.doctype,.token.cdata{color:#708090}.token.punctuation{color:#999}.token.namespace{opacity:.7}.token.property,.token.tag,.token.boolean,.token.number,.token.constant,.token.symbol,.token.deleted{color:#905}.token.selector,.token.attr-name,.token.string,.token.char,.token.builtin,.token.inserted{color:#690}.token.operator,.token.entity,.token.url,.language-css .token.string,.style .token.string{color:#9a6e3a;background:#ffffff80}.token.atrule,.token.attr-value,.token.keyword{color:#07a}.token.function,.token.class-name{color:#dd4a68}.token.regex,.token.important,.token.variable{color:#e90}.token.important,.token.bold{font-weight:700}.token.italic{font-style:italic}.token.entity{cursor:help}.yaml-config{font-size:10px}.custom-metric-container{display:inline-flex;flex-wrap:wrap;gap:.25rem;font-weight:400}.custom-metric-container>span{padding:.25rem .5rem;border-radius:4px;color:var(--text-color);border:1px solid var(--border-color);margin-right:.25rem}.custom-metric-container>span.clickable{cursor:pointer}.custom-metric-container>span.clickable:hover{border-color:var(--link-color)}.fail-reason-carousel-controls{float:right;color:var(--text-color);font-weight:400}pre{white-space:pre-wrap}.font-bold{font-weight:700}table.results-table,.divTable{border-collapse:collapse;width:100%;margin:1rem 0;background-color:#fff}[data-theme=dark] table.results-table,[data-theme=dark] .divTable{background-color:#1a1a1a}.results-table ins{background-color:var(--insert-highlight-color);text-decoration:none}.results-table del{background-color:var(--delete-highlight-color);text-decoration:strikethrough}.tr{display:flex}.results-table tr,.tr{width:fit-content}.results-table tr:hover,.tr:hover{background-color:#00000005}.results-table th,.th,.results-table td,.td{position:relative;border:1px solid var(--border-color);vertical-align:top}.compact.results-table th,.compact .th,.compact.results-table td,.compact .td{padding:.5rem}.results-table th.variable,.th.variable,.results-table td.variable,.td.variable{background-color:var(--variable-background-color)}.variable .cell{max-height:100%;overflow-y:auto}.results-table tr.header{background-color:var(--header-background-color)}.results-table th,.th{padding:1rem;position:relative;vertical-align:bottom;text-align:left;white-space:pre-wrap;font-weight:400}.results-table th .action{cursor:pointer;margin-left:.5rem}.results-table th .action svg{vertical-align:middle}.results-table td,.results-table th{height:1px}@-moz-document url-prefix(){.results-table td,.results-table th{height:100%}}.firefox-fix{display:table;height:100%}.results-table tr .cell{display:flex;flex-direction:column;white-space:pre-wrap;height:100%;padding:1rem}.results-table tr .cell img{max-width:256px;max-height:256px;cursor:zoom-in}.results-table tr .cell .prompt{background-color:var(--variable-background-color);border:1px solid var(--border-color);padding:1rem;border-radius:4px;margin-bottom:.5rem;font-family:Courier New,Courier,monospace;font-size:.8rem}.results-table tr .cell .prompt .pill{display:block;background-color:var(--neutral-background-color);border:1px solid var(--border-color);border-radius:4px;padding:.25rem;width:fit-content;margin-bottom:.5rem}.results-table tr .cell-actions{display:flex;gap:.5rem;visibility:hidden;position:absolute;top:1.25rem;right:.75rem;line-height:0;font-size:1.25rem}.results-table tr:hover .cell-actions{visibility:visible}.results-table tr .cell-detail{display:flex;flex-wrap:wrap;column-gap:.5rem;row-gap:.25rem;padding-top:1rem;margin-top:auto}.results-table tr .stat-item{font-weight:400;font-size:.75rem;color:#888}.results-table tr .cell-actions .action{cursor:pointer}.results-table tr .cell table{width:100%;border-collapse:collapse;margin:1rem 0}.results-table tr .cell table th,.results-table tr .cell table td{border:1px solid var(--border-color);padding:.5rem;text-align:left}.results-table tr .cell table th{background-color:var(--header-background-color);font-weight:700}.results-table tr .cell table tr:nth-child(2n){background-color:var(--row-background-color)}.results-table tr .cell table tr:hover{background-color:var(--hover-background-color)}.results-table th .output-header{display:flex;flex-direction:column;height:100%;align-items:flex-start}.results-table th .output-header .pills{display:flex;flex-wrap:wrap;gap:.25rem;align-items:center;margin-bottom:.5rem}.results-table th .prompt-container{font-weight:700;margin-bottom:.5rem}.results-table th .prompt-container>*{display:inline}.results-table th .provider{display:inline-block;padding:.25rem .5rem;background-color:var(--neutral-background-color);border:1px solid var(--border-color);border-radius:4px;margin-right:.25rem}.results-table th .summary{font-weight:400}.results-table th .prompt-detail{display:flex;flex-wrap:wrap;column-gap:.5rem;row-gap:.25rem;font-size:.75rem;color:#888;margin-top:auto}.results-table th .summary .highlight{padding:.25rem .5rem;border-radius:4px;background-color:var(--fail-background-color);border:1px solid var(--border-color)}.results-table th .summary .highlight.success{background-color:var(--success-background-color);border:1px solid var(--pass-color)}.results-table .status{display:flex;flex-direction:column;gap:.25rem;font-weight:700;margin-bottom:.5rem}.results-table .status .pill{display:inline-block;padding:.25rem .5rem;border-radius:4px;margin-right:.25rem;align-self:flex-start}.results-table .pass .pill{background-color:var(--success-background-color);color:var(--pass-color);border:1px solid var(--pass-color)}.results-table .fail .pill{border:1px solid var(--fail-color)}.results-table .fail{color:var(--fail-color)}.fail-reason{color:var(--fail-color);font-weight:700}.compact .fail-reason{display:inline}.results-table .fail .pill{background-color:var(--fail-background-color)}.results-table td .score{font-weight:400}.results-table .comment{margin-top:.5rem;padding:.25rem .5rem;border-radius:4px;background-color:var(--neutral-background-color);font-style:italic;cursor:pointer}.results-table td.first-prompt-col{border-left:2px solid #888}.results-table td.first-prompt-row{border-top:2px solid #888}.search-highlight{color:var(--search-text-color);background-color:var(--search-highlight-color)}.results-table tr .cell .lightbox{position:fixed;top:0;left:0;right:0;bottom:0;background-color:#000000b3;display:flex;align-items:center;justify-content:center;z-index:1000}.results-table tr .cell .lightbox img{max-width:90%;max-height:90%;cursor:zoom-out}.resizer{position:absolute;right:0;top:0;height:100%;width:5px;cursor:col-resize;-webkit-user-select:none;user-select:none;touch-action:none;background:var(--text-color);opacity:.5}.resizer.isResizing{background:var(--text-color);opacity:1}@media (hover: hover){.resizer{opacity:0}*:hover>.resizer{opacity:1}}.description{cursor:pointer;transition:background-color .3s ease}.description:hover{background-color:#f0f0f0}[data-theme=dark] .description:hover{background-color:#2a2a2a}.eval-header{display:flex;align-items:center;flex-grow:1}.eval-header strong{font-weight:500}body{background-color:var(--background-color);color:var(--text-color)}.notice{display:flex;flex-direction:column;gap:1.5rem;justify-content:center;align-items:center;height:9rem}.framework-compliance-card{transition:box-shadow .3s ease-in-out;border-radius:12px;overflow:hidden;background-color:#f5f5f5}.framework-compliance-card:hover{box-shadow:0 8px 16px #0000001a}.compliance-summary{text-align:center;margin-bottom:32px;padding:24px;background-color:#fff;border-radius:8px;box-shadow:0 2px 4px #0000000d}.framework-grid{margin-top:24px}.framework-item{height:100%;transition:transform .3s ease,box-shadow .3s ease}.framework-item:hover{box-shadow:0 4px 8px #0000001a}.framework-item.compliant{background-color:#e8f5e9}.framework-item.non-compliant{background-color:#ffebee}.icon-compliant{color:#4caf50;font-size:24px}.icon-non-compliant{color:#f44336;font-size:24px}[data-theme=dark] .framework-compliance-card{background-color:#1e1e1e}[data-theme=dark] .compliance-summary{background-color:#2a2a2a}[data-theme=dark] .framework-item.compliant{background-color:#4caf501a}[data-theme=dark] .framework-item.non-compliant{background-color:#f443361a}[data-theme=dark] .framework-compliance-card:hover,[data-theme=dark] .framework-item:hover{box-shadow:0 8px 16px #ffffff1a}.framework-item .MuiList-root,.framework-item .MuiListItem-root{padding-top:0;padding-bottom:0}.framework-item .MuiListItemIcon-root{min-width:24px}.framework-item .MuiListItemText-root{margin-top:0;margin-bottom:0}.severity-card{transition:transform .3s ease-in-out,box-shadow .3s ease-in-out}.severity-card:hover{transform:translateY(-5px);box-shadow:0 8px 16px #0000001a}[data-theme=dark] .severity-card:hover{box-shadow:none}.card-critical{border-left:5px solid #ff1744}.card-high{border-left:5px solid #ff9100}.card-medium{border-left:5px solid #ffc400}.card-low{border-left:5px solid #00e676}.risk-category-drawer .MuiListItem-root.failure-item{border:1px solid #e0e0e0;border-radius:8px;margin-bottom:16px;padding:16px;transition:all .3s ease;box-shadow:0 2px 4px #0000000d}.risk-category-drawer .MuiListItem-root.failure-item:hover{border-color:#bdbdbd;box-shadow:0 4px 8px #0000001a}.risk-category-drawer .prompt{font-weight:500;margin-bottom:8px}.risk-category-drawer .output{color:#555;white-space:pre-wrap;word-break:break-word;max-height:100px;overflow-y:auto;background-color:#f5f5f5;padding:8px;border-radius:4px}.risk-category-drawer .failed-tests-header{margin-top:8px;margin-bottom:8px;font-weight:500;border-top:2px solid #e0e0e0;padding-top:1.5rem}[data-theme=dark] .risk-category-drawer .MuiListItem-root.failure-item{border-color:#424242;background-color:#2c2c2c;box-shadow:0 2px 4px #0003}[data-theme=dark] .risk-category-drawer .MuiListItem-root.failure-item:hover{border-color:#616161;box-shadow:0 4px 8px #0000004d}[data-theme=dark] .risk-category-drawer .prompt{color:#e0e0e0}[data-theme=dark] .risk-category-drawer .output{color:#b0b0b0;background-color:#1e1e1e}[data-theme=dark] .risk-category-drawer .failed-tests-header{color:#e0e0e0;border-top-color:#424242}.risk-card-container{padding-top:3rem;padding-bottom:3rem}.risk-card-title{font-weight:700}.risk-card-progress{margin-bottom:1rem}.risk-card-issues{color:red}.risk-card-list-item{padding:0;transition:background-color .3s,box-shadow .3s}.risk-card-list-item:hover{background-color:#f0f8ff;box-shadow:0 0 0 1px #add8e6;border-radius:4px}[data-theme=dark] .risk-card-list-item:hover{background-color:#2a2a2a;box-shadow:0 0 0 1px #555;border-radius:4px}.risk-card-icon-passed{color:green;font-size:16px}.risk-card-icon-failed{color:red;font-size:16px}.risk-card-icon-no-tests{color:#888;font-size:16px}.risk-card-percentage{margin-left:8px;font-weight:700}.risk-card-percentage-high{color:#4caf50}.risk-card-percentage-medium{color:#ff9800}.risk-card-percentage-low{color:#f44336}[data-theme=dark] .risk-card-percentage-high{color:#81c784}[data-theme=dark] .risk-card-percentage-medium{color:#ffb74d}[data-theme=dark] .risk-card-percentage-low{color:#e57373}.strategy-stats-card{border-radius:.75rem}div.strategy-stats-content{padding:2rem}.strategy-grid{display:grid;grid-template-columns:repeat(2,1fr);gap:1.5rem}.strategy-item{border:1px solid #eee;border-radius:.5rem;padding:1rem;height:100%}[data-theme=dark] .strategy-item{border:1px solid #424242}.strategy-name{font-weight:500;margin-bottom:.25rem}.strategy-description{margin-bottom:.75rem}.progress-container{margin-bottom:.5rem}.fail-rate{text-align:right}.attack-stats{display:block;text-align:right}.pass-high{color:green!important}.pass-medium{color:#cb8503!important}.pass-low{color:red!important}.vuln-critical{color:red!important;font-weight:700!important}.vuln-high{color:red!important}.vuln-medium{color:#cb8503!important}.vuln-low{color:green!important}.report-header{padding:24px;border-radius:12px;box-shadow:0 4px 6px #0000001a}[data-theme=dark] .report-header{box-shadow:none}.report-details{display:flex;flex-wrap:wrap;gap:12px}.report-details .MuiChip-root{font-size:.875rem}.page-content{padding:0;margin-top:-15px}:root{font-family:Inter,system-ui,Avenir,Helvetica,Arial,sans-serif;font-style:normal;font-weight-normal:400;font-weight-bold:700;font-synthesis:none;text-rendering:optimizeLegibility;-webkit-font-smoothing:antialiased;-moz-osx-font-smoothing:grayscale;-webkit-text-size-adjust:100%;--background-color: #f5f5f5;--text-color: #404040;--link-color: #2e6585;--border-color: lightgray;--table-border-color: lightgray;--pass-color: green;--fail-color: #ad0000;--success-background-color: #d1ffd7;--fail-background-color: #ffd1d1;--neutral-background-color: #eee;--variable-background-color: #f7f7f7;--header-background-color: #fffdf7;--insert-highlight-color: #d4fcbc;--delete-highlight-color: #fbb6c2;--prompt-highlight-color: linen;--textarea-background-color: #ffffff;--textarea-color: #404040;--search-highlight-color: #ffff00;--search-text-color: inherit}[data-theme=dark]{--background-color: #1a1a1a;--text-color: #f0f0f0;--link-color: #6fcaff;--border-color: #444444;--table-border-color: #444444;--pass-color: #b1e9b3;--fail-color: #ee726a;--success-background-color: #216d2b;--fail-background-color: #6d2121;--neutral-background-color: #424242;--variable-background-color: #333;--header-background-color: #333;--insert-highlight-color: #4f8a34;--delete-highlight-color: #8a3434;--prompt-highlight-color: #67605a;--textarea-background-color: #2d2d2d;--textarea-color: #fff;--search-highlight-color: #ffff00;--search-text-color: #404040}html{font-size:16px;background-color:var(--background-color);color:var(--text-color)}body{margin:0}a{color:var(--link-color);text-decoration:none}a:hover{text-decoration:underline}textarea{background-color:var(--textarea-background-color);color:var(--textarea-color);padding:.5em;border:1px solid var(--border-color);border-radius:.25em;resize:vertical}*{box-sizing:border-box}