promptfoo 0.98.0 → 0.99.1

package/package.json

@@ -2,7 +2,7 @@
   "name": "promptfoo",
   "description": "LLM eval & testing toolkit",
   "author": "Ian Webster",
-  "version": "0.98.0",
+  "version": "0.99.1",
   "license": "MIT",
   "type": "commonjs",
   "repository": {
@@ -44,6 +44,7 @@
     "build:app": "npm run build --prefix src/app",
     "build:watch": "tsc --watch",
     "build": "tsc && shx cp src/*.html dist/src && shx cp src/python/wrapper.py dist/src/python && shx mkdir -p dist/src/golang && shx cp src/golang/wrapper.go dist/src/golang && shx rm -rf dist/drizzle && shx cp -r drizzle dist/drizzle && npm run build:app && shx chmod +x dist/src/main.js",
+    "citation:generate": "ts-node scripts/generateCitation.ts",
     "test": "jest",
     "test:integration": "jest --config jest.integration.config.ts integration",
     "test:watch": "jest --watch",
@@ -51,8 +52,8 @@
     "format": "prettier -w .",
     "format:check": "prettier --check .",
     "db:migrate": "npx tsx src/migrate.ts",
-    "citation:generate": "ts-node scripts/generateCitation.ts",
     "db:generate": "npx drizzle-kit generate",
+    "db:studio": "npx drizzle-kit studio",
     "jsonSchema:generate": "npx -y tsx scripts/generateJsonSchema.ts > site/static/config-schema.json | npx prettier --write site/static/config-schema.json",
     "audit:fix": "npm audit fix && npm audit fix --prefix src/app && npm audit fix --prefix site",
     "prepublishOnly": "npm run build:clean && npm run build",
@@ -77,18 +78,19 @@
     "ibm-cloud-sdk-core": "^5.0.2",
     "langfuse": "^3.7.0",
     "node-sql-parser": "^5.2.0",
+    "pdf-parse": "^1.1.1",
     "playwright": "^1.47.2",
     "playwright-extra": "^4.3.6",
     "puppeteer-extra-plugin-stealth": "^2.11.2"
   },
   "devDependencies": {
-    "@aws-sdk/client-bedrock-runtime": "^3.696.0",
-    "@aws-sdk/credential-provider-sso": "^3.696.0",
+    "@aws-sdk/client-bedrock-runtime": "^3.699.0",
+    "@aws-sdk/credential-provider-sso": "^3.699.0",
     "@azure/identity": "^4.5.0",
     "@eslint/js": "^9.15.0",
     "@libsql/client": "^0.14.0",
     "@swc/cli": "^0.5.1",
-    "@swc/core": "^1.9.2",
+    "@swc/core": "^1.9.3",
     "@swc/jest": "^0.2.37",
     "@trivago/prettier-plugin-sort-imports": "^4.3.0",
     "@types/async": "^3.2.24",
@@ -107,6 +109,7 @@
     "@types/js-yaml": "^4.0.9",
     "@types/nunjucks": "^3.2.6",
     "@types/opener": "^1.4.3",
+    "@types/pdf-parse": "^1.1.4",
     "@types/semver": "^7.5.8",
     "@types/supertest": "^6.0.2",
     "@types/uuid": "^10.0.0",
@@ -126,13 +129,13 @@
     "madge": "^8.0.0",
     "nock": "^13.5.6",
     "nodemon": "^3.1.7",
-    "prettier": "^3.3.3",
+    "prettier": "^3.4.1",
     "shx": "^0.3.4",
     "supertest": "^7.0.0",
     "ts-node": "^10.9.2",
     "tsconfig-paths": "^4.2.0",
-    "typescript": "^5.6.3",
-    "typescript-eslint": "^8.15.0",
+    "typescript": "^5.7.2",
+    "typescript-eslint": "^8.16.0",
     "zod-to-json-schema": "^3.23.5"
   },
   "dependencies": {
@@ -152,7 +155,7 @@
     "@emotion/react": "^11.13.5",
     "@emotion/styled": "^11.13.5",
     "@googleapis/sheets": "^9.3.1",
-    "@mui/material": "^6.1.8",
+    "@mui/material": "^6.1.9",
     "ajv": "^8.17.1",
     "ajv-formats": "^3.0.1",
     "async": "^3.2.6",
@@ -166,9 +169,9 @@
     "commander": "^12.1.0",
     "compression": "^1.7.5",
     "cors": "^2.8.5",
-    "csv-parse": "^5.5.6",
-    "csv-stringify": "^6.5.1",
-    "debounce": "^1.2.1",
+    "csv-parse": "^5.6.0",
+    "csv-stringify": "^6.5.2",
+    "debounce": "^2.2.0",
     "dedent": "^1.5.3",
     "dotenv": "^16.4.5",
     "drizzle-orm": "^0.35.2",
@@ -180,12 +183,12 @@
     "groq-sdk": "^0.8.0",
     "http-z": "^7.1.1",
     "inquirer": "^11.1.0",
-    "js-rouge": "3.0.0",
+    "js-rouge": "^3.0.0",
     "js-yaml": "^4.1.0",
     "mathjs": "^13.2.2",
     "node-cache": "^5.1.2",
     "nunjucks": "^3.2.4",
-    "openai": "^4.73.0",
+    "openai": "^4.73.1",
     "opener": "^1.5.2",
     "proxy-agent": "^6.4.0",
     "python-shell": "^5.0.0",

package/dist/src/app/assets/index-DatcZSwS.css

@@ -1 +0,0 @@
-.nav{padding:.25rem 0 .25rem 1rem;gap:1rem;background-color:#333;margin-bottom:1rem;box-shadow:0 4px 6px #0000001a}[data-theme=dark] .nav{background-color:#121212}.nav a{text-decoration:none;align-self:center}@media (max-width: 760px){.nav{font-size:.75rem}}.nav .active{font-weight:700}.nav a{color:#f0f0f0}.nav .right-aligned{display:flex;align-items:center;gap:1rem;margin-left:auto;margin-right:.5rem}.nav a:hover{text-decoration:underline}.nav a:hover{color:#ddd}.prompt-var-highlight{background-color:var(--prompt-highlight-color);padding:.25rem;border-radius:4px}.glowing-border{border:1px solid #5cb3ff;box-shadow:0 0 8px 2px #5cb3ff}code[class*=language-],pre[class*=language-]{color:#000;background:none;text-shadow:0 1px white;font-family:Consolas,Monaco,Andale Mono,Ubuntu Mono,monospace;font-size:1em;text-align:left;white-space:pre;word-spacing:normal;word-break:normal;word-wrap:normal;line-height:1.5;-moz-tab-size:4;-o-tab-size:4;tab-size:4;-webkit-hyphens:none;-moz-hyphens:none;-ms-hyphens:none;hyphens:none}pre[class*=language-]::-moz-selection,pre[class*=language-] ::-moz-selection,code[class*=language-]::-moz-selection,code[class*=language-] ::-moz-selection{text-shadow:none;background:#b3d4fc}pre[class*=language-]::selection,pre[class*=language-] ::selection,code[class*=language-]::selection,code[class*=language-] ::selection{text-shadow:none;background:#b3d4fc}@media print{code[class*=language-],pre[class*=language-]{text-shadow:none}}pre[class*=language-]{padding:1em;margin:.5em 0;overflow:auto}:not(pre)>code[class*=language-],pre[class*=language-]{background:#f5f2f0}:not(pre)>code[class*=language-]{padding:.1em;border-radius:.3em;white-space:normal}.token.comment,.token.prolog,.token.doctype,.token.cdata{color:#708090}.token.punctuation{color:#999}.token.namespace{opacity:.7}.token.property,.token.tag,.token.boolean,.token.number,.token.constant,.token.symbol,.token.deleted{color:#905}.token.selector,.token.attr-name,.token.string,.token.char,.token.builtin,.token.inserted{color:#690}.token.operator,.token.entity,.token.url,.language-css .token.string,.style .token.string{color:#9a6e3a;background:#ffffff80}.token.atrule,.token.attr-value,.token.keyword{color:#07a}.token.function,.token.class-name{color:#dd4a68}.token.regex,.token.important,.token.variable{color:#e90}.token.important,.token.bold{font-weight:700}.token.italic{font-style:italic}.token.entity{cursor:help}.yaml-config{font-size:10px}.custom-metric-container{display:inline-flex;flex-wrap:wrap;gap:.25rem;font-weight:400}.custom-metric-container>span{padding:.25rem .5rem;border-radius:4px;color:var(--text-color);border:1px solid var(--border-color);margin-right:.25rem}.custom-metric-container>span.clickable{cursor:pointer}.custom-metric-container>span.clickable:hover{border-color:var(--link-color)}.fail-reason-carousel-controls{float:right;color:var(--text-color);font-weight:400}pre{white-space:pre-wrap}.font-bold{font-weight:700}table.results-table,.divTable{border-collapse:collapse;width:100%;margin:1rem 0;background-color:#fff}[data-theme=dark] table.results-table,[data-theme=dark] .divTable{background-color:#1a1a1a}.results-table ins{background-color:var(--insert-highlight-color);text-decoration:none}.results-table del{background-color:var(--delete-highlight-color);text-decoration:strikethrough}.tr{display:flex}.results-table tr,.tr{width:fit-content}.results-table tr:hover,.tr:hover{background-color:#00000005}.results-table th,.th,.results-table td,.td{position:relative;border:1px solid var(--border-color);vertical-align:top}.compact.results-table th,.compact .th,.compact.results-table td,.compact .td{padding:.5rem}.results-table th.variable,.th.variable,.results-table td.variable,.td.variable{background-color:var(--variable-background-color)}.variable .cell{max-height:100%;overflow-y:auto}.results-table tr.header{background-color:var(--header-background-color)}.results-table th,.th{padding:1rem;position:relative;vertical-align:bottom;text-align:left;white-space:pre-wrap;font-weight:400}.results-table th .action{cursor:pointer;margin-left:.5rem}.results-table th .action svg{vertical-align:middle}.results-table td,.results-table th{height:1px}@-moz-document url-prefix(){.results-table td,.results-table th{height:100%}}.firefox-fix{display:table;height:100%}.results-table tr .cell{display:flex;flex-direction:column;white-space:pre-wrap;height:100%;padding:1rem}.results-table tr .cell img{max-width:256px;max-height:256px;cursor:zoom-in}.results-table tr .cell .prompt{background-color:var(--variable-background-color);border:1px solid var(--border-color);padding:1rem;border-radius:4px;margin-bottom:.5rem;font-family:Courier New,Courier,monospace;font-size:.8rem}.results-table tr .cell .prompt .pill{display:block;background-color:var(--neutral-background-color);border:1px solid var(--border-color);border-radius:4px;padding:.25rem;width:fit-content;margin-bottom:.5rem}.results-table tr .cell-actions{display:flex;gap:.5rem;visibility:hidden;position:absolute;top:1.25rem;right:.75rem;line-height:0;font-size:1.25rem}.results-table tr:hover .cell-actions{visibility:visible}.results-table tr .cell-detail{display:flex;flex-wrap:wrap;column-gap:.5rem;row-gap:.25rem;padding-top:1rem;margin-top:auto}.results-table tr .stat-item{font-weight:400;font-size:.75rem;color:#888}.results-table tr .cell-actions .action{cursor:pointer}.results-table tr .cell table{width:100%;border-collapse:collapse;margin:1rem 0}.results-table tr .cell table th,.results-table tr .cell table td{border:1px solid var(--border-color);padding:.5rem;text-align:left}.results-table tr .cell table th{background-color:var(--header-background-color);font-weight:700}.results-table tr .cell table tr:nth-child(2n){background-color:var(--row-background-color)}.results-table tr .cell table tr:hover{background-color:var(--hover-background-color)}.results-table th .output-header{display:flex;flex-direction:column;height:100%;align-items:flex-start}.results-table th .output-header .pills{display:flex;flex-wrap:wrap;gap:.25rem;align-items:center;margin-bottom:.5rem}.results-table th .prompt-container{font-weight:700;margin-bottom:.5rem}.results-table th .prompt-container>*{display:inline}.results-table th .provider{display:inline-block;padding:.25rem .5rem;background-color:var(--neutral-background-color);border:1px solid var(--border-color);border-radius:4px;margin-right:.25rem}.results-table th .summary{font-weight:400}.results-table th .prompt-detail{display:flex;flex-wrap:wrap;column-gap:.5rem;row-gap:.25rem;font-size:.75rem;color:#888;margin-top:auto}.results-table th .summary .highlight{padding:.25rem .5rem;border-radius:4px;background-color:var(--fail-background-color);border:1px solid var(--border-color)}.results-table th .summary .highlight.success{background-color:var(--success-background-color);border:1px solid var(--pass-color)}.results-table .status{display:flex;flex-direction:column;gap:.25rem;font-weight:700;margin-bottom:.5rem}.results-table .status .pill{display:inline-block;padding:.25rem .5rem;border-radius:4px;margin-right:.25rem;align-self:flex-start}.results-table .pass .pill{background-color:var(--success-background-color);color:var(--pass-color);border:1px solid var(--pass-color)}.results-table .fail .pill{border:1px solid var(--fail-color)}.results-table .fail{color:var(--fail-color)}.fail-reason{color:var(--fail-color);font-weight:700}.compact .fail-reason{display:inline}.results-table .fail .pill{background-color:var(--fail-background-color)}.results-table td .score{font-weight:400}.results-table .comment{margin-top:.5rem;padding:.25rem .5rem;border-radius:4px;background-color:var(--neutral-background-color);font-style:italic;cursor:pointer}.results-table td.first-prompt-col{border-left:2px solid #888}.results-table td.first-prompt-row{border-top:2px solid #888}.search-highlight{color:var(--search-text-color);background-color:var(--search-highlight-color)}.results-table tr .cell .lightbox{position:fixed;top:0;left:0;right:0;bottom:0;background-color:#000000b3;display:flex;align-items:center;justify-content:center;z-index:1000}.results-table tr .cell .lightbox img{max-width:90%;max-height:90%;cursor:zoom-out}.resizer{position:absolute;right:0;top:0;height:100%;width:5px;cursor:col-resize;-webkit-user-select:none;user-select:none;touch-action:none;background:var(--text-color);opacity:.5}.resizer.isResizing{background:var(--text-color);opacity:1}@media (hover: hover){.resizer{opacity:0}*:hover>.resizer{opacity:1}}.results-table thead.sticky{position:sticky;top:0;z-index:100;transition:transform .2s ease-out}[data-theme=dark] .results-table thead.sticky{background:#1a1a1a}.header-dismiss{position:absolute;top:8px;right:8px;cursor:pointer;z-index:101;padding:4px;border-radius:4px;background:#eee;display:none}[data-theme=dark] .header-dismiss{background:#333}.header-dismiss:hover{background:#ddd}[data-theme=dark] .header-dismiss:hover{background:#444}thead.collapsed .header-dismiss{display:block}.results-table thead.collapsed .prompt-detail,.results-table thead.collapsed .custom-metric-container{display:none}.results-table thead.collapsed .MuiFormControlLabel-root{margin-top:-8px}.results-table{border-collapse:separate;border-spacing:0}.results-table thead.collapsed th{padding-top:4px;padding-bottom:4px}.results-table thead th{transition:padding .2s ease-out}.results-table thead .output-header{transition:max-height .2s ease-out}.results-table thead.collapsed{border-bottom:2px solid var(--border-color);box-shadow:0 2px 4px #0000001a}.results-table th{background:#fff;box-shadow:0 1px 2px #0000001a}[data-theme=dark] .results-table th{background:#1a1a1a}.lightbox{position:fixed;top:0;left:0;width:100%;height:100%;background:#000c;display:flex;justify-content:center;align-items:center;z-index:1000;cursor:pointer}.lightbox img{max-width:90%;max-height:90%;object-fit:contain}.description{cursor:pointer;transition:background-color .3s ease}.description:hover{background-color:#f0f0f0}[data-theme=dark] .description:hover{background-color:#2a2a2a}.eval-header{display:flex;align-items:center;flex-grow:1}.eval-header strong{font-weight:500}body{background-color:var(--background-color);color:var(--text-color)}.notice{display:flex;flex-direction:column;gap:1.5rem;justify-content:center;align-items:center;height:9rem}.framework-compliance-card{transition:box-shadow .3s ease-in-out;border-radius:12px;overflow:hidden;background-color:#f5f5f5}.framework-compliance-card:hover{box-shadow:0 8px 16px #0000001a}.compliance-summary{text-align:center;margin-bottom:32px;padding:24px;background-color:#fff;border-radius:8px;box-shadow:0 2px 4px #0000000d}.framework-grid{margin-top:24px}.framework-item{height:100%;transition:transform .3s ease,box-shadow .3s ease}.framework-item:hover{box-shadow:0 4px 8px #0000001a}.framework-item.compliant{background-color:#e8f5e9}.framework-item.non-compliant{background-color:#ffebee}.icon-compliant{color:#4caf50;font-size:24px}.icon-non-compliant{color:#f44336;font-size:24px}[data-theme=dark] .framework-compliance-card{background-color:#1e1e1e}[data-theme=dark] .compliance-summary{background-color:#2a2a2a}[data-theme=dark] .framework-item.compliant{background-color:#4caf501a}[data-theme=dark] .framework-item.non-compliant{background-color:#f443361a}[data-theme=dark] .framework-compliance-card:hover,[data-theme=dark] .framework-item:hover{box-shadow:0 8px 16px #ffffff1a}.framework-item .MuiList-root,.framework-item .MuiListItem-root{padding-top:0;padding-bottom:0}.framework-item .MuiListItemIcon-root{min-width:24px}.framework-item .MuiListItemText-root{margin-top:0;margin-bottom:0}.severity-card{transition:transform .3s ease-in-out,box-shadow .3s ease-in-out}.severity-card:hover{transform:translateY(-5px);box-shadow:0 8px 16px #0000001a}[data-theme=dark] .severity-card:hover{box-shadow:none}.card-critical{border-left:5px solid #ff1744}.card-high{border-left:5px solid #ff9100}.card-medium{border-left:5px solid #ffc400}.card-low{border-left:5px solid #00e676}.risk-category-drawer .MuiListItem-root.failure-item{border:1px solid #e0e0e0;border-radius:8px;margin-bottom:16px;padding:16px;transition:all .3s ease;box-shadow:0 2px 4px #0000000d}.risk-category-drawer .MuiListItem-root.failure-item:hover{border-color:#bdbdbd;box-shadow:0 4px 8px #0000001a}.risk-category-drawer .prompt{font-weight:500;margin-bottom:8px}.risk-category-drawer .output{color:#555;white-space:pre-wrap;word-break:break-word;max-height:100px;overflow-y:auto;background-color:#f5f5f5;padding:8px;border-radius:4px}.risk-category-drawer .failed-tests-header{margin-top:8px;margin-bottom:8px;font-weight:500;border-top:2px solid #e0e0e0;padding-top:1.5rem}[data-theme=dark] .risk-category-drawer .MuiListItem-root.failure-item{border-color:#424242;background-color:#2c2c2c;box-shadow:0 2px 4px #0003}[data-theme=dark] .risk-category-drawer .MuiListItem-root.failure-item:hover{border-color:#616161;box-shadow:0 4px 8px #0000004d}[data-theme=dark] .risk-category-drawer .prompt{color:#e0e0e0}[data-theme=dark] .risk-category-drawer .output{color:#b0b0b0;background-color:#1e1e1e}[data-theme=dark] .risk-category-drawer .failed-tests-header{color:#e0e0e0;border-top-color:#424242}.risk-card-container{padding-top:3rem;padding-bottom:3rem}.risk-card-title{font-weight:700}.risk-card-progress{margin-bottom:1rem}.risk-card-issues{color:red}.risk-card-list-item{padding:0;transition:background-color .3s,box-shadow .3s}.risk-card-list-item:hover{background-color:#f0f8ff;box-shadow:0 0 0 1px #add8e6;border-radius:4px}[data-theme=dark] .risk-card-list-item:hover{background-color:#2a2a2a;box-shadow:0 0 0 1px #555;border-radius:4px}.risk-card-icon-passed{color:green;font-size:16px}.risk-card-icon-failed{color:red;font-size:16px}.risk-card-icon-no-tests{color:#888;font-size:16px}.risk-card-percentage{margin-left:8px;font-weight:700}.risk-card-percentage-high{color:#4caf50}.risk-card-percentage-medium{color:#ff9800}.risk-card-percentage-low{color:#f44336}[data-theme=dark] .risk-card-percentage-high{color:#81c784}[data-theme=dark] .risk-card-percentage-medium{color:#ffb74d}[data-theme=dark] .risk-card-percentage-low{color:#e57373}.strategy-stats-card{border-radius:.75rem}div.strategy-stats-content{padding:2rem}.strategy-grid{display:grid;grid-template-columns:repeat(2,1fr);gap:1.5rem}.strategy-item{border:1px solid #eee;border-radius:.5rem;padding:1rem;height:100%}[data-theme=dark] .strategy-item{border:1px solid #424242}.strategy-name{font-weight:500;margin-bottom:.25rem}.strategy-description{margin-bottom:.75rem}.progress-container{margin-bottom:.5rem}.fail-rate{text-align:right}.attack-stats{display:block;text-align:right}.pass-high{color:green!important}.pass-medium{color:#cb8503!important}.pass-low{color:red!important}.vuln-critical{color:red!important;font-weight:700!important}.vuln-high{color:red!important}.vuln-medium{color:#cb8503!important}.vuln-low{color:green!important}.report-header{padding:24px;border-radius:12px;box-shadow:0 4px 6px #0000001a}[data-theme=dark] .report-header{box-shadow:none}.report-details{display:flex;flex-wrap:wrap;gap:12px}.report-details .MuiChip-root{font-size:.875rem}.page-content{padding:0;margin-top:-15px}:root{font-family:Inter,system-ui,Avenir,Helvetica,Arial,sans-serif;font-style:normal;font-weight-normal:400;font-weight-bold:700;font-synthesis:none;text-rendering:optimizeLegibility;-webkit-font-smoothing:antialiased;-moz-osx-font-smoothing:grayscale;-webkit-text-size-adjust:100%;--background-color: #f5f5f5;--text-color: #404040;--link-color: #2e6585;--border-color: lightgray;--table-border-color: lightgray;--pass-color: green;--fail-color: #ad0000;--success-background-color: #d1ffd7;--fail-background-color: #ffd1d1;--neutral-background-color: #eee;--variable-background-color: #f7f7f7;--header-background-color: #fffdf7;--insert-highlight-color: #d4fcbc;--delete-highlight-color: #fbb6c2;--prompt-highlight-color: linen;--textarea-background-color: #ffffff;--textarea-color: #404040;--search-highlight-color: #ffff00;--search-text-color: inherit}[data-theme=dark]{--background-color: #1a1a1a;--text-color: #f0f0f0;--link-color: #6fcaff;--border-color: #444444;--table-border-color: #444444;--pass-color: #b1e9b3;--fail-color: #ee726a;--success-background-color: #216d2b;--fail-background-color: #6d2121;--neutral-background-color: #424242;--variable-background-color: #333;--header-background-color: #333;--insert-highlight-color: #4f8a34;--delete-highlight-color: #8a3434;--prompt-highlight-color: #67605a;--textarea-background-color: #2d2d2d;--textarea-color: #fff;--search-highlight-color: #ffff00;--search-text-color: #404040}html{font-size:16px;background-color:var(--background-color);color:var(--text-color)}body{margin:0}a{color:var(--link-color);text-decoration:none}a:hover{text-decoration:underline}textarea{background-color:var(--textarea-background-color);color:var(--textarea-color);padding:.5em;border:1px solid var(--border-color);border-radius:.25em;resize:vertical}*{box-sizing:border-box}