promptfoo 0.47.0 → 0.48.0

package/README.md

@@ -340,6 +340,8 @@ Here are some of the available scripts:
 - `build:watch`: Continuously watch and transpile TypeScript files on changes
 - `test`: Run test suite
 - `test:watch`: Continuously run test suite on changes
+- `db:generate`: Generate new db migrations (and create the db if it doesn't already exist). Note that after generating a new migration, you'll have to `npm i` to copy the migrations into `dist/`.
+- `db:migrate`: Run existing db migrations (and create the db if it doesn't already exist)
 
 # [» View full documentation «](https://promptfoo.dev/docs/intro)
 

package/dist/drizzle/0000_lush_hellion.sql

@@ -0,0 +1,36 @@
+CREATE TABLE `datasets` (
+	`id` text PRIMARY KEY NOT NULL,
+	`test_case_id` text NOT NULL,
+	`created_at` integer DEFAULT CURRENT_TIMESTAMP NOT NULL
+);
+--> statement-breakpoint
+CREATE TABLE `evals` (
+	`id` text PRIMARY KEY NOT NULL,
+	`created_at` integer DEFAULT CURRENT_TIMESTAMP NOT NULL,
+	`description` text,
+	`results` text NOT NULL,
+	`config` text NOT NULL
+);
+--> statement-breakpoint
+CREATE TABLE `evals_to_datasets` (
+	`eval_id` text NOT NULL,
+	`dataset_id` text NOT NULL,
+	PRIMARY KEY(`dataset_id`, `eval_id`),
+	FOREIGN KEY (`eval_id`) REFERENCES `evals`(`id`) ON UPDATE no action ON DELETE no action,
+	FOREIGN KEY (`dataset_id`) REFERENCES `datasets`(`id`) ON UPDATE no action ON DELETE no action
+);
+--> statement-breakpoint
+CREATE TABLE `evals_to_prompts` (
+	`eval_id` text NOT NULL,
+	`prompt_id` text NOT NULL,
+	PRIMARY KEY(`eval_id`, `prompt_id`),
+	FOREIGN KEY (`eval_id`) REFERENCES `evals`(`id`) ON UPDATE no action ON DELETE no action,
+	FOREIGN KEY (`prompt_id`) REFERENCES `prompts`(`id`) ON UPDATE no action ON DELETE no action
+);
+--> statement-breakpoint
+CREATE TABLE `prompts` (
+	`id` text PRIMARY KEY NOT NULL,
+	`created_at` integer DEFAULT CURRENT_TIMESTAMP NOT NULL,
+	`prompt` text NOT NULL,
+	`hash` text NOT NULL
+);

package/dist/drizzle/0001_wide_calypso.sql

@@ -0,0 +1,3 @@
+ALTER TABLE datasets ADD `tests` text;--> statement-breakpoint
+ALTER TABLE `datasets` DROP COLUMN `test_case_id`;--> statement-breakpoint
+ALTER TABLE `prompts` DROP COLUMN `hash`;

package/dist/drizzle/meta/0000_snapshot.json

@@ -0,0 +1,244 @@
+{
+  "version": "5",
+  "dialect": "sqlite",
+  "id": "8b53403f-5b6f-436a-862e-9fd17a52204e",
+  "prevId": "00000000-0000-0000-0000-000000000000",
+  "tables": {
+    "datasets": {
+      "name": "datasets",
+      "columns": {
+        "id": {
+          "name": "id",
+          "type": "text",
+          "primaryKey": true,
+          "notNull": true,
+          "autoincrement": false
+        },
+        "test_case_id": {
+          "name": "test_case_id",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true,
+          "autoincrement": false
+        },
+        "created_at": {
+          "name": "created_at",
+          "type": "integer",
+          "primaryKey": false,
+          "notNull": true,
+          "autoincrement": false,
+          "default": "CURRENT_TIMESTAMP"
+        }
+      },
+      "indexes": {},
+      "foreignKeys": {},
+      "compositePrimaryKeys": {},
+      "uniqueConstraints": {}
+    },
+    "evals": {
+      "name": "evals",
+      "columns": {
+        "id": {
+          "name": "id",
+          "type": "text",
+          "primaryKey": true,
+          "notNull": true,
+          "autoincrement": false
+        },
+        "created_at": {
+          "name": "created_at",
+          "type": "integer",
+          "primaryKey": false,
+          "notNull": true,
+          "autoincrement": false,
+          "default": "CURRENT_TIMESTAMP"
+        },
+        "description": {
+          "name": "description",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false,
+          "autoincrement": false
+        },
+        "results": {
+          "name": "results",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true,
+          "autoincrement": false
+        },
+        "config": {
+          "name": "config",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true,
+          "autoincrement": false
+        }
+      },
+      "indexes": {},
+      "foreignKeys": {},
+      "compositePrimaryKeys": {},
+      "uniqueConstraints": {}
+    },
+    "evals_to_datasets": {
+      "name": "evals_to_datasets",
+      "columns": {
+        "eval_id": {
+          "name": "eval_id",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true,
+          "autoincrement": false
+        },
+        "dataset_id": {
+          "name": "dataset_id",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true,
+          "autoincrement": false
+        }
+      },
+      "indexes": {},
+      "foreignKeys": {
+        "evals_to_datasets_eval_id_evals_id_fk": {
+          "name": "evals_to_datasets_eval_id_evals_id_fk",
+          "tableFrom": "evals_to_datasets",
+          "tableTo": "evals",
+          "columnsFrom": [
+            "eval_id"
+          ],
+          "columnsTo": [
+            "id"
+          ],
+          "onDelete": "no action",
+          "onUpdate": "no action"
+        },
+        "evals_to_datasets_dataset_id_datasets_id_fk": {
+          "name": "evals_to_datasets_dataset_id_datasets_id_fk",
+          "tableFrom": "evals_to_datasets",
+          "tableTo": "datasets",
+          "columnsFrom": [
+            "dataset_id"
+          ],
+          "columnsTo": [
+            "id"
+          ],
+          "onDelete": "no action",
+          "onUpdate": "no action"
+        }
+      },
+      "compositePrimaryKeys": {
+        "evals_to_datasets_eval_id_dataset_id_pk": {
+          "columns": [
+            "dataset_id",
+            "eval_id"
+          ],
+          "name": "evals_to_datasets_eval_id_dataset_id_pk"
+        }
+      },
+      "uniqueConstraints": {}
+    },
+    "evals_to_prompts": {
+      "name": "evals_to_prompts",
+      "columns": {
+        "eval_id": {
+          "name": "eval_id",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true,
+          "autoincrement": false
+        },
+        "prompt_id": {
+          "name": "prompt_id",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true,
+          "autoincrement": false
+        }
+      },
+      "indexes": {},
+      "foreignKeys": {
+        "evals_to_prompts_eval_id_evals_id_fk": {
+          "name": "evals_to_prompts_eval_id_evals_id_fk",
+          "tableFrom": "evals_to_prompts",
+          "tableTo": "evals",
+          "columnsFrom": [
+            "eval_id"
+          ],
+          "columnsTo": [
+            "id"
+          ],
+          "onDelete": "no action",
+          "onUpdate": "no action"
+        },
+        "evals_to_prompts_prompt_id_prompts_id_fk": {
+          "name": "evals_to_prompts_prompt_id_prompts_id_fk",
+          "tableFrom": "evals_to_prompts",
+          "tableTo": "prompts",
+          "columnsFrom": [
+            "prompt_id"
+          ],
+          "columnsTo": [
+            "id"
+          ],
+          "onDelete": "no action",
+          "onUpdate": "no action"
+        }
+      },
+      "compositePrimaryKeys": {
+        "evals_to_prompts_eval_id_prompt_id_pk": {
+          "columns": [
+            "eval_id",
+            "prompt_id"
+          ],
+          "name": "evals_to_prompts_eval_id_prompt_id_pk"
+        }
+      },
+      "uniqueConstraints": {}
+    },
+    "prompts": {
+      "name": "prompts",
+      "columns": {
+        "id": {
+          "name": "id",
+          "type": "text",
+          "primaryKey": true,
+          "notNull": true,
+          "autoincrement": false
+        },
+        "created_at": {
+          "name": "created_at",
+          "type": "integer",
+          "primaryKey": false,
+          "notNull": true,
+          "autoincrement": false,
+          "default": "CURRENT_TIMESTAMP"
+        },
+        "prompt": {
+          "name": "prompt",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true,
+          "autoincrement": false
+        },
+        "hash": {
+          "name": "hash",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true,
+          "autoincrement": false
+        }
+      },
+      "indexes": {},
+      "foreignKeys": {},
+      "compositePrimaryKeys": {},
+      "uniqueConstraints": {}
+    }
+  },
+  "enums": {},
+  "_meta": {
+    "schemas": {},
+    "tables": {},
+    "columns": {}
+  }
+}

package/dist/drizzle/meta/0001_snapshot.json

@@ -0,0 +1,237 @@
+{
+  "version": "5",
+  "dialect": "sqlite",
+  "id": "476c887f-edcb-4b29-b054-50a8383cc6c2",
+  "prevId": "8b53403f-5b6f-436a-862e-9fd17a52204e",
+  "tables": {
+    "datasets": {
+      "name": "datasets",
+      "columns": {
+        "id": {
+          "name": "id",
+          "type": "text",
+          "primaryKey": true,
+          "notNull": true,
+          "autoincrement": false
+        },
+        "tests": {
+          "name": "tests",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false,
+          "autoincrement": false
+        },
+        "created_at": {
+          "name": "created_at",
+          "type": "integer",
+          "primaryKey": false,
+          "notNull": true,
+          "autoincrement": false,
+          "default": "CURRENT_TIMESTAMP"
+        }
+      },
+      "indexes": {},
+      "foreignKeys": {},
+      "compositePrimaryKeys": {},
+      "uniqueConstraints": {}
+    },
+    "evals": {
+      "name": "evals",
+      "columns": {
+        "id": {
+          "name": "id",
+          "type": "text",
+          "primaryKey": true,
+          "notNull": true,
+          "autoincrement": false
+        },
+        "created_at": {
+          "name": "created_at",
+          "type": "integer",
+          "primaryKey": false,
+          "notNull": true,
+          "autoincrement": false,
+          "default": "CURRENT_TIMESTAMP"
+        },
+        "description": {
+          "name": "description",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false,
+          "autoincrement": false
+        },
+        "results": {
+          "name": "results",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true,
+          "autoincrement": false
+        },
+        "config": {
+          "name": "config",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true,
+          "autoincrement": false
+        }
+      },
+      "indexes": {},
+      "foreignKeys": {},
+      "compositePrimaryKeys": {},
+      "uniqueConstraints": {}
+    },
+    "evals_to_datasets": {
+      "name": "evals_to_datasets",
+      "columns": {
+        "eval_id": {
+          "name": "eval_id",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true,
+          "autoincrement": false
+        },
+        "dataset_id": {
+          "name": "dataset_id",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true,
+          "autoincrement": false
+        }
+      },
+      "indexes": {},
+      "foreignKeys": {
+        "evals_to_datasets_eval_id_evals_id_fk": {
+          "name": "evals_to_datasets_eval_id_evals_id_fk",
+          "tableFrom": "evals_to_datasets",
+          "tableTo": "evals",
+          "columnsFrom": [
+            "eval_id"
+          ],
+          "columnsTo": [
+            "id"
+          ],
+          "onDelete": "no action",
+          "onUpdate": "no action"
+        },
+        "evals_to_datasets_dataset_id_datasets_id_fk": {
+          "name": "evals_to_datasets_dataset_id_datasets_id_fk",
+          "tableFrom": "evals_to_datasets",
+          "tableTo": "datasets",
+          "columnsFrom": [
+            "dataset_id"
+          ],
+          "columnsTo": [
+            "id"
+          ],
+          "onDelete": "no action",
+          "onUpdate": "no action"
+        }
+      },
+      "compositePrimaryKeys": {
+        "evals_to_datasets_eval_id_dataset_id_pk": {
+          "columns": [
+            "dataset_id",
+            "eval_id"
+          ],
+          "name": "evals_to_datasets_eval_id_dataset_id_pk"
+        }
+      },
+      "uniqueConstraints": {}
+    },
+    "evals_to_prompts": {
+      "name": "evals_to_prompts",
+      "columns": {
+        "eval_id": {
+          "name": "eval_id",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true,
+          "autoincrement": false
+        },
+        "prompt_id": {
+          "name": "prompt_id",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true,
+          "autoincrement": false
+        }
+      },
+      "indexes": {},
+      "foreignKeys": {
+        "evals_to_prompts_eval_id_evals_id_fk": {
+          "name": "evals_to_prompts_eval_id_evals_id_fk",
+          "tableFrom": "evals_to_prompts",
+          "tableTo": "evals",
+          "columnsFrom": [
+            "eval_id"
+          ],
+          "columnsTo": [
+            "id"
+          ],
+          "onDelete": "no action",
+          "onUpdate": "no action"
+        },
+        "evals_to_prompts_prompt_id_prompts_id_fk": {
+          "name": "evals_to_prompts_prompt_id_prompts_id_fk",
+          "tableFrom": "evals_to_prompts",
+          "tableTo": "prompts",
+          "columnsFrom": [
+            "prompt_id"
+          ],
+          "columnsTo": [
+            "id"
+          ],
+          "onDelete": "no action",
+          "onUpdate": "no action"
+        }
+      },
+      "compositePrimaryKeys": {
+        "evals_to_prompts_eval_id_prompt_id_pk": {
+          "columns": [
+            "eval_id",
+            "prompt_id"
+          ],
+          "name": "evals_to_prompts_eval_id_prompt_id_pk"
+        }
+      },
+      "uniqueConstraints": {}
+    },
+    "prompts": {
+      "name": "prompts",
+      "columns": {
+        "id": {
+          "name": "id",
+          "type": "text",
+          "primaryKey": true,
+          "notNull": true,
+          "autoincrement": false
+        },
+        "created_at": {
+          "name": "created_at",
+          "type": "integer",
+          "primaryKey": false,
+          "notNull": true,
+          "autoincrement": false,
+          "default": "CURRENT_TIMESTAMP"
+        },
+        "prompt": {
+          "name": "prompt",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true,
+          "autoincrement": false
+        }
+      },
+      "indexes": {},
+      "foreignKeys": {},
+      "compositePrimaryKeys": {},
+      "uniqueConstraints": {}
+    }
+  },
+  "enums": {},
+  "_meta": {
+    "schemas": {},
+    "tables": {},
+    "columns": {}
+  }
+}

package/dist/drizzle/meta/_journal.json

@@ -0,0 +1,20 @@
+{
+  "version": "5",
+  "dialect": "sqlite",
+  "entries": [
+    {
+      "idx": 0,
+      "version": "5",
+      "when": 1710348564252,
+      "tag": "0000_lush_hellion",
+      "breakpoints": true
+    },
+    {
+      "idx": 1,
+      "version": "5",
+      "when": 1710706957023,
+      "tag": "0001_wide_calypso",
+      "breakpoints": true
+    }
+  ]
+}

package/dist/package.json

@@ -2,7 +2,7 @@
     "name": "promptfoo",
     "description": "LLM eval & testing toolkit",
     "author": "Ian Webster",
-    "version": "0.47.0",
+    "version": "0.48.0",
     "license": "MIT",
     "type": "commonjs",
     "repository": "promptfoo/promptfoo",
@@ -32,11 +32,14 @@
         "build:clean": "rm -rf dist",
         "build:nextui": "cd src/web/nextui && npm run build && { [ -z \"$NEXT_PUBLIC_PROMPTFOO_BUILD_STANDALONE_SERVER\" ] && cp -r out/ ../../../dist/src/web/nextui; } || true",
         "build:watch": "tsc --watch",
-        "build": "tsc && cp src/*.html dist/src && cp src/python/wrapper.py dist/src/python && npm run build:nextui && chmod +x dist/src/main.js",
+        "build": "tsc && cp src/*.html dist/src && cp src/python/wrapper.py dist/src/python && cp -r drizzle/ dist/ && npm run build:nextui && chmod +x dist/src/main.js",
         "prepare": "npm run install:nextui && npm run build:clean && npm run build",
         "test": "jest",
         "test:watch": "jest --watch",
-        "format": "prettier -w ."
+        "format": "prettier -w .",
+        "db:migrate": "npx tsx src/migrate.ts",
+        "db:generate": "npx drizzle-kit generate:sqlite",
+        "audit:fix": "npm audit fix && cd src/web/nextui && npm audit fix && cd ../../site && npm audit fix"
     },
     "peerDependencies": {
         "@aws-sdk/client-bedrock-runtime": "^3.458.0",
@@ -46,6 +49,7 @@
         "@aws-sdk/client-bedrock-runtime": "^3.458.0",
         "@azure/identity": "^4.0.0",
         "@types/async": "^3.2.20",
+        "@types/better-sqlite3": "^7.6.8",
         "@types/cache-manager": "^4.0.2",
         "@types/cache-manager-fs-hash": "^0.0.1",
         "@types/cli-progress": "^3.11.0",
@@ -62,6 +66,7 @@
         "@types/semver": "^7.5.0",
         "@types/uuid": "^9.0.2",
         "babel-jest": "^29.5.0",
+        "drizzle-kit": "^0.20.13",
         "jest": "^29.5.0",
         "jest-watch-typeahead": "^2.2.2",
         "next": "^13.4.13",
@@ -76,6 +81,7 @@
         "ajv": "^8.12.0",
         "ajv-formats": "^2.1.1",
         "async": "^3.2.4",
+        "better-sqlite3": "^9.3.0",
         "cache-manager": "^4.1.0",
         "cache-manager-fs-hash": "^1.0.0",
         "chalk": "^4.1.2",
@@ -88,6 +94,7 @@
         "csv-parse": "^5.3.8",
         "csv-stringify": "^6.3.2",
         "debounce": "^1.2.1",
+        "drizzle-orm": "^0.29.3",
         "express": "^4.18.2",
         "fastest-levenshtein": "^1.0.16",
         "glob": "^10.2.6",

package/dist/src/__mocks__/database.d.ts

@@ -0,0 +1,5 @@
+/// <reference types="jest" />
+declare const mockDbInstance: {};
+declare const mockRelations: jest.Mock<any, any, any>;
+declare const mockSqliteTable: jest.Mock<any, any, any>;
+//# sourceMappingURL=database.d.ts.map

package/dist/src/__mocks__/database.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"database.d.ts","sourceRoot":"","sources":["../../../src/__mocks__/database.ts"],"names":[],"mappings":";AAAA,QAAA,MAAM,cAAc,IAInB,CAAC;AAEF,QAAA,MAAM,aAAa,0BAAY,CAAC;AAEhC,QAAA,MAAM,eAAe,0BAGnB,CAAC"}

package/dist/src/__mocks__/database.js

@@ -0,0 +1,27 @@
+"use strict";
+const mockDbInstance = {
+// Mock any method you use from the dbInstance
+// For example:
+// query: jest.fn().mockResolvedValue({}),
+};
+const mockRelations = jest.fn();
+const mockSqliteTable = jest.fn().mockImplementation((tableName, schema) => {
+    // You can customize this mock based on your testing needs
+    return { tableName, schema };
+});
+module.exports = {
+    prompts: mockSqliteTable('prompts', { /* schema definition */}),
+    promptsRelations: mockRelations,
+    datasets: mockSqliteTable('datasets', { /* schema definition */}),
+    datasetsRelations: mockRelations,
+    evals: mockSqliteTable('evals', { /* schema definition */}),
+    evalsRelations: mockRelations,
+    evalsToPrompts: mockSqliteTable('evals_to_prompts', { /* schema definition */}),
+    evalsToPromptsRelations: mockRelations,
+    evalsToDatasets: mockSqliteTable('evals_to_datasets', { /* schema definition */}),
+    evalsToDatasetsRelations: mockRelations,
+    llmOutputs: mockSqliteTable('llm_outputs', { /* schema definition */}),
+    llmOutputsRelations: mockRelations,
+    getDb: jest.fn(() => mockDbInstance),
+};
+//# sourceMappingURL=database.js.map

package/dist/src/__mocks__/database.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"database.js","sourceRoot":"","sources":["../../../src/__mocks__/database.ts"],"names":[],"mappings":";AAAA,MAAM,cAAc,GAAG;AACrB,8CAA8C;AAC9C,eAAe;AACf,0CAA0C;CAC3C,CAAC;AAEF,MAAM,aAAa,GAAG,IAAI,CAAC,EAAE,EAAE,CAAC;AAEhC,MAAM,eAAe,GAAG,IAAI,CAAC,EAAE,EAAE,CAAC,kBAAkB,CAAC,CAAC,SAAS,EAAE,MAAM,EAAE,EAAE;IACzE,0DAA0D;IAC1D,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC;AAC/B,CAAC,CAAC,CAAC;AAEH,MAAM,CAAC,OAAO,GAAG;IACf,OAAO,EAAE,eAAe,CAAC,SAAS,EAAE,EAAC,uBAAuB,CAAC,CAAC;IAC9D,gBAAgB,EAAE,aAAa;IAC/B,QAAQ,EAAE,eAAe,CAAC,UAAU,EAAE,EAAC,uBAAuB,CAAC,CAAC;IAChE,iBAAiB,EAAE,aAAa;IAChC,KAAK,EAAE,eAAe,CAAC,OAAO,EAAE,EAAC,uBAAuB,CAAC,CAAC;IAC1D,cAAc,EAAE,aAAa;IAC7B,cAAc,EAAE,eAAe,CAAC,kBAAkB,EAAE,EAAC,uBAAuB,CAAC,CAAC;IAC9E,uBAAuB,EAAE,aAAa;IACtC,eAAe,EAAE,eAAe,CAAC,mBAAmB,EAAE,EAAC,uBAAuB,CAAC,CAAC;IAChF,wBAAwB,EAAE,aAAa;IACvC,UAAU,EAAE,eAAe,CAAC,aAAa,EAAE,EAAC,uBAAuB,CAAC,CAAC;IACrE,mBAAmB,EAAE,aAAa;IAClC,KAAK,EAAE,IAAI,CAAC,EAAE,CAAC,GAAG,EAAE,CAAC,cAAc,CAAC;CACrC,CAAC"}

package/dist/src/commands/list.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"list.d.ts","sourceRoot":"","sources":["../../../src/commands/list.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAOpC,wBAAgB,WAAW,CAAC,OAAO,EAAE,OAAO,QA6F3C"}
+{"version":3,"file":"list.d.ts","sourceRoot":"","sources":["../../../src/commands/list.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAOpC,wBAAgB,WAAW,CAAC,OAAO,EAAE,OAAO,QA4F3C"}