promptfoo 0.47.0 → 0.49.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -0
- package/dist/drizzle/0000_lush_hellion.sql +36 -0
- package/dist/drizzle/0001_wide_calypso.sql +3 -0
- package/dist/drizzle/meta/0000_snapshot.json +244 -0
- package/dist/drizzle/meta/0001_snapshot.json +237 -0
- package/dist/drizzle/meta/_journal.json +20 -0
- package/dist/package.json +13 -4
- package/dist/src/__mocks__/database.d.ts +5 -0
- package/dist/src/__mocks__/database.d.ts.map +1 -0
- package/dist/src/__mocks__/database.js +27 -0
- package/dist/src/__mocks__/database.js.map +1 -0
- package/dist/src/assertions.d.ts.map +1 -1
- package/dist/src/assertions.js +16 -7
- package/dist/src/assertions.js.map +1 -1
- package/dist/src/cache.d.ts.map +1 -1
- package/dist/src/cache.js +13 -6
- package/dist/src/cache.js.map +1 -1
- package/dist/src/commands/list.d.ts.map +1 -1
- package/dist/src/commands/list.js +4 -5
- package/dist/src/commands/list.js.map +1 -1
- package/dist/src/commands/show.d.ts +1 -1
- package/dist/src/commands/show.d.ts.map +1 -1
- package/dist/src/commands/show.js +7 -7
- package/dist/src/commands/show.js.map +1 -1
- package/dist/src/csv.d.ts +1 -1
- package/dist/src/csv.d.ts.map +1 -1
- package/dist/src/csv.js +5 -0
- package/dist/src/csv.js.map +1 -1
- package/dist/src/database.d.ts +238 -0
- package/dist/src/database.d.ts.map +1 -0
- package/dist/src/database.js +141 -0
- package/dist/src/database.js.map +1 -0
- package/dist/src/evaluator.d.ts.map +1 -1
- package/dist/src/evaluator.js +22 -18
- package/dist/src/evaluator.js.map +1 -1
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +2 -1
- package/dist/src/index.js.map +1 -1
- package/dist/src/main.js +23 -10
- package/dist/src/main.js.map +1 -1
- package/dist/src/matchers.d.ts +9 -1
- package/dist/src/matchers.d.ts.map +1 -1
- package/dist/src/matchers.js +20 -3
- package/dist/src/matchers.js.map +1 -1
- package/dist/src/migrate.d.ts +5 -0
- package/dist/src/migrate.d.ts.map +1 -0
- package/dist/src/migrate.js +50 -0
- package/dist/src/migrate.js.map +1 -0
- package/dist/src/prompts.d.ts.map +1 -1
- package/dist/src/prompts.js +3 -0
- package/dist/src/prompts.js.map +1 -1
- package/dist/src/providers/anthropic.d.ts +10 -0
- package/dist/src/providers/anthropic.d.ts.map +1 -1
- package/dist/src/providers/anthropic.js +3 -2
- package/dist/src/providers/anthropic.js.map +1 -1
- package/dist/src/providers/azureopenai.d.ts +23 -0
- package/dist/src/providers/azureopenai.d.ts.map +1 -1
- package/dist/src/providers/azureopenai.js +141 -3
- package/dist/src/providers/azureopenai.js.map +1 -1
- package/dist/src/providers/bedrock.d.ts.map +1 -1
- package/dist/src/providers/bedrock.js +40 -12
- package/dist/src/providers/bedrock.js.map +1 -1
- package/dist/src/providers/huggingface.d.ts +18 -0
- package/dist/src/providers/huggingface.d.ts.map +1 -1
- package/dist/src/providers/huggingface.js +71 -1
- package/dist/src/providers/huggingface.js.map +1 -1
- package/dist/src/providers/mistral.d.ts +5 -0
- package/dist/src/providers/mistral.d.ts.map +1 -1
- package/dist/src/providers/mistral.js +23 -8
- package/dist/src/providers/mistral.js.map +1 -1
- package/dist/src/providers/openai.d.ts.map +1 -1
- package/dist/src/providers/openai.js +1 -4
- package/dist/src/providers/openai.js.map +1 -1
- package/dist/src/providers/shared.d.ts +1 -0
- package/dist/src/providers/shared.d.ts.map +1 -1
- package/dist/src/providers/shared.js +5 -1
- package/dist/src/providers/shared.js.map +1 -1
- package/dist/src/providers/vertex.d.ts +1 -1
- package/dist/src/providers/vertex.d.ts.map +1 -1
- package/dist/src/providers/vertex.js +120 -39
- package/dist/src/providers/vertex.js.map +1 -1
- package/dist/src/providers/vertexUtil.d.ts +20 -5
- package/dist/src/providers/vertexUtil.d.ts.map +1 -1
- package/dist/src/providers/vertexUtil.js.map +1 -1
- package/dist/src/providers.d.ts.map +1 -1
- package/dist/src/providers.js +13 -10
- package/dist/src/providers.js.map +1 -1
- package/dist/src/share.d.ts.map +1 -1
- package/dist/src/share.js +5 -2
- package/dist/src/share.js.map +1 -1
- package/dist/src/table.js +1 -1
- package/dist/src/table.js.map +1 -1
- package/dist/src/types.d.ts +7 -6
- package/dist/src/types.d.ts.map +1 -1
- package/dist/src/util.d.ts +46 -19
- package/dist/src/util.d.ts.map +1 -1
- package/dist/src/util.js +363 -121
- package/dist/src/util.js.map +1 -1
- package/dist/src/web/nextui/404/index.html +1 -1
- package/dist/src/web/nextui/404.html +1 -1
- package/dist/src/web/nextui/_next/static/chunks/133-1318f46ed801a49d.js +6 -0
- package/dist/src/web/nextui/_next/static/chunks/{782-cc7e5fff4712ff69.js → 782-e07d19365960282c.js} +1 -1
- package/dist/src/web/nextui/_next/static/chunks/922.b293c6bd1040dbd7.js +1 -0
- package/dist/src/web/nextui/_next/static/chunks/952-ede6b209625d42a2.js +1 -0
- package/dist/src/web/nextui/_next/static/chunks/app/datasets/page-ad55f89d622ef8e7.js +1 -0
- package/dist/src/web/nextui/_next/static/chunks/app/eval/[id]/{page-f2a24fd303cf1367.js → page-bba5a197d928a824.js} +1 -1
- package/dist/src/web/nextui/_next/static/chunks/app/eval/{page-19730873b1e61513.js → page-f10300e40e4331ab.js} +1 -1
- package/dist/src/web/nextui/_next/static/chunks/app/prompts/page-01ab4878803b7068.js +1 -0
- package/dist/src/web/nextui/_next/static/chunks/app/setup/page-c3d032b73f8d1cc5.js +1 -0
- package/dist/src/web/nextui/_next/static/chunks/webpack-939efbb9ee2a5baf.js +1 -0
- package/dist/src/web/nextui/api/results +1 -1
- package/dist/src/web/nextui/auth/login/index.html +1 -1
- package/dist/src/web/nextui/auth/login/index.txt +3 -3
- package/dist/src/web/nextui/auth/signup/index.html +1 -1
- package/dist/src/web/nextui/auth/signup/index.txt +3 -3
- package/dist/src/web/nextui/datasets/index.html +1 -1
- package/dist/src/web/nextui/datasets/index.txt +4 -4
- package/dist/src/web/nextui/eval/index.html +1 -1
- package/dist/src/web/nextui/eval/index.txt +4 -4
- package/dist/src/web/nextui/index.html +1 -1
- package/dist/src/web/nextui/index.txt +3 -3
- package/dist/src/web/nextui/prompts/index.html +1 -1
- package/dist/src/web/nextui/prompts/index.txt +4 -4
- package/dist/src/web/nextui/setup/index.html +2 -2
- package/dist/src/web/nextui/setup/index.txt +5 -5
- package/dist/src/web/server.d.ts +1 -1
- package/dist/src/web/server.d.ts.map +1 -1
- package/dist/src/web/server.js +25 -43
- package/dist/src/web/server.js.map +1 -1
- package/package.json +13 -4
- package/dist/src/web/nextui/_next/static/chunks/631-b333d4a78100eabe.js +0 -6
- package/dist/src/web/nextui/_next/static/chunks/922.2b3e75a995f09e1b.js +0 -1
- package/dist/src/web/nextui/_next/static/chunks/952-1367984f076e3060.js +0 -1
- package/dist/src/web/nextui/_next/static/chunks/app/datasets/page-44ab188f3b846712.js +0 -1
- package/dist/src/web/nextui/_next/static/chunks/app/prompts/page-0bf3409d6a6bfa22.js +0 -1
- package/dist/src/web/nextui/_next/static/chunks/app/setup/page-83c7e62787113081.js +0 -1
- package/dist/src/web/nextui/_next/static/chunks/webpack-ff9953f1ad73867d.js +0 -1
- /package/dist/src/web/nextui/_next/static/{kWF8sUISiIgB0hKr0muJH → cmu3QLMtYY8lxMsDUojmv}/_buildManifest.js +0 -0
- /package/dist/src/web/nextui/_next/static/{kWF8sUISiIgB0hKr0muJH → cmu3QLMtYY8lxMsDUojmv}/_ssgManifest.js +0 -0
package/README.md
CHANGED
|
@@ -340,6 +340,8 @@ Here are some of the available scripts:
|
|
|
340
340
|
- `build:watch`: Continuously watch and transpile TypeScript files on changes
|
|
341
341
|
- `test`: Run test suite
|
|
342
342
|
- `test:watch`: Continuously run test suite on changes
|
|
343
|
+
- `db:generate`: Generate new db migrations (and create the db if it doesn't already exist). Note that after generating a new migration, you'll have to `npm i` to copy the migrations into `dist/`.
|
|
344
|
+
- `db:migrate`: Run existing db migrations (and create the db if it doesn't already exist)
|
|
343
345
|
|
|
344
346
|
# [» View full documentation «](https://promptfoo.dev/docs/intro)
|
|
345
347
|
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
CREATE TABLE `datasets` (
|
|
2
|
+
`id` text PRIMARY KEY NOT NULL,
|
|
3
|
+
`test_case_id` text NOT NULL,
|
|
4
|
+
`created_at` integer DEFAULT CURRENT_TIMESTAMP NOT NULL
|
|
5
|
+
);
|
|
6
|
+
--> statement-breakpoint
|
|
7
|
+
CREATE TABLE `evals` (
|
|
8
|
+
`id` text PRIMARY KEY NOT NULL,
|
|
9
|
+
`created_at` integer DEFAULT CURRENT_TIMESTAMP NOT NULL,
|
|
10
|
+
`description` text,
|
|
11
|
+
`results` text NOT NULL,
|
|
12
|
+
`config` text NOT NULL
|
|
13
|
+
);
|
|
14
|
+
--> statement-breakpoint
|
|
15
|
+
CREATE TABLE `evals_to_datasets` (
|
|
16
|
+
`eval_id` text NOT NULL,
|
|
17
|
+
`dataset_id` text NOT NULL,
|
|
18
|
+
PRIMARY KEY(`dataset_id`, `eval_id`),
|
|
19
|
+
FOREIGN KEY (`eval_id`) REFERENCES `evals`(`id`) ON UPDATE no action ON DELETE no action,
|
|
20
|
+
FOREIGN KEY (`dataset_id`) REFERENCES `datasets`(`id`) ON UPDATE no action ON DELETE no action
|
|
21
|
+
);
|
|
22
|
+
--> statement-breakpoint
|
|
23
|
+
CREATE TABLE `evals_to_prompts` (
|
|
24
|
+
`eval_id` text NOT NULL,
|
|
25
|
+
`prompt_id` text NOT NULL,
|
|
26
|
+
PRIMARY KEY(`eval_id`, `prompt_id`),
|
|
27
|
+
FOREIGN KEY (`eval_id`) REFERENCES `evals`(`id`) ON UPDATE no action ON DELETE no action,
|
|
28
|
+
FOREIGN KEY (`prompt_id`) REFERENCES `prompts`(`id`) ON UPDATE no action ON DELETE no action
|
|
29
|
+
);
|
|
30
|
+
--> statement-breakpoint
|
|
31
|
+
CREATE TABLE `prompts` (
|
|
32
|
+
`id` text PRIMARY KEY NOT NULL,
|
|
33
|
+
`created_at` integer DEFAULT CURRENT_TIMESTAMP NOT NULL,
|
|
34
|
+
`prompt` text NOT NULL,
|
|
35
|
+
`hash` text NOT NULL
|
|
36
|
+
);
|
|
@@ -0,0 +1,244 @@
|
|
|
1
|
+
{
|
|
2
|
+
"version": "5",
|
|
3
|
+
"dialect": "sqlite",
|
|
4
|
+
"id": "8b53403f-5b6f-436a-862e-9fd17a52204e",
|
|
5
|
+
"prevId": "00000000-0000-0000-0000-000000000000",
|
|
6
|
+
"tables": {
|
|
7
|
+
"datasets": {
|
|
8
|
+
"name": "datasets",
|
|
9
|
+
"columns": {
|
|
10
|
+
"id": {
|
|
11
|
+
"name": "id",
|
|
12
|
+
"type": "text",
|
|
13
|
+
"primaryKey": true,
|
|
14
|
+
"notNull": true,
|
|
15
|
+
"autoincrement": false
|
|
16
|
+
},
|
|
17
|
+
"test_case_id": {
|
|
18
|
+
"name": "test_case_id",
|
|
19
|
+
"type": "text",
|
|
20
|
+
"primaryKey": false,
|
|
21
|
+
"notNull": true,
|
|
22
|
+
"autoincrement": false
|
|
23
|
+
},
|
|
24
|
+
"created_at": {
|
|
25
|
+
"name": "created_at",
|
|
26
|
+
"type": "integer",
|
|
27
|
+
"primaryKey": false,
|
|
28
|
+
"notNull": true,
|
|
29
|
+
"autoincrement": false,
|
|
30
|
+
"default": "CURRENT_TIMESTAMP"
|
|
31
|
+
}
|
|
32
|
+
},
|
|
33
|
+
"indexes": {},
|
|
34
|
+
"foreignKeys": {},
|
|
35
|
+
"compositePrimaryKeys": {},
|
|
36
|
+
"uniqueConstraints": {}
|
|
37
|
+
},
|
|
38
|
+
"evals": {
|
|
39
|
+
"name": "evals",
|
|
40
|
+
"columns": {
|
|
41
|
+
"id": {
|
|
42
|
+
"name": "id",
|
|
43
|
+
"type": "text",
|
|
44
|
+
"primaryKey": true,
|
|
45
|
+
"notNull": true,
|
|
46
|
+
"autoincrement": false
|
|
47
|
+
},
|
|
48
|
+
"created_at": {
|
|
49
|
+
"name": "created_at",
|
|
50
|
+
"type": "integer",
|
|
51
|
+
"primaryKey": false,
|
|
52
|
+
"notNull": true,
|
|
53
|
+
"autoincrement": false,
|
|
54
|
+
"default": "CURRENT_TIMESTAMP"
|
|
55
|
+
},
|
|
56
|
+
"description": {
|
|
57
|
+
"name": "description",
|
|
58
|
+
"type": "text",
|
|
59
|
+
"primaryKey": false,
|
|
60
|
+
"notNull": false,
|
|
61
|
+
"autoincrement": false
|
|
62
|
+
},
|
|
63
|
+
"results": {
|
|
64
|
+
"name": "results",
|
|
65
|
+
"type": "text",
|
|
66
|
+
"primaryKey": false,
|
|
67
|
+
"notNull": true,
|
|
68
|
+
"autoincrement": false
|
|
69
|
+
},
|
|
70
|
+
"config": {
|
|
71
|
+
"name": "config",
|
|
72
|
+
"type": "text",
|
|
73
|
+
"primaryKey": false,
|
|
74
|
+
"notNull": true,
|
|
75
|
+
"autoincrement": false
|
|
76
|
+
}
|
|
77
|
+
},
|
|
78
|
+
"indexes": {},
|
|
79
|
+
"foreignKeys": {},
|
|
80
|
+
"compositePrimaryKeys": {},
|
|
81
|
+
"uniqueConstraints": {}
|
|
82
|
+
},
|
|
83
|
+
"evals_to_datasets": {
|
|
84
|
+
"name": "evals_to_datasets",
|
|
85
|
+
"columns": {
|
|
86
|
+
"eval_id": {
|
|
87
|
+
"name": "eval_id",
|
|
88
|
+
"type": "text",
|
|
89
|
+
"primaryKey": false,
|
|
90
|
+
"notNull": true,
|
|
91
|
+
"autoincrement": false
|
|
92
|
+
},
|
|
93
|
+
"dataset_id": {
|
|
94
|
+
"name": "dataset_id",
|
|
95
|
+
"type": "text",
|
|
96
|
+
"primaryKey": false,
|
|
97
|
+
"notNull": true,
|
|
98
|
+
"autoincrement": false
|
|
99
|
+
}
|
|
100
|
+
},
|
|
101
|
+
"indexes": {},
|
|
102
|
+
"foreignKeys": {
|
|
103
|
+
"evals_to_datasets_eval_id_evals_id_fk": {
|
|
104
|
+
"name": "evals_to_datasets_eval_id_evals_id_fk",
|
|
105
|
+
"tableFrom": "evals_to_datasets",
|
|
106
|
+
"tableTo": "evals",
|
|
107
|
+
"columnsFrom": [
|
|
108
|
+
"eval_id"
|
|
109
|
+
],
|
|
110
|
+
"columnsTo": [
|
|
111
|
+
"id"
|
|
112
|
+
],
|
|
113
|
+
"onDelete": "no action",
|
|
114
|
+
"onUpdate": "no action"
|
|
115
|
+
},
|
|
116
|
+
"evals_to_datasets_dataset_id_datasets_id_fk": {
|
|
117
|
+
"name": "evals_to_datasets_dataset_id_datasets_id_fk",
|
|
118
|
+
"tableFrom": "evals_to_datasets",
|
|
119
|
+
"tableTo": "datasets",
|
|
120
|
+
"columnsFrom": [
|
|
121
|
+
"dataset_id"
|
|
122
|
+
],
|
|
123
|
+
"columnsTo": [
|
|
124
|
+
"id"
|
|
125
|
+
],
|
|
126
|
+
"onDelete": "no action",
|
|
127
|
+
"onUpdate": "no action"
|
|
128
|
+
}
|
|
129
|
+
},
|
|
130
|
+
"compositePrimaryKeys": {
|
|
131
|
+
"evals_to_datasets_eval_id_dataset_id_pk": {
|
|
132
|
+
"columns": [
|
|
133
|
+
"dataset_id",
|
|
134
|
+
"eval_id"
|
|
135
|
+
],
|
|
136
|
+
"name": "evals_to_datasets_eval_id_dataset_id_pk"
|
|
137
|
+
}
|
|
138
|
+
},
|
|
139
|
+
"uniqueConstraints": {}
|
|
140
|
+
},
|
|
141
|
+
"evals_to_prompts": {
|
|
142
|
+
"name": "evals_to_prompts",
|
|
143
|
+
"columns": {
|
|
144
|
+
"eval_id": {
|
|
145
|
+
"name": "eval_id",
|
|
146
|
+
"type": "text",
|
|
147
|
+
"primaryKey": false,
|
|
148
|
+
"notNull": true,
|
|
149
|
+
"autoincrement": false
|
|
150
|
+
},
|
|
151
|
+
"prompt_id": {
|
|
152
|
+
"name": "prompt_id",
|
|
153
|
+
"type": "text",
|
|
154
|
+
"primaryKey": false,
|
|
155
|
+
"notNull": true,
|
|
156
|
+
"autoincrement": false
|
|
157
|
+
}
|
|
158
|
+
},
|
|
159
|
+
"indexes": {},
|
|
160
|
+
"foreignKeys": {
|
|
161
|
+
"evals_to_prompts_eval_id_evals_id_fk": {
|
|
162
|
+
"name": "evals_to_prompts_eval_id_evals_id_fk",
|
|
163
|
+
"tableFrom": "evals_to_prompts",
|
|
164
|
+
"tableTo": "evals",
|
|
165
|
+
"columnsFrom": [
|
|
166
|
+
"eval_id"
|
|
167
|
+
],
|
|
168
|
+
"columnsTo": [
|
|
169
|
+
"id"
|
|
170
|
+
],
|
|
171
|
+
"onDelete": "no action",
|
|
172
|
+
"onUpdate": "no action"
|
|
173
|
+
},
|
|
174
|
+
"evals_to_prompts_prompt_id_prompts_id_fk": {
|
|
175
|
+
"name": "evals_to_prompts_prompt_id_prompts_id_fk",
|
|
176
|
+
"tableFrom": "evals_to_prompts",
|
|
177
|
+
"tableTo": "prompts",
|
|
178
|
+
"columnsFrom": [
|
|
179
|
+
"prompt_id"
|
|
180
|
+
],
|
|
181
|
+
"columnsTo": [
|
|
182
|
+
"id"
|
|
183
|
+
],
|
|
184
|
+
"onDelete": "no action",
|
|
185
|
+
"onUpdate": "no action"
|
|
186
|
+
}
|
|
187
|
+
},
|
|
188
|
+
"compositePrimaryKeys": {
|
|
189
|
+
"evals_to_prompts_eval_id_prompt_id_pk": {
|
|
190
|
+
"columns": [
|
|
191
|
+
"eval_id",
|
|
192
|
+
"prompt_id"
|
|
193
|
+
],
|
|
194
|
+
"name": "evals_to_prompts_eval_id_prompt_id_pk"
|
|
195
|
+
}
|
|
196
|
+
},
|
|
197
|
+
"uniqueConstraints": {}
|
|
198
|
+
},
|
|
199
|
+
"prompts": {
|
|
200
|
+
"name": "prompts",
|
|
201
|
+
"columns": {
|
|
202
|
+
"id": {
|
|
203
|
+
"name": "id",
|
|
204
|
+
"type": "text",
|
|
205
|
+
"primaryKey": true,
|
|
206
|
+
"notNull": true,
|
|
207
|
+
"autoincrement": false
|
|
208
|
+
},
|
|
209
|
+
"created_at": {
|
|
210
|
+
"name": "created_at",
|
|
211
|
+
"type": "integer",
|
|
212
|
+
"primaryKey": false,
|
|
213
|
+
"notNull": true,
|
|
214
|
+
"autoincrement": false,
|
|
215
|
+
"default": "CURRENT_TIMESTAMP"
|
|
216
|
+
},
|
|
217
|
+
"prompt": {
|
|
218
|
+
"name": "prompt",
|
|
219
|
+
"type": "text",
|
|
220
|
+
"primaryKey": false,
|
|
221
|
+
"notNull": true,
|
|
222
|
+
"autoincrement": false
|
|
223
|
+
},
|
|
224
|
+
"hash": {
|
|
225
|
+
"name": "hash",
|
|
226
|
+
"type": "text",
|
|
227
|
+
"primaryKey": false,
|
|
228
|
+
"notNull": true,
|
|
229
|
+
"autoincrement": false
|
|
230
|
+
}
|
|
231
|
+
},
|
|
232
|
+
"indexes": {},
|
|
233
|
+
"foreignKeys": {},
|
|
234
|
+
"compositePrimaryKeys": {},
|
|
235
|
+
"uniqueConstraints": {}
|
|
236
|
+
}
|
|
237
|
+
},
|
|
238
|
+
"enums": {},
|
|
239
|
+
"_meta": {
|
|
240
|
+
"schemas": {},
|
|
241
|
+
"tables": {},
|
|
242
|
+
"columns": {}
|
|
243
|
+
}
|
|
244
|
+
}
|
|
@@ -0,0 +1,237 @@
|
|
|
1
|
+
{
|
|
2
|
+
"version": "5",
|
|
3
|
+
"dialect": "sqlite",
|
|
4
|
+
"id": "476c887f-edcb-4b29-b054-50a8383cc6c2",
|
|
5
|
+
"prevId": "8b53403f-5b6f-436a-862e-9fd17a52204e",
|
|
6
|
+
"tables": {
|
|
7
|
+
"datasets": {
|
|
8
|
+
"name": "datasets",
|
|
9
|
+
"columns": {
|
|
10
|
+
"id": {
|
|
11
|
+
"name": "id",
|
|
12
|
+
"type": "text",
|
|
13
|
+
"primaryKey": true,
|
|
14
|
+
"notNull": true,
|
|
15
|
+
"autoincrement": false
|
|
16
|
+
},
|
|
17
|
+
"tests": {
|
|
18
|
+
"name": "tests",
|
|
19
|
+
"type": "text",
|
|
20
|
+
"primaryKey": false,
|
|
21
|
+
"notNull": false,
|
|
22
|
+
"autoincrement": false
|
|
23
|
+
},
|
|
24
|
+
"created_at": {
|
|
25
|
+
"name": "created_at",
|
|
26
|
+
"type": "integer",
|
|
27
|
+
"primaryKey": false,
|
|
28
|
+
"notNull": true,
|
|
29
|
+
"autoincrement": false,
|
|
30
|
+
"default": "CURRENT_TIMESTAMP"
|
|
31
|
+
}
|
|
32
|
+
},
|
|
33
|
+
"indexes": {},
|
|
34
|
+
"foreignKeys": {},
|
|
35
|
+
"compositePrimaryKeys": {},
|
|
36
|
+
"uniqueConstraints": {}
|
|
37
|
+
},
|
|
38
|
+
"evals": {
|
|
39
|
+
"name": "evals",
|
|
40
|
+
"columns": {
|
|
41
|
+
"id": {
|
|
42
|
+
"name": "id",
|
|
43
|
+
"type": "text",
|
|
44
|
+
"primaryKey": true,
|
|
45
|
+
"notNull": true,
|
|
46
|
+
"autoincrement": false
|
|
47
|
+
},
|
|
48
|
+
"created_at": {
|
|
49
|
+
"name": "created_at",
|
|
50
|
+
"type": "integer",
|
|
51
|
+
"primaryKey": false,
|
|
52
|
+
"notNull": true,
|
|
53
|
+
"autoincrement": false,
|
|
54
|
+
"default": "CURRENT_TIMESTAMP"
|
|
55
|
+
},
|
|
56
|
+
"description": {
|
|
57
|
+
"name": "description",
|
|
58
|
+
"type": "text",
|
|
59
|
+
"primaryKey": false,
|
|
60
|
+
"notNull": false,
|
|
61
|
+
"autoincrement": false
|
|
62
|
+
},
|
|
63
|
+
"results": {
|
|
64
|
+
"name": "results",
|
|
65
|
+
"type": "text",
|
|
66
|
+
"primaryKey": false,
|
|
67
|
+
"notNull": true,
|
|
68
|
+
"autoincrement": false
|
|
69
|
+
},
|
|
70
|
+
"config": {
|
|
71
|
+
"name": "config",
|
|
72
|
+
"type": "text",
|
|
73
|
+
"primaryKey": false,
|
|
74
|
+
"notNull": true,
|
|
75
|
+
"autoincrement": false
|
|
76
|
+
}
|
|
77
|
+
},
|
|
78
|
+
"indexes": {},
|
|
79
|
+
"foreignKeys": {},
|
|
80
|
+
"compositePrimaryKeys": {},
|
|
81
|
+
"uniqueConstraints": {}
|
|
82
|
+
},
|
|
83
|
+
"evals_to_datasets": {
|
|
84
|
+
"name": "evals_to_datasets",
|
|
85
|
+
"columns": {
|
|
86
|
+
"eval_id": {
|
|
87
|
+
"name": "eval_id",
|
|
88
|
+
"type": "text",
|
|
89
|
+
"primaryKey": false,
|
|
90
|
+
"notNull": true,
|
|
91
|
+
"autoincrement": false
|
|
92
|
+
},
|
|
93
|
+
"dataset_id": {
|
|
94
|
+
"name": "dataset_id",
|
|
95
|
+
"type": "text",
|
|
96
|
+
"primaryKey": false,
|
|
97
|
+
"notNull": true,
|
|
98
|
+
"autoincrement": false
|
|
99
|
+
}
|
|
100
|
+
},
|
|
101
|
+
"indexes": {},
|
|
102
|
+
"foreignKeys": {
|
|
103
|
+
"evals_to_datasets_eval_id_evals_id_fk": {
|
|
104
|
+
"name": "evals_to_datasets_eval_id_evals_id_fk",
|
|
105
|
+
"tableFrom": "evals_to_datasets",
|
|
106
|
+
"tableTo": "evals",
|
|
107
|
+
"columnsFrom": [
|
|
108
|
+
"eval_id"
|
|
109
|
+
],
|
|
110
|
+
"columnsTo": [
|
|
111
|
+
"id"
|
|
112
|
+
],
|
|
113
|
+
"onDelete": "no action",
|
|
114
|
+
"onUpdate": "no action"
|
|
115
|
+
},
|
|
116
|
+
"evals_to_datasets_dataset_id_datasets_id_fk": {
|
|
117
|
+
"name": "evals_to_datasets_dataset_id_datasets_id_fk",
|
|
118
|
+
"tableFrom": "evals_to_datasets",
|
|
119
|
+
"tableTo": "datasets",
|
|
120
|
+
"columnsFrom": [
|
|
121
|
+
"dataset_id"
|
|
122
|
+
],
|
|
123
|
+
"columnsTo": [
|
|
124
|
+
"id"
|
|
125
|
+
],
|
|
126
|
+
"onDelete": "no action",
|
|
127
|
+
"onUpdate": "no action"
|
|
128
|
+
}
|
|
129
|
+
},
|
|
130
|
+
"compositePrimaryKeys": {
|
|
131
|
+
"evals_to_datasets_eval_id_dataset_id_pk": {
|
|
132
|
+
"columns": [
|
|
133
|
+
"dataset_id",
|
|
134
|
+
"eval_id"
|
|
135
|
+
],
|
|
136
|
+
"name": "evals_to_datasets_eval_id_dataset_id_pk"
|
|
137
|
+
}
|
|
138
|
+
},
|
|
139
|
+
"uniqueConstraints": {}
|
|
140
|
+
},
|
|
141
|
+
"evals_to_prompts": {
|
|
142
|
+
"name": "evals_to_prompts",
|
|
143
|
+
"columns": {
|
|
144
|
+
"eval_id": {
|
|
145
|
+
"name": "eval_id",
|
|
146
|
+
"type": "text",
|
|
147
|
+
"primaryKey": false,
|
|
148
|
+
"notNull": true,
|
|
149
|
+
"autoincrement": false
|
|
150
|
+
},
|
|
151
|
+
"prompt_id": {
|
|
152
|
+
"name": "prompt_id",
|
|
153
|
+
"type": "text",
|
|
154
|
+
"primaryKey": false,
|
|
155
|
+
"notNull": true,
|
|
156
|
+
"autoincrement": false
|
|
157
|
+
}
|
|
158
|
+
},
|
|
159
|
+
"indexes": {},
|
|
160
|
+
"foreignKeys": {
|
|
161
|
+
"evals_to_prompts_eval_id_evals_id_fk": {
|
|
162
|
+
"name": "evals_to_prompts_eval_id_evals_id_fk",
|
|
163
|
+
"tableFrom": "evals_to_prompts",
|
|
164
|
+
"tableTo": "evals",
|
|
165
|
+
"columnsFrom": [
|
|
166
|
+
"eval_id"
|
|
167
|
+
],
|
|
168
|
+
"columnsTo": [
|
|
169
|
+
"id"
|
|
170
|
+
],
|
|
171
|
+
"onDelete": "no action",
|
|
172
|
+
"onUpdate": "no action"
|
|
173
|
+
},
|
|
174
|
+
"evals_to_prompts_prompt_id_prompts_id_fk": {
|
|
175
|
+
"name": "evals_to_prompts_prompt_id_prompts_id_fk",
|
|
176
|
+
"tableFrom": "evals_to_prompts",
|
|
177
|
+
"tableTo": "prompts",
|
|
178
|
+
"columnsFrom": [
|
|
179
|
+
"prompt_id"
|
|
180
|
+
],
|
|
181
|
+
"columnsTo": [
|
|
182
|
+
"id"
|
|
183
|
+
],
|
|
184
|
+
"onDelete": "no action",
|
|
185
|
+
"onUpdate": "no action"
|
|
186
|
+
}
|
|
187
|
+
},
|
|
188
|
+
"compositePrimaryKeys": {
|
|
189
|
+
"evals_to_prompts_eval_id_prompt_id_pk": {
|
|
190
|
+
"columns": [
|
|
191
|
+
"eval_id",
|
|
192
|
+
"prompt_id"
|
|
193
|
+
],
|
|
194
|
+
"name": "evals_to_prompts_eval_id_prompt_id_pk"
|
|
195
|
+
}
|
|
196
|
+
},
|
|
197
|
+
"uniqueConstraints": {}
|
|
198
|
+
},
|
|
199
|
+
"prompts": {
|
|
200
|
+
"name": "prompts",
|
|
201
|
+
"columns": {
|
|
202
|
+
"id": {
|
|
203
|
+
"name": "id",
|
|
204
|
+
"type": "text",
|
|
205
|
+
"primaryKey": true,
|
|
206
|
+
"notNull": true,
|
|
207
|
+
"autoincrement": false
|
|
208
|
+
},
|
|
209
|
+
"created_at": {
|
|
210
|
+
"name": "created_at",
|
|
211
|
+
"type": "integer",
|
|
212
|
+
"primaryKey": false,
|
|
213
|
+
"notNull": true,
|
|
214
|
+
"autoincrement": false,
|
|
215
|
+
"default": "CURRENT_TIMESTAMP"
|
|
216
|
+
},
|
|
217
|
+
"prompt": {
|
|
218
|
+
"name": "prompt",
|
|
219
|
+
"type": "text",
|
|
220
|
+
"primaryKey": false,
|
|
221
|
+
"notNull": true,
|
|
222
|
+
"autoincrement": false
|
|
223
|
+
}
|
|
224
|
+
},
|
|
225
|
+
"indexes": {},
|
|
226
|
+
"foreignKeys": {},
|
|
227
|
+
"compositePrimaryKeys": {},
|
|
228
|
+
"uniqueConstraints": {}
|
|
229
|
+
}
|
|
230
|
+
},
|
|
231
|
+
"enums": {},
|
|
232
|
+
"_meta": {
|
|
233
|
+
"schemas": {},
|
|
234
|
+
"tables": {},
|
|
235
|
+
"columns": {}
|
|
236
|
+
}
|
|
237
|
+
}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
{
|
|
2
|
+
"version": "5",
|
|
3
|
+
"dialect": "sqlite",
|
|
4
|
+
"entries": [
|
|
5
|
+
{
|
|
6
|
+
"idx": 0,
|
|
7
|
+
"version": "5",
|
|
8
|
+
"when": 1710348564252,
|
|
9
|
+
"tag": "0000_lush_hellion",
|
|
10
|
+
"breakpoints": true
|
|
11
|
+
},
|
|
12
|
+
{
|
|
13
|
+
"idx": 1,
|
|
14
|
+
"version": "5",
|
|
15
|
+
"when": 1710706957023,
|
|
16
|
+
"tag": "0001_wide_calypso",
|
|
17
|
+
"breakpoints": true
|
|
18
|
+
}
|
|
19
|
+
]
|
|
20
|
+
}
|
package/dist/package.json
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
"name": "promptfoo",
|
|
3
3
|
"description": "LLM eval & testing toolkit",
|
|
4
4
|
"author": "Ian Webster",
|
|
5
|
-
"version": "0.
|
|
5
|
+
"version": "0.49.0",
|
|
6
6
|
"license": "MIT",
|
|
7
7
|
"type": "commonjs",
|
|
8
8
|
"repository": "promptfoo/promptfoo",
|
|
@@ -32,20 +32,26 @@
|
|
|
32
32
|
"build:clean": "rm -rf dist",
|
|
33
33
|
"build:nextui": "cd src/web/nextui && npm run build && { [ -z \"$NEXT_PUBLIC_PROMPTFOO_BUILD_STANDALONE_SERVER\" ] && cp -r out/ ../../../dist/src/web/nextui; } || true",
|
|
34
34
|
"build:watch": "tsc --watch",
|
|
35
|
-
"build": "tsc && cp src/*.html dist/src && cp src/python/wrapper.py dist/src/python && npm run build:nextui && chmod +x dist/src/main.js",
|
|
35
|
+
"build": "tsc && cp src/*.html dist/src && cp src/python/wrapper.py dist/src/python && cp -r drizzle/ dist/ && npm run build:nextui && chmod +x dist/src/main.js",
|
|
36
36
|
"prepare": "npm run install:nextui && npm run build:clean && npm run build",
|
|
37
37
|
"test": "jest",
|
|
38
38
|
"test:watch": "jest --watch",
|
|
39
|
-
"format": "prettier -w ."
|
|
39
|
+
"format": "prettier -w .",
|
|
40
|
+
"db:migrate": "npx tsx src/migrate.ts",
|
|
41
|
+
"db:generate": "npx drizzle-kit generate:sqlite",
|
|
42
|
+
"audit:fix": "npm audit fix && cd src/web/nextui && npm audit fix && cd ../../site && npm audit fix"
|
|
40
43
|
},
|
|
41
44
|
"peerDependencies": {
|
|
42
45
|
"@aws-sdk/client-bedrock-runtime": "^3.458.0",
|
|
43
|
-
"@azure/identity": "^4.0.0"
|
|
46
|
+
"@azure/identity": "^4.0.0",
|
|
47
|
+
"@azure/openai-assistants": "^1.0.0-beta.5",
|
|
48
|
+
"google-auth-library": "^9.7.0"
|
|
44
49
|
},
|
|
45
50
|
"devDependencies": {
|
|
46
51
|
"@aws-sdk/client-bedrock-runtime": "^3.458.0",
|
|
47
52
|
"@azure/identity": "^4.0.0",
|
|
48
53
|
"@types/async": "^3.2.20",
|
|
54
|
+
"@types/better-sqlite3": "^7.6.8",
|
|
49
55
|
"@types/cache-manager": "^4.0.2",
|
|
50
56
|
"@types/cache-manager-fs-hash": "^0.0.1",
|
|
51
57
|
"@types/cli-progress": "^3.11.0",
|
|
@@ -62,6 +68,7 @@
|
|
|
62
68
|
"@types/semver": "^7.5.0",
|
|
63
69
|
"@types/uuid": "^9.0.2",
|
|
64
70
|
"babel-jest": "^29.5.0",
|
|
71
|
+
"drizzle-kit": "^0.20.13",
|
|
65
72
|
"jest": "^29.5.0",
|
|
66
73
|
"jest-watch-typeahead": "^2.2.2",
|
|
67
74
|
"next": "^13.4.13",
|
|
@@ -76,6 +83,7 @@
|
|
|
76
83
|
"ajv": "^8.12.0",
|
|
77
84
|
"ajv-formats": "^2.1.1",
|
|
78
85
|
"async": "^3.2.4",
|
|
86
|
+
"better-sqlite3": "^9.3.0",
|
|
79
87
|
"cache-manager": "^4.1.0",
|
|
80
88
|
"cache-manager-fs-hash": "^1.0.0",
|
|
81
89
|
"chalk": "^4.1.2",
|
|
@@ -88,6 +96,7 @@
|
|
|
88
96
|
"csv-parse": "^5.3.8",
|
|
89
97
|
"csv-stringify": "^6.3.2",
|
|
90
98
|
"debounce": "^1.2.1",
|
|
99
|
+
"drizzle-orm": "^0.29.3",
|
|
91
100
|
"express": "^4.18.2",
|
|
92
101
|
"fastest-levenshtein": "^1.0.16",
|
|
93
102
|
"glob": "^10.2.6",
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"database.d.ts","sourceRoot":"","sources":["../../../src/__mocks__/database.ts"],"names":[],"mappings":";AAAA,QAAA,MAAM,cAAc,IAInB,CAAC;AAEF,QAAA,MAAM,aAAa,0BAAY,CAAC;AAEhC,QAAA,MAAM,eAAe,0BAGnB,CAAC"}
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
const mockDbInstance = {
|
|
3
|
+
// Mock any method you use from the dbInstance
|
|
4
|
+
// For example:
|
|
5
|
+
// query: jest.fn().mockResolvedValue({}),
|
|
6
|
+
};
|
|
7
|
+
const mockRelations = jest.fn();
|
|
8
|
+
const mockSqliteTable = jest.fn().mockImplementation((tableName, schema) => {
|
|
9
|
+
// You can customize this mock based on your testing needs
|
|
10
|
+
return { tableName, schema };
|
|
11
|
+
});
|
|
12
|
+
module.exports = {
|
|
13
|
+
prompts: mockSqliteTable('prompts', { /* schema definition */}),
|
|
14
|
+
promptsRelations: mockRelations,
|
|
15
|
+
datasets: mockSqliteTable('datasets', { /* schema definition */}),
|
|
16
|
+
datasetsRelations: mockRelations,
|
|
17
|
+
evals: mockSqliteTable('evals', { /* schema definition */}),
|
|
18
|
+
evalsRelations: mockRelations,
|
|
19
|
+
evalsToPrompts: mockSqliteTable('evals_to_prompts', { /* schema definition */}),
|
|
20
|
+
evalsToPromptsRelations: mockRelations,
|
|
21
|
+
evalsToDatasets: mockSqliteTable('evals_to_datasets', { /* schema definition */}),
|
|
22
|
+
evalsToDatasetsRelations: mockRelations,
|
|
23
|
+
llmOutputs: mockSqliteTable('llm_outputs', { /* schema definition */}),
|
|
24
|
+
llmOutputsRelations: mockRelations,
|
|
25
|
+
getDb: jest.fn(() => mockDbInstance),
|
|
26
|
+
};
|
|
27
|
+
//# sourceMappingURL=database.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"database.js","sourceRoot":"","sources":["../../../src/__mocks__/database.ts"],"names":[],"mappings":";AAAA,MAAM,cAAc,GAAG;AACrB,8CAA8C;AAC9C,eAAe;AACf,0CAA0C;CAC3C,CAAC;AAEF,MAAM,aAAa,GAAG,IAAI,CAAC,EAAE,EAAE,CAAC;AAEhC,MAAM,eAAe,GAAG,IAAI,CAAC,EAAE,EAAE,CAAC,kBAAkB,CAAC,CAAC,SAAS,EAAE,MAAM,EAAE,EAAE;IACzE,0DAA0D;IAC1D,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC;AAC/B,CAAC,CAAC,CAAC;AAEH,MAAM,CAAC,OAAO,GAAG;IACf,OAAO,EAAE,eAAe,CAAC,SAAS,EAAE,EAAC,uBAAuB,CAAC,CAAC;IAC9D,gBAAgB,EAAE,aAAa;IAC/B,QAAQ,EAAE,eAAe,CAAC,UAAU,EAAE,EAAC,uBAAuB,CAAC,CAAC;IAChE,iBAAiB,EAAE,aAAa;IAChC,KAAK,EAAE,eAAe,CAAC,OAAO,EAAE,EAAC,uBAAuB,CAAC,CAAC;IAC1D,cAAc,EAAE,aAAa;IAC7B,cAAc,EAAE,eAAe,CAAC,kBAAkB,EAAE,EAAC,uBAAuB,CAAC,CAAC;IAC9E,uBAAuB,EAAE,aAAa;IACtC,eAAe,EAAE,eAAe,CAAC,mBAAmB,EAAE,EAAC,uBAAuB,CAAC,CAAC;IAChF,wBAAwB,EAAE,aAAa;IACvC,UAAU,EAAE,eAAe,CAAC,aAAa,EAAE,EAAC,uBAAuB,CAAC,CAAC;IACrE,mBAAmB,EAAE,aAAa;IAClC,KAAK,EAAE,IAAI,CAAC,EAAE,CAAC,GAAG,EAAE,CAAC,cAAc,CAAC;CACrC,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"assertions.d.ts","sourceRoot":"","sources":["../../src/assertions.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"assertions.d.ts","sourceRoot":"","sources":["../../src/assertions.ts"],"names":[],"mappings":"AAgBA,OAAO,EACL,iBAAiB,EACjB,gBAAgB,EAChB,iBAAiB,EACjB,eAAe,EACf,qBAAqB,EACrB,sBAAsB,EACtB,oBAAoB,EACpB,uBAAuB,EACvB,0BAA0B,EAC1B,iBAAiB,EAClB,MAAM,YAAY,CAAC;AAKpB,OAAO,KAAK,EAAE,SAAS,EAAE,aAAa,EAAE,aAAa,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AAOpG,eAAO,MAAM,4BAA4B,oBASvC,CAAC;AAwCH,wBAAsB,aAAa,CAAC,EAClC,MAAM,EACN,QAAQ,EACR,IAAI,EACJ,MAAM,EACN,SAAS,EACT,QAAQ,EACR,IAAI,GACL,EAAE;IACD,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,WAAW,CAAC;IACvB,IAAI,EAAE,cAAc,CAAC;IACrB,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC;IACxB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;IACpB,IAAI,CAAC,EAAE,MAAM,CAAC;CACf,GAAG,OAAO,CAAC,aAAa,CAAC,CAwEzB;AAED,wBAAsB,YAAY,CAAC,EACjC,MAAM,EACN,QAAQ,EACR,SAAS,EACT,IAAI,EACJ,MAAM,EACN,SAAS,EACT,QAAQ,EACR,IAAI,GACL,EAAE;IACD,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,WAAW,CAAC;IACvB,SAAS,EAAE,SAAS,CAAC;IACrB,IAAI,EAAE,cAAc,CAAC;IACrB,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC;IACxB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;IACpB,IAAI,CAAC,EAAE,MAAM,CAAC;CACf,GAAG,OAAO,CAAC,aAAa,CAAC,CA87BzB;AAgCD,wBAAsB,mBAAmB,CACvC,IAAI,EAAE,cAAc,EACpB,SAAS,EAAE,SAAS,EACpB,OAAO,EAAE,MAAM,EAAE,GAChB,OAAO,CAAC,aAAa,EAAE,CAAC,CAe1B;AAED,wBAAsB,cAAc,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC,CAU3E;;;;;;;;;;;;;AAGD,wBAWE"}
|