promptfoo 0.47.0 → 0.49.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (139) hide show
  1. package/README.md +2 -0
  2. package/dist/drizzle/0000_lush_hellion.sql +36 -0
  3. package/dist/drizzle/0001_wide_calypso.sql +3 -0
  4. package/dist/drizzle/meta/0000_snapshot.json +244 -0
  5. package/dist/drizzle/meta/0001_snapshot.json +237 -0
  6. package/dist/drizzle/meta/_journal.json +20 -0
  7. package/dist/package.json +13 -4
  8. package/dist/src/__mocks__/database.d.ts +5 -0
  9. package/dist/src/__mocks__/database.d.ts.map +1 -0
  10. package/dist/src/__mocks__/database.js +27 -0
  11. package/dist/src/__mocks__/database.js.map +1 -0
  12. package/dist/src/assertions.d.ts.map +1 -1
  13. package/dist/src/assertions.js +16 -7
  14. package/dist/src/assertions.js.map +1 -1
  15. package/dist/src/cache.d.ts.map +1 -1
  16. package/dist/src/cache.js +13 -6
  17. package/dist/src/cache.js.map +1 -1
  18. package/dist/src/commands/list.d.ts.map +1 -1
  19. package/dist/src/commands/list.js +4 -5
  20. package/dist/src/commands/list.js.map +1 -1
  21. package/dist/src/commands/show.d.ts +1 -1
  22. package/dist/src/commands/show.d.ts.map +1 -1
  23. package/dist/src/commands/show.js +7 -7
  24. package/dist/src/commands/show.js.map +1 -1
  25. package/dist/src/csv.d.ts +1 -1
  26. package/dist/src/csv.d.ts.map +1 -1
  27. package/dist/src/csv.js +5 -0
  28. package/dist/src/csv.js.map +1 -1
  29. package/dist/src/database.d.ts +238 -0
  30. package/dist/src/database.d.ts.map +1 -0
  31. package/dist/src/database.js +141 -0
  32. package/dist/src/database.js.map +1 -0
  33. package/dist/src/evaluator.d.ts.map +1 -1
  34. package/dist/src/evaluator.js +22 -18
  35. package/dist/src/evaluator.js.map +1 -1
  36. package/dist/src/index.d.ts.map +1 -1
  37. package/dist/src/index.js +2 -1
  38. package/dist/src/index.js.map +1 -1
  39. package/dist/src/main.js +23 -10
  40. package/dist/src/main.js.map +1 -1
  41. package/dist/src/matchers.d.ts +9 -1
  42. package/dist/src/matchers.d.ts.map +1 -1
  43. package/dist/src/matchers.js +20 -3
  44. package/dist/src/matchers.js.map +1 -1
  45. package/dist/src/migrate.d.ts +5 -0
  46. package/dist/src/migrate.d.ts.map +1 -0
  47. package/dist/src/migrate.js +50 -0
  48. package/dist/src/migrate.js.map +1 -0
  49. package/dist/src/prompts.d.ts.map +1 -1
  50. package/dist/src/prompts.js +3 -0
  51. package/dist/src/prompts.js.map +1 -1
  52. package/dist/src/providers/anthropic.d.ts +10 -0
  53. package/dist/src/providers/anthropic.d.ts.map +1 -1
  54. package/dist/src/providers/anthropic.js +3 -2
  55. package/dist/src/providers/anthropic.js.map +1 -1
  56. package/dist/src/providers/azureopenai.d.ts +23 -0
  57. package/dist/src/providers/azureopenai.d.ts.map +1 -1
  58. package/dist/src/providers/azureopenai.js +141 -3
  59. package/dist/src/providers/azureopenai.js.map +1 -1
  60. package/dist/src/providers/bedrock.d.ts.map +1 -1
  61. package/dist/src/providers/bedrock.js +40 -12
  62. package/dist/src/providers/bedrock.js.map +1 -1
  63. package/dist/src/providers/huggingface.d.ts +18 -0
  64. package/dist/src/providers/huggingface.d.ts.map +1 -1
  65. package/dist/src/providers/huggingface.js +71 -1
  66. package/dist/src/providers/huggingface.js.map +1 -1
  67. package/dist/src/providers/mistral.d.ts +5 -0
  68. package/dist/src/providers/mistral.d.ts.map +1 -1
  69. package/dist/src/providers/mistral.js +23 -8
  70. package/dist/src/providers/mistral.js.map +1 -1
  71. package/dist/src/providers/openai.d.ts.map +1 -1
  72. package/dist/src/providers/openai.js +1 -4
  73. package/dist/src/providers/openai.js.map +1 -1
  74. package/dist/src/providers/shared.d.ts +1 -0
  75. package/dist/src/providers/shared.d.ts.map +1 -1
  76. package/dist/src/providers/shared.js +5 -1
  77. package/dist/src/providers/shared.js.map +1 -1
  78. package/dist/src/providers/vertex.d.ts +1 -1
  79. package/dist/src/providers/vertex.d.ts.map +1 -1
  80. package/dist/src/providers/vertex.js +120 -39
  81. package/dist/src/providers/vertex.js.map +1 -1
  82. package/dist/src/providers/vertexUtil.d.ts +20 -5
  83. package/dist/src/providers/vertexUtil.d.ts.map +1 -1
  84. package/dist/src/providers/vertexUtil.js.map +1 -1
  85. package/dist/src/providers.d.ts.map +1 -1
  86. package/dist/src/providers.js +13 -10
  87. package/dist/src/providers.js.map +1 -1
  88. package/dist/src/share.d.ts.map +1 -1
  89. package/dist/src/share.js +5 -2
  90. package/dist/src/share.js.map +1 -1
  91. package/dist/src/table.js +1 -1
  92. package/dist/src/table.js.map +1 -1
  93. package/dist/src/types.d.ts +7 -6
  94. package/dist/src/types.d.ts.map +1 -1
  95. package/dist/src/util.d.ts +46 -19
  96. package/dist/src/util.d.ts.map +1 -1
  97. package/dist/src/util.js +363 -121
  98. package/dist/src/util.js.map +1 -1
  99. package/dist/src/web/nextui/404/index.html +1 -1
  100. package/dist/src/web/nextui/404.html +1 -1
  101. package/dist/src/web/nextui/_next/static/chunks/133-1318f46ed801a49d.js +6 -0
  102. package/dist/src/web/nextui/_next/static/chunks/{782-cc7e5fff4712ff69.js → 782-e07d19365960282c.js} +1 -1
  103. package/dist/src/web/nextui/_next/static/chunks/922.b293c6bd1040dbd7.js +1 -0
  104. package/dist/src/web/nextui/_next/static/chunks/952-ede6b209625d42a2.js +1 -0
  105. package/dist/src/web/nextui/_next/static/chunks/app/datasets/page-ad55f89d622ef8e7.js +1 -0
  106. package/dist/src/web/nextui/_next/static/chunks/app/eval/[id]/{page-f2a24fd303cf1367.js → page-bba5a197d928a824.js} +1 -1
  107. package/dist/src/web/nextui/_next/static/chunks/app/eval/{page-19730873b1e61513.js → page-f10300e40e4331ab.js} +1 -1
  108. package/dist/src/web/nextui/_next/static/chunks/app/prompts/page-01ab4878803b7068.js +1 -0
  109. package/dist/src/web/nextui/_next/static/chunks/app/setup/page-c3d032b73f8d1cc5.js +1 -0
  110. package/dist/src/web/nextui/_next/static/chunks/webpack-939efbb9ee2a5baf.js +1 -0
  111. package/dist/src/web/nextui/api/results +1 -1
  112. package/dist/src/web/nextui/auth/login/index.html +1 -1
  113. package/dist/src/web/nextui/auth/login/index.txt +3 -3
  114. package/dist/src/web/nextui/auth/signup/index.html +1 -1
  115. package/dist/src/web/nextui/auth/signup/index.txt +3 -3
  116. package/dist/src/web/nextui/datasets/index.html +1 -1
  117. package/dist/src/web/nextui/datasets/index.txt +4 -4
  118. package/dist/src/web/nextui/eval/index.html +1 -1
  119. package/dist/src/web/nextui/eval/index.txt +4 -4
  120. package/dist/src/web/nextui/index.html +1 -1
  121. package/dist/src/web/nextui/index.txt +3 -3
  122. package/dist/src/web/nextui/prompts/index.html +1 -1
  123. package/dist/src/web/nextui/prompts/index.txt +4 -4
  124. package/dist/src/web/nextui/setup/index.html +2 -2
  125. package/dist/src/web/nextui/setup/index.txt +5 -5
  126. package/dist/src/web/server.d.ts +1 -1
  127. package/dist/src/web/server.d.ts.map +1 -1
  128. package/dist/src/web/server.js +25 -43
  129. package/dist/src/web/server.js.map +1 -1
  130. package/package.json +13 -4
  131. package/dist/src/web/nextui/_next/static/chunks/631-b333d4a78100eabe.js +0 -6
  132. package/dist/src/web/nextui/_next/static/chunks/922.2b3e75a995f09e1b.js +0 -1
  133. package/dist/src/web/nextui/_next/static/chunks/952-1367984f076e3060.js +0 -1
  134. package/dist/src/web/nextui/_next/static/chunks/app/datasets/page-44ab188f3b846712.js +0 -1
  135. package/dist/src/web/nextui/_next/static/chunks/app/prompts/page-0bf3409d6a6bfa22.js +0 -1
  136. package/dist/src/web/nextui/_next/static/chunks/app/setup/page-83c7e62787113081.js +0 -1
  137. package/dist/src/web/nextui/_next/static/chunks/webpack-ff9953f1ad73867d.js +0 -1
  138. /package/dist/src/web/nextui/_next/static/{kWF8sUISiIgB0hKr0muJH → cmu3QLMtYY8lxMsDUojmv}/_buildManifest.js +0 -0
  139. /package/dist/src/web/nextui/_next/static/{kWF8sUISiIgB0hKr0muJH → cmu3QLMtYY8lxMsDUojmv}/_ssgManifest.js +0 -0
package/README.md CHANGED
@@ -340,6 +340,8 @@ Here are some of the available scripts:
340
340
  - `build:watch`: Continuously watch and transpile TypeScript files on changes
341
341
  - `test`: Run test suite
342
342
  - `test:watch`: Continuously run test suite on changes
343
+ - `db:generate`: Generate new db migrations (and create the db if it doesn't already exist). Note that after generating a new migration, you'll have to `npm i` to copy the migrations into `dist/`.
344
+ - `db:migrate`: Run existing db migrations (and create the db if it doesn't already exist)
343
345
 
344
346
  # [» View full documentation «](https://promptfoo.dev/docs/intro)
345
347
 
@@ -0,0 +1,36 @@
1
+ CREATE TABLE `datasets` (
2
+ `id` text PRIMARY KEY NOT NULL,
3
+ `test_case_id` text NOT NULL,
4
+ `created_at` integer DEFAULT CURRENT_TIMESTAMP NOT NULL
5
+ );
6
+ --> statement-breakpoint
7
+ CREATE TABLE `evals` (
8
+ `id` text PRIMARY KEY NOT NULL,
9
+ `created_at` integer DEFAULT CURRENT_TIMESTAMP NOT NULL,
10
+ `description` text,
11
+ `results` text NOT NULL,
12
+ `config` text NOT NULL
13
+ );
14
+ --> statement-breakpoint
15
+ CREATE TABLE `evals_to_datasets` (
16
+ `eval_id` text NOT NULL,
17
+ `dataset_id` text NOT NULL,
18
+ PRIMARY KEY(`dataset_id`, `eval_id`),
19
+ FOREIGN KEY (`eval_id`) REFERENCES `evals`(`id`) ON UPDATE no action ON DELETE no action,
20
+ FOREIGN KEY (`dataset_id`) REFERENCES `datasets`(`id`) ON UPDATE no action ON DELETE no action
21
+ );
22
+ --> statement-breakpoint
23
+ CREATE TABLE `evals_to_prompts` (
24
+ `eval_id` text NOT NULL,
25
+ `prompt_id` text NOT NULL,
26
+ PRIMARY KEY(`eval_id`, `prompt_id`),
27
+ FOREIGN KEY (`eval_id`) REFERENCES `evals`(`id`) ON UPDATE no action ON DELETE no action,
28
+ FOREIGN KEY (`prompt_id`) REFERENCES `prompts`(`id`) ON UPDATE no action ON DELETE no action
29
+ );
30
+ --> statement-breakpoint
31
+ CREATE TABLE `prompts` (
32
+ `id` text PRIMARY KEY NOT NULL,
33
+ `created_at` integer DEFAULT CURRENT_TIMESTAMP NOT NULL,
34
+ `prompt` text NOT NULL,
35
+ `hash` text NOT NULL
36
+ );
@@ -0,0 +1,3 @@
1
+ ALTER TABLE datasets ADD `tests` text;--> statement-breakpoint
2
+ ALTER TABLE `datasets` DROP COLUMN `test_case_id`;--> statement-breakpoint
3
+ ALTER TABLE `prompts` DROP COLUMN `hash`;
@@ -0,0 +1,244 @@
1
+ {
2
+ "version": "5",
3
+ "dialect": "sqlite",
4
+ "id": "8b53403f-5b6f-436a-862e-9fd17a52204e",
5
+ "prevId": "00000000-0000-0000-0000-000000000000",
6
+ "tables": {
7
+ "datasets": {
8
+ "name": "datasets",
9
+ "columns": {
10
+ "id": {
11
+ "name": "id",
12
+ "type": "text",
13
+ "primaryKey": true,
14
+ "notNull": true,
15
+ "autoincrement": false
16
+ },
17
+ "test_case_id": {
18
+ "name": "test_case_id",
19
+ "type": "text",
20
+ "primaryKey": false,
21
+ "notNull": true,
22
+ "autoincrement": false
23
+ },
24
+ "created_at": {
25
+ "name": "created_at",
26
+ "type": "integer",
27
+ "primaryKey": false,
28
+ "notNull": true,
29
+ "autoincrement": false,
30
+ "default": "CURRENT_TIMESTAMP"
31
+ }
32
+ },
33
+ "indexes": {},
34
+ "foreignKeys": {},
35
+ "compositePrimaryKeys": {},
36
+ "uniqueConstraints": {}
37
+ },
38
+ "evals": {
39
+ "name": "evals",
40
+ "columns": {
41
+ "id": {
42
+ "name": "id",
43
+ "type": "text",
44
+ "primaryKey": true,
45
+ "notNull": true,
46
+ "autoincrement": false
47
+ },
48
+ "created_at": {
49
+ "name": "created_at",
50
+ "type": "integer",
51
+ "primaryKey": false,
52
+ "notNull": true,
53
+ "autoincrement": false,
54
+ "default": "CURRENT_TIMESTAMP"
55
+ },
56
+ "description": {
57
+ "name": "description",
58
+ "type": "text",
59
+ "primaryKey": false,
60
+ "notNull": false,
61
+ "autoincrement": false
62
+ },
63
+ "results": {
64
+ "name": "results",
65
+ "type": "text",
66
+ "primaryKey": false,
67
+ "notNull": true,
68
+ "autoincrement": false
69
+ },
70
+ "config": {
71
+ "name": "config",
72
+ "type": "text",
73
+ "primaryKey": false,
74
+ "notNull": true,
75
+ "autoincrement": false
76
+ }
77
+ },
78
+ "indexes": {},
79
+ "foreignKeys": {},
80
+ "compositePrimaryKeys": {},
81
+ "uniqueConstraints": {}
82
+ },
83
+ "evals_to_datasets": {
84
+ "name": "evals_to_datasets",
85
+ "columns": {
86
+ "eval_id": {
87
+ "name": "eval_id",
88
+ "type": "text",
89
+ "primaryKey": false,
90
+ "notNull": true,
91
+ "autoincrement": false
92
+ },
93
+ "dataset_id": {
94
+ "name": "dataset_id",
95
+ "type": "text",
96
+ "primaryKey": false,
97
+ "notNull": true,
98
+ "autoincrement": false
99
+ }
100
+ },
101
+ "indexes": {},
102
+ "foreignKeys": {
103
+ "evals_to_datasets_eval_id_evals_id_fk": {
104
+ "name": "evals_to_datasets_eval_id_evals_id_fk",
105
+ "tableFrom": "evals_to_datasets",
106
+ "tableTo": "evals",
107
+ "columnsFrom": [
108
+ "eval_id"
109
+ ],
110
+ "columnsTo": [
111
+ "id"
112
+ ],
113
+ "onDelete": "no action",
114
+ "onUpdate": "no action"
115
+ },
116
+ "evals_to_datasets_dataset_id_datasets_id_fk": {
117
+ "name": "evals_to_datasets_dataset_id_datasets_id_fk",
118
+ "tableFrom": "evals_to_datasets",
119
+ "tableTo": "datasets",
120
+ "columnsFrom": [
121
+ "dataset_id"
122
+ ],
123
+ "columnsTo": [
124
+ "id"
125
+ ],
126
+ "onDelete": "no action",
127
+ "onUpdate": "no action"
128
+ }
129
+ },
130
+ "compositePrimaryKeys": {
131
+ "evals_to_datasets_eval_id_dataset_id_pk": {
132
+ "columns": [
133
+ "dataset_id",
134
+ "eval_id"
135
+ ],
136
+ "name": "evals_to_datasets_eval_id_dataset_id_pk"
137
+ }
138
+ },
139
+ "uniqueConstraints": {}
140
+ },
141
+ "evals_to_prompts": {
142
+ "name": "evals_to_prompts",
143
+ "columns": {
144
+ "eval_id": {
145
+ "name": "eval_id",
146
+ "type": "text",
147
+ "primaryKey": false,
148
+ "notNull": true,
149
+ "autoincrement": false
150
+ },
151
+ "prompt_id": {
152
+ "name": "prompt_id",
153
+ "type": "text",
154
+ "primaryKey": false,
155
+ "notNull": true,
156
+ "autoincrement": false
157
+ }
158
+ },
159
+ "indexes": {},
160
+ "foreignKeys": {
161
+ "evals_to_prompts_eval_id_evals_id_fk": {
162
+ "name": "evals_to_prompts_eval_id_evals_id_fk",
163
+ "tableFrom": "evals_to_prompts",
164
+ "tableTo": "evals",
165
+ "columnsFrom": [
166
+ "eval_id"
167
+ ],
168
+ "columnsTo": [
169
+ "id"
170
+ ],
171
+ "onDelete": "no action",
172
+ "onUpdate": "no action"
173
+ },
174
+ "evals_to_prompts_prompt_id_prompts_id_fk": {
175
+ "name": "evals_to_prompts_prompt_id_prompts_id_fk",
176
+ "tableFrom": "evals_to_prompts",
177
+ "tableTo": "prompts",
178
+ "columnsFrom": [
179
+ "prompt_id"
180
+ ],
181
+ "columnsTo": [
182
+ "id"
183
+ ],
184
+ "onDelete": "no action",
185
+ "onUpdate": "no action"
186
+ }
187
+ },
188
+ "compositePrimaryKeys": {
189
+ "evals_to_prompts_eval_id_prompt_id_pk": {
190
+ "columns": [
191
+ "eval_id",
192
+ "prompt_id"
193
+ ],
194
+ "name": "evals_to_prompts_eval_id_prompt_id_pk"
195
+ }
196
+ },
197
+ "uniqueConstraints": {}
198
+ },
199
+ "prompts": {
200
+ "name": "prompts",
201
+ "columns": {
202
+ "id": {
203
+ "name": "id",
204
+ "type": "text",
205
+ "primaryKey": true,
206
+ "notNull": true,
207
+ "autoincrement": false
208
+ },
209
+ "created_at": {
210
+ "name": "created_at",
211
+ "type": "integer",
212
+ "primaryKey": false,
213
+ "notNull": true,
214
+ "autoincrement": false,
215
+ "default": "CURRENT_TIMESTAMP"
216
+ },
217
+ "prompt": {
218
+ "name": "prompt",
219
+ "type": "text",
220
+ "primaryKey": false,
221
+ "notNull": true,
222
+ "autoincrement": false
223
+ },
224
+ "hash": {
225
+ "name": "hash",
226
+ "type": "text",
227
+ "primaryKey": false,
228
+ "notNull": true,
229
+ "autoincrement": false
230
+ }
231
+ },
232
+ "indexes": {},
233
+ "foreignKeys": {},
234
+ "compositePrimaryKeys": {},
235
+ "uniqueConstraints": {}
236
+ }
237
+ },
238
+ "enums": {},
239
+ "_meta": {
240
+ "schemas": {},
241
+ "tables": {},
242
+ "columns": {}
243
+ }
244
+ }
@@ -0,0 +1,237 @@
1
+ {
2
+ "version": "5",
3
+ "dialect": "sqlite",
4
+ "id": "476c887f-edcb-4b29-b054-50a8383cc6c2",
5
+ "prevId": "8b53403f-5b6f-436a-862e-9fd17a52204e",
6
+ "tables": {
7
+ "datasets": {
8
+ "name": "datasets",
9
+ "columns": {
10
+ "id": {
11
+ "name": "id",
12
+ "type": "text",
13
+ "primaryKey": true,
14
+ "notNull": true,
15
+ "autoincrement": false
16
+ },
17
+ "tests": {
18
+ "name": "tests",
19
+ "type": "text",
20
+ "primaryKey": false,
21
+ "notNull": false,
22
+ "autoincrement": false
23
+ },
24
+ "created_at": {
25
+ "name": "created_at",
26
+ "type": "integer",
27
+ "primaryKey": false,
28
+ "notNull": true,
29
+ "autoincrement": false,
30
+ "default": "CURRENT_TIMESTAMP"
31
+ }
32
+ },
33
+ "indexes": {},
34
+ "foreignKeys": {},
35
+ "compositePrimaryKeys": {},
36
+ "uniqueConstraints": {}
37
+ },
38
+ "evals": {
39
+ "name": "evals",
40
+ "columns": {
41
+ "id": {
42
+ "name": "id",
43
+ "type": "text",
44
+ "primaryKey": true,
45
+ "notNull": true,
46
+ "autoincrement": false
47
+ },
48
+ "created_at": {
49
+ "name": "created_at",
50
+ "type": "integer",
51
+ "primaryKey": false,
52
+ "notNull": true,
53
+ "autoincrement": false,
54
+ "default": "CURRENT_TIMESTAMP"
55
+ },
56
+ "description": {
57
+ "name": "description",
58
+ "type": "text",
59
+ "primaryKey": false,
60
+ "notNull": false,
61
+ "autoincrement": false
62
+ },
63
+ "results": {
64
+ "name": "results",
65
+ "type": "text",
66
+ "primaryKey": false,
67
+ "notNull": true,
68
+ "autoincrement": false
69
+ },
70
+ "config": {
71
+ "name": "config",
72
+ "type": "text",
73
+ "primaryKey": false,
74
+ "notNull": true,
75
+ "autoincrement": false
76
+ }
77
+ },
78
+ "indexes": {},
79
+ "foreignKeys": {},
80
+ "compositePrimaryKeys": {},
81
+ "uniqueConstraints": {}
82
+ },
83
+ "evals_to_datasets": {
84
+ "name": "evals_to_datasets",
85
+ "columns": {
86
+ "eval_id": {
87
+ "name": "eval_id",
88
+ "type": "text",
89
+ "primaryKey": false,
90
+ "notNull": true,
91
+ "autoincrement": false
92
+ },
93
+ "dataset_id": {
94
+ "name": "dataset_id",
95
+ "type": "text",
96
+ "primaryKey": false,
97
+ "notNull": true,
98
+ "autoincrement": false
99
+ }
100
+ },
101
+ "indexes": {},
102
+ "foreignKeys": {
103
+ "evals_to_datasets_eval_id_evals_id_fk": {
104
+ "name": "evals_to_datasets_eval_id_evals_id_fk",
105
+ "tableFrom": "evals_to_datasets",
106
+ "tableTo": "evals",
107
+ "columnsFrom": [
108
+ "eval_id"
109
+ ],
110
+ "columnsTo": [
111
+ "id"
112
+ ],
113
+ "onDelete": "no action",
114
+ "onUpdate": "no action"
115
+ },
116
+ "evals_to_datasets_dataset_id_datasets_id_fk": {
117
+ "name": "evals_to_datasets_dataset_id_datasets_id_fk",
118
+ "tableFrom": "evals_to_datasets",
119
+ "tableTo": "datasets",
120
+ "columnsFrom": [
121
+ "dataset_id"
122
+ ],
123
+ "columnsTo": [
124
+ "id"
125
+ ],
126
+ "onDelete": "no action",
127
+ "onUpdate": "no action"
128
+ }
129
+ },
130
+ "compositePrimaryKeys": {
131
+ "evals_to_datasets_eval_id_dataset_id_pk": {
132
+ "columns": [
133
+ "dataset_id",
134
+ "eval_id"
135
+ ],
136
+ "name": "evals_to_datasets_eval_id_dataset_id_pk"
137
+ }
138
+ },
139
+ "uniqueConstraints": {}
140
+ },
141
+ "evals_to_prompts": {
142
+ "name": "evals_to_prompts",
143
+ "columns": {
144
+ "eval_id": {
145
+ "name": "eval_id",
146
+ "type": "text",
147
+ "primaryKey": false,
148
+ "notNull": true,
149
+ "autoincrement": false
150
+ },
151
+ "prompt_id": {
152
+ "name": "prompt_id",
153
+ "type": "text",
154
+ "primaryKey": false,
155
+ "notNull": true,
156
+ "autoincrement": false
157
+ }
158
+ },
159
+ "indexes": {},
160
+ "foreignKeys": {
161
+ "evals_to_prompts_eval_id_evals_id_fk": {
162
+ "name": "evals_to_prompts_eval_id_evals_id_fk",
163
+ "tableFrom": "evals_to_prompts",
164
+ "tableTo": "evals",
165
+ "columnsFrom": [
166
+ "eval_id"
167
+ ],
168
+ "columnsTo": [
169
+ "id"
170
+ ],
171
+ "onDelete": "no action",
172
+ "onUpdate": "no action"
173
+ },
174
+ "evals_to_prompts_prompt_id_prompts_id_fk": {
175
+ "name": "evals_to_prompts_prompt_id_prompts_id_fk",
176
+ "tableFrom": "evals_to_prompts",
177
+ "tableTo": "prompts",
178
+ "columnsFrom": [
179
+ "prompt_id"
180
+ ],
181
+ "columnsTo": [
182
+ "id"
183
+ ],
184
+ "onDelete": "no action",
185
+ "onUpdate": "no action"
186
+ }
187
+ },
188
+ "compositePrimaryKeys": {
189
+ "evals_to_prompts_eval_id_prompt_id_pk": {
190
+ "columns": [
191
+ "eval_id",
192
+ "prompt_id"
193
+ ],
194
+ "name": "evals_to_prompts_eval_id_prompt_id_pk"
195
+ }
196
+ },
197
+ "uniqueConstraints": {}
198
+ },
199
+ "prompts": {
200
+ "name": "prompts",
201
+ "columns": {
202
+ "id": {
203
+ "name": "id",
204
+ "type": "text",
205
+ "primaryKey": true,
206
+ "notNull": true,
207
+ "autoincrement": false
208
+ },
209
+ "created_at": {
210
+ "name": "created_at",
211
+ "type": "integer",
212
+ "primaryKey": false,
213
+ "notNull": true,
214
+ "autoincrement": false,
215
+ "default": "CURRENT_TIMESTAMP"
216
+ },
217
+ "prompt": {
218
+ "name": "prompt",
219
+ "type": "text",
220
+ "primaryKey": false,
221
+ "notNull": true,
222
+ "autoincrement": false
223
+ }
224
+ },
225
+ "indexes": {},
226
+ "foreignKeys": {},
227
+ "compositePrimaryKeys": {},
228
+ "uniqueConstraints": {}
229
+ }
230
+ },
231
+ "enums": {},
232
+ "_meta": {
233
+ "schemas": {},
234
+ "tables": {},
235
+ "columns": {}
236
+ }
237
+ }
@@ -0,0 +1,20 @@
1
+ {
2
+ "version": "5",
3
+ "dialect": "sqlite",
4
+ "entries": [
5
+ {
6
+ "idx": 0,
7
+ "version": "5",
8
+ "when": 1710348564252,
9
+ "tag": "0000_lush_hellion",
10
+ "breakpoints": true
11
+ },
12
+ {
13
+ "idx": 1,
14
+ "version": "5",
15
+ "when": 1710706957023,
16
+ "tag": "0001_wide_calypso",
17
+ "breakpoints": true
18
+ }
19
+ ]
20
+ }
package/dist/package.json CHANGED
@@ -2,7 +2,7 @@
2
2
  "name": "promptfoo",
3
3
  "description": "LLM eval & testing toolkit",
4
4
  "author": "Ian Webster",
5
- "version": "0.47.0",
5
+ "version": "0.49.0",
6
6
  "license": "MIT",
7
7
  "type": "commonjs",
8
8
  "repository": "promptfoo/promptfoo",
@@ -32,20 +32,26 @@
32
32
  "build:clean": "rm -rf dist",
33
33
  "build:nextui": "cd src/web/nextui && npm run build && { [ -z \"$NEXT_PUBLIC_PROMPTFOO_BUILD_STANDALONE_SERVER\" ] && cp -r out/ ../../../dist/src/web/nextui; } || true",
34
34
  "build:watch": "tsc --watch",
35
- "build": "tsc && cp src/*.html dist/src && cp src/python/wrapper.py dist/src/python && npm run build:nextui && chmod +x dist/src/main.js",
35
+ "build": "tsc && cp src/*.html dist/src && cp src/python/wrapper.py dist/src/python && cp -r drizzle/ dist/ && npm run build:nextui && chmod +x dist/src/main.js",
36
36
  "prepare": "npm run install:nextui && npm run build:clean && npm run build",
37
37
  "test": "jest",
38
38
  "test:watch": "jest --watch",
39
- "format": "prettier -w ."
39
+ "format": "prettier -w .",
40
+ "db:migrate": "npx tsx src/migrate.ts",
41
+ "db:generate": "npx drizzle-kit generate:sqlite",
42
+ "audit:fix": "npm audit fix && cd src/web/nextui && npm audit fix && cd ../../site && npm audit fix"
40
43
  },
41
44
  "peerDependencies": {
42
45
  "@aws-sdk/client-bedrock-runtime": "^3.458.0",
43
- "@azure/identity": "^4.0.0"
46
+ "@azure/identity": "^4.0.0",
47
+ "@azure/openai-assistants": "^1.0.0-beta.5",
48
+ "google-auth-library": "^9.7.0"
44
49
  },
45
50
  "devDependencies": {
46
51
  "@aws-sdk/client-bedrock-runtime": "^3.458.0",
47
52
  "@azure/identity": "^4.0.0",
48
53
  "@types/async": "^3.2.20",
54
+ "@types/better-sqlite3": "^7.6.8",
49
55
  "@types/cache-manager": "^4.0.2",
50
56
  "@types/cache-manager-fs-hash": "^0.0.1",
51
57
  "@types/cli-progress": "^3.11.0",
@@ -62,6 +68,7 @@
62
68
  "@types/semver": "^7.5.0",
63
69
  "@types/uuid": "^9.0.2",
64
70
  "babel-jest": "^29.5.0",
71
+ "drizzle-kit": "^0.20.13",
65
72
  "jest": "^29.5.0",
66
73
  "jest-watch-typeahead": "^2.2.2",
67
74
  "next": "^13.4.13",
@@ -76,6 +83,7 @@
76
83
  "ajv": "^8.12.0",
77
84
  "ajv-formats": "^2.1.1",
78
85
  "async": "^3.2.4",
86
+ "better-sqlite3": "^9.3.0",
79
87
  "cache-manager": "^4.1.0",
80
88
  "cache-manager-fs-hash": "^1.0.0",
81
89
  "chalk": "^4.1.2",
@@ -88,6 +96,7 @@
88
96
  "csv-parse": "^5.3.8",
89
97
  "csv-stringify": "^6.3.2",
90
98
  "debounce": "^1.2.1",
99
+ "drizzle-orm": "^0.29.3",
91
100
  "express": "^4.18.2",
92
101
  "fastest-levenshtein": "^1.0.16",
93
102
  "glob": "^10.2.6",
@@ -0,0 +1,5 @@
1
+ /// <reference types="jest" />
2
+ declare const mockDbInstance: {};
3
+ declare const mockRelations: jest.Mock<any, any, any>;
4
+ declare const mockSqliteTable: jest.Mock<any, any, any>;
5
+ //# sourceMappingURL=database.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"database.d.ts","sourceRoot":"","sources":["../../../src/__mocks__/database.ts"],"names":[],"mappings":";AAAA,QAAA,MAAM,cAAc,IAInB,CAAC;AAEF,QAAA,MAAM,aAAa,0BAAY,CAAC;AAEhC,QAAA,MAAM,eAAe,0BAGnB,CAAC"}
@@ -0,0 +1,27 @@
1
+ "use strict";
2
+ const mockDbInstance = {
3
+ // Mock any method you use from the dbInstance
4
+ // For example:
5
+ // query: jest.fn().mockResolvedValue({}),
6
+ };
7
+ const mockRelations = jest.fn();
8
+ const mockSqliteTable = jest.fn().mockImplementation((tableName, schema) => {
9
+ // You can customize this mock based on your testing needs
10
+ return { tableName, schema };
11
+ });
12
+ module.exports = {
13
+ prompts: mockSqliteTable('prompts', { /* schema definition */}),
14
+ promptsRelations: mockRelations,
15
+ datasets: mockSqliteTable('datasets', { /* schema definition */}),
16
+ datasetsRelations: mockRelations,
17
+ evals: mockSqliteTable('evals', { /* schema definition */}),
18
+ evalsRelations: mockRelations,
19
+ evalsToPrompts: mockSqliteTable('evals_to_prompts', { /* schema definition */}),
20
+ evalsToPromptsRelations: mockRelations,
21
+ evalsToDatasets: mockSqliteTable('evals_to_datasets', { /* schema definition */}),
22
+ evalsToDatasetsRelations: mockRelations,
23
+ llmOutputs: mockSqliteTable('llm_outputs', { /* schema definition */}),
24
+ llmOutputsRelations: mockRelations,
25
+ getDb: jest.fn(() => mockDbInstance),
26
+ };
27
+ //# sourceMappingURL=database.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"database.js","sourceRoot":"","sources":["../../../src/__mocks__/database.ts"],"names":[],"mappings":";AAAA,MAAM,cAAc,GAAG;AACrB,8CAA8C;AAC9C,eAAe;AACf,0CAA0C;CAC3C,CAAC;AAEF,MAAM,aAAa,GAAG,IAAI,CAAC,EAAE,EAAE,CAAC;AAEhC,MAAM,eAAe,GAAG,IAAI,CAAC,EAAE,EAAE,CAAC,kBAAkB,CAAC,CAAC,SAAS,EAAE,MAAM,EAAE,EAAE;IACzE,0DAA0D;IAC1D,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC;AAC/B,CAAC,CAAC,CAAC;AAEH,MAAM,CAAC,OAAO,GAAG;IACf,OAAO,EAAE,eAAe,CAAC,SAAS,EAAE,EAAC,uBAAuB,CAAC,CAAC;IAC9D,gBAAgB,EAAE,aAAa;IAC/B,QAAQ,EAAE,eAAe,CAAC,UAAU,EAAE,EAAC,uBAAuB,CAAC,CAAC;IAChE,iBAAiB,EAAE,aAAa;IAChC,KAAK,EAAE,eAAe,CAAC,OAAO,EAAE,EAAC,uBAAuB,CAAC,CAAC;IAC1D,cAAc,EAAE,aAAa;IAC7B,cAAc,EAAE,eAAe,CAAC,kBAAkB,EAAE,EAAC,uBAAuB,CAAC,CAAC;IAC9E,uBAAuB,EAAE,aAAa;IACtC,eAAe,EAAE,eAAe,CAAC,mBAAmB,EAAE,EAAC,uBAAuB,CAAC,CAAC;IAChF,wBAAwB,EAAE,aAAa;IACvC,UAAU,EAAE,eAAe,CAAC,aAAa,EAAE,EAAC,uBAAuB,CAAC,CAAC;IACrE,mBAAmB,EAAE,aAAa;IAClC,KAAK,EAAE,IAAI,CAAC,EAAE,CAAC,GAAG,EAAE,CAAC,cAAc,CAAC;CACrC,CAAC"}
@@ -1 +1 @@
1
- {"version":3,"file":"assertions.d.ts","sourceRoot":"","sources":["../../src/assertions.ts"],"names":[],"mappings":"AAeA,OAAO,EACL,iBAAiB,EACjB,gBAAgB,EAChB,iBAAiB,EACjB,eAAe,EACf,qBAAqB,EACrB,sBAAsB,EACtB,oBAAoB,EACpB,uBAAuB,EACvB,0BAA0B,EAC1B,iBAAiB,EAClB,MAAM,YAAY,CAAC;AAKpB,OAAO,KAAK,EAAE,SAAS,EAAE,aAAa,EAAE,aAAa,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AAGpG,eAAO,MAAM,4BAA4B,oBASvC,CAAC;AAwCH,wBAAsB,aAAa,CAAC,EAClC,MAAM,EACN,QAAQ,EACR,IAAI,EACJ,MAAM,EACN,SAAS,EACT,QAAQ,EACR,IAAI,GACL,EAAE;IACD,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,WAAW,CAAC;IACvB,IAAI,EAAE,cAAc,CAAC;IACrB,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC;IACxB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;IACpB,IAAI,CAAC,EAAE,MAAM,CAAC;CACf,GAAG,OAAO,CAAC,aAAa,CAAC,CA8EzB;AAED,wBAAsB,YAAY,CAAC,EACjC,MAAM,EACN,QAAQ,EACR,SAAS,EACT,IAAI,EACJ,MAAM,EACN,SAAS,EACT,QAAQ,EACR,IAAI,GACL,EAAE;IACD,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,WAAW,CAAC;IACvB,SAAS,EAAE,SAAS,CAAC;IACrB,IAAI,EAAE,cAAc,CAAC;IACrB,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC;IACxB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;IACpB,IAAI,CAAC,EAAE,MAAM,CAAC;CACf,GAAG,OAAO,CAAC,aAAa,CAAC,CAu7BzB;AAgCD,wBAAsB,mBAAmB,CACvC,IAAI,EAAE,cAAc,EACpB,SAAS,EAAE,SAAS,EACpB,OAAO,EAAE,MAAM,EAAE,GAChB,OAAO,CAAC,aAAa,EAAE,CAAC,CAe1B;AAED,wBAAsB,cAAc,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC,CAU3E;;;;;;;;;;;;;AAGD,wBAWE"}
1
+ {"version":3,"file":"assertions.d.ts","sourceRoot":"","sources":["../../src/assertions.ts"],"names":[],"mappings":"AAgBA,OAAO,EACL,iBAAiB,EACjB,gBAAgB,EAChB,iBAAiB,EACjB,eAAe,EACf,qBAAqB,EACrB,sBAAsB,EACtB,oBAAoB,EACpB,uBAAuB,EACvB,0BAA0B,EAC1B,iBAAiB,EAClB,MAAM,YAAY,CAAC;AAKpB,OAAO,KAAK,EAAE,SAAS,EAAE,aAAa,EAAE,aAAa,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AAOpG,eAAO,MAAM,4BAA4B,oBASvC,CAAC;AAwCH,wBAAsB,aAAa,CAAC,EAClC,MAAM,EACN,QAAQ,EACR,IAAI,EACJ,MAAM,EACN,SAAS,EACT,QAAQ,EACR,IAAI,GACL,EAAE;IACD,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,WAAW,CAAC;IACvB,IAAI,EAAE,cAAc,CAAC;IACrB,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC;IACxB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;IACpB,IAAI,CAAC,EAAE,MAAM,CAAC;CACf,GAAG,OAAO,CAAC,aAAa,CAAC,CAwEzB;AAED,wBAAsB,YAAY,CAAC,EACjC,MAAM,EACN,QAAQ,EACR,SAAS,EACT,IAAI,EACJ,MAAM,EACN,SAAS,EACT,QAAQ,EACR,IAAI,GACL,EAAE;IACD,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,WAAW,CAAC;IACvB,SAAS,EAAE,SAAS,CAAC;IACrB,IAAI,EAAE,cAAc,CAAC;IACrB,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC;IACxB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;IACpB,IAAI,CAAC,EAAE,MAAM,CAAC;CACf,GAAG,OAAO,CAAC,aAAa,CAAC,CA87BzB;AAgCD,wBAAsB,mBAAmB,CACvC,IAAI,EAAE,cAAc,EACpB,SAAS,EAAE,SAAS,EACpB,OAAO,EAAE,MAAM,EAAE,GAChB,OAAO,CAAC,aAAa,EAAE,CAAC,CAe1B;AAED,wBAAsB,cAAc,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC,CAU3E;;;;;;;;;;;;;AAGD,wBAWE"}