promptfoo 0.91.3 → 0.92.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/drizzle/0006_harsh_caretaker.sql +42 -0
- package/dist/drizzle/0007_cloudy_wong.sql +1 -0
- package/dist/drizzle/meta/0006_snapshot.json +721 -0
- package/dist/drizzle/meta/0007_snapshot.json +723 -0
- package/dist/drizzle/meta/_journal.json +14 -0
- package/dist/package.json +10 -8
- package/dist/src/app/assets/{index-C6z1nbLN.js → index-BpjzEMiv.js} +243 -241
- package/dist/src/app/assets/{index.es-oqbvfIxR.js → index.es-ihzvEu35.js} +1 -1
- package/dist/src/app/assets/{sync-D2s75VlC.js → sync-BosjlpGJ.js} +1 -1
- package/dist/src/app/index.html +3 -3
- package/dist/src/assertions.js +2 -2
- package/dist/src/assertions.js.map +1 -1
- package/dist/src/commands/cache.d.ts.map +1 -1
- package/dist/src/commands/cache.js +0 -2
- package/dist/src/commands/cache.js.map +1 -1
- package/dist/src/commands/eval.d.ts.map +1 -1
- package/dist/src/commands/eval.js +19 -16
- package/dist/src/commands/eval.js.map +1 -1
- package/dist/src/commands/export.d.ts.map +1 -1
- package/dist/src/commands/export.js +8 -31
- package/dist/src/commands/export.js.map +1 -1
- package/dist/src/commands/import.d.ts.map +1 -1
- package/dist/src/commands/import.js +52 -13
- package/dist/src/commands/import.js.map +1 -1
- package/dist/src/commands/list.d.ts.map +1 -1
- package/dist/src/commands/list.js +35 -7
- package/dist/src/commands/list.js.map +1 -1
- package/dist/src/commands/share.d.ts +2 -2
- package/dist/src/commands/share.d.ts.map +1 -1
- package/dist/src/commands/share.js +12 -13
- package/dist/src/commands/share.js.map +1 -1
- package/dist/src/commands/show.d.ts.map +1 -1
- package/dist/src/commands/show.js +10 -6
- package/dist/src/commands/show.js.map +1 -1
- package/dist/src/constants.d.ts +1 -0
- package/dist/src/constants.d.ts.map +1 -1
- package/dist/src/constants.js +2 -1
- package/dist/src/constants.js.map +1 -1
- package/dist/src/database/index.js +1 -1
- package/dist/src/database/index.js.map +1 -1
- package/dist/src/database/tables.d.ts +609 -11
- package/dist/src/database/tables.d.ts.map +1 -1
- package/dist/src/database/tables.js +111 -52
- package/dist/src/database/tables.js.map +1 -1
- package/dist/src/database/types.d.ts +3 -3
- package/dist/src/database/types.d.ts.map +1 -1
- package/dist/src/evaluator.d.ts +3 -2
- package/dist/src/evaluator.d.ts.map +1 -1
- package/dist/src/evaluator.js +75 -104
- package/dist/src/evaluator.js.map +1 -1
- package/dist/src/evaluatorHelpers.d.ts.map +1 -1
- package/dist/src/evaluatorHelpers.js +2 -1
- package/dist/src/evaluatorHelpers.js.map +1 -1
- package/dist/src/index.d.ts +2 -1
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +18 -10
- package/dist/src/index.js.map +1 -1
- package/dist/src/models/eval.d.ts +95 -0
- package/dist/src/models/eval.d.ts.map +1 -0
- package/dist/src/models/eval.js +390 -0
- package/dist/src/models/eval.js.map +1 -0
- package/dist/src/models/evalResult.d.ts +50 -0
- package/dist/src/models/evalResult.d.ts.map +1 -0
- package/dist/src/models/evalResult.js +122 -0
- package/dist/src/models/evalResult.js.map +1 -0
- package/dist/src/models/provider.d.ts +9 -0
- package/dist/src/models/provider.d.ts.map +1 -0
- package/dist/src/models/provider.js +47 -0
- package/dist/src/models/provider.js.map +1 -0
- package/dist/src/prompts/index.d.ts.map +1 -1
- package/dist/src/prompts/index.js +2 -1
- package/dist/src/prompts/index.js.map +1 -1
- package/dist/src/prompts/utils.d.ts +1 -0
- package/dist/src/prompts/utils.d.ts.map +1 -1
- package/dist/src/prompts/utils.js +7 -0
- package/dist/src/prompts/utils.js.map +1 -1
- package/dist/src/providers/fal.d.ts +2 -2
- package/dist/src/providers/fal.d.ts.map +1 -1
- package/dist/src/providers/fal.js +2 -1
- package/dist/src/providers/fal.js.map +1 -1
- package/dist/src/providers/http.js +2 -2
- package/dist/src/providers/http.js.map +1 -1
- package/dist/src/providers/palm.d.ts +4 -3
- package/dist/src/providers/palm.d.ts.map +1 -1
- package/dist/src/providers/palm.js +13 -3
- package/dist/src/providers/palm.js.map +1 -1
- package/dist/src/providers.js +5 -5
- package/dist/src/providers.js.map +1 -1
- package/dist/src/redteam/eval/excessive-agency/llm_rubric-20240617.json +10 -0
- package/dist/src/redteam/eval/excessive-agency/llm_rubric-20240618.json +10 -0
- package/dist/src/redteam/eval/harmful/llm_rubric-20240723.json +10 -0
- package/dist/src/redteam/eval/harmful/llm_rubric-20240724.json +10 -0
- package/dist/src/server/server.d.ts +1 -0
- package/dist/src/server/server.d.ts.map +1 -1
- package/dist/src/server/server.js +70 -31
- package/dist/src/server/server.js.map +1 -1
- package/dist/src/share.d.ts +2 -2
- package/dist/src/share.d.ts.map +1 -1
- package/dist/src/share.js +93 -34
- package/dist/src/share.js.map +1 -1
- package/dist/src/table.d.ts +2 -2
- package/dist/src/table.d.ts.map +1 -1
- package/dist/src/table.js +3 -3
- package/dist/src/table.js.map +1 -1
- package/dist/src/types/index.d.ts +163 -11
- package/dist/src/types/index.d.ts.map +1 -1
- package/dist/src/types/index.js +21 -1
- package/dist/src/types/index.js.map +1 -1
- package/dist/src/util/config/load.d.ts.map +1 -1
- package/dist/src/util/config/load.js +2 -1
- package/dist/src/util/config/load.js.map +1 -1
- package/dist/src/util/config/manage.d.ts.map +1 -1
- package/dist/src/util/config/manage.js.map +1 -1
- package/dist/src/util/convertEvalResultsToTable.d.ts +16 -0
- package/dist/src/util/convertEvalResultsToTable.d.ts.map +1 -0
- package/dist/src/util/convertEvalResultsToTable.js +136 -0
- package/dist/src/util/convertEvalResultsToTable.js.map +1 -0
- package/dist/src/util/createHash.d.ts +1 -0
- package/dist/src/util/createHash.d.ts.map +1 -1
- package/dist/src/util/createHash.js +9 -0
- package/dist/src/util/createHash.js.map +1 -1
- package/dist/src/util/file.d.ts +8 -0
- package/dist/src/util/file.d.ts.map +1 -0
- package/dist/src/util/file.js +13 -0
- package/dist/src/util/file.js.map +1 -0
- package/dist/src/util/index.d.ts +9 -14
- package/dist/src/util/index.d.ts.map +1 -1
- package/dist/src/util/index.js +132 -268
- package/dist/src/util/index.js.map +1 -1
- package/dist/src/util/time.d.ts +2 -0
- package/dist/src/util/time.d.ts.map +1 -0
- package/dist/src/util/time.js +7 -0
- package/dist/src/util/time.js.map +1 -0
- package/dist/src/util/transform.js +2 -2
- package/dist/src/util/transform.js.map +1 -1
- package/dist/src/validators/providers.d.ts +6 -0
- package/dist/src/validators/providers.d.ts.map +1 -1
- package/dist/src/validators/providers.js +1 -0
- package/dist/src/validators/providers.js.map +1 -1
- package/dist/src/validators/redteam.d.ts +6 -0
- package/dist/src/validators/redteam.d.ts.map +1 -1
- package/dist/test/commands/eval/filterFailingTests.test.js +24 -2
- package/dist/test/commands/eval/filterFailingTests.test.js.map +1 -1
- package/dist/test/evaluator.test.js +152 -74
- package/dist/test/evaluator.test.js.map +1 -1
- package/dist/test/factories/data/eval/database_records.d.ts +142 -0
- package/dist/test/factories/data/eval/database_records.d.ts.map +1 -0
- package/dist/test/factories/data/eval/database_records.js +251 -0
- package/dist/test/factories/data/eval/database_records.js.map +1 -0
- package/dist/test/factories/evalFactory.d.ts +768 -0
- package/dist/test/factories/evalFactory.d.ts.map +1 -0
- package/dist/test/factories/evalFactory.js +121 -0
- package/dist/test/factories/evalFactory.js.map +1 -0
- package/dist/test/index.test.js +20 -35
- package/dist/test/index.test.js.map +1 -1
- package/dist/test/models/eval.test.d.ts +2 -0
- package/dist/test/models/eval.test.d.ts.map +1 -0
- package/dist/test/models/eval.test.js +34 -0
- package/dist/test/models/eval.test.js.map +1 -0
- package/dist/test/providers.test.js +3 -3
- package/dist/test/providers.test.js.map +1 -1
- package/dist/test/server/share.test.d.ts +2 -0
- package/dist/test/server/share.test.d.ts.map +1 -0
- package/dist/test/server/share.test.js +36 -0
- package/dist/test/server/share.test.js.map +1 -0
- package/dist/test/server/v3evalToShare.json +507 -0
- package/dist/test/server/v4evalToShare.json +421 -0
- package/dist/test/types.test.js +56 -3
- package/dist/test/types.test.js.map +1 -1
- package/dist/test/util.file.test.d.ts +2 -0
- package/dist/test/util.file.test.d.ts.map +1 -0
- package/dist/test/util.file.test.js +32 -0
- package/dist/test/util.file.test.js.map +1 -0
- package/dist/test/util.listPrevious.test.d.ts +2 -0
- package/dist/test/util.listPrevious.test.d.ts.map +1 -0
- package/dist/test/util.listPrevious.test.js +37 -0
- package/dist/test/util.listPrevious.test.js.map +1 -0
- package/dist/test/util.test.js +38 -311
- package/dist/test/util.test.js.map +1 -1
- package/dist/tsconfig.tsbuildinfo +1 -0
- package/package.json +10 -8
|
@@ -0,0 +1,421 @@
|
|
|
1
|
+
{
|
|
2
|
+
"createdAt": 1727905040496,
|
|
3
|
+
"id": "eval-2024-10-02T21:37:20-kMQ",
|
|
4
|
+
"config": {
|
|
5
|
+
"tags": {},
|
|
6
|
+
"description": "A translator built with LLM",
|
|
7
|
+
"prompts": ["file:///Users/steve/src/promptfoo/examples/simple-csv/prompts.txt"],
|
|
8
|
+
"providers": ["openai:gpt-4o-mini"],
|
|
9
|
+
"tests": [
|
|
10
|
+
{
|
|
11
|
+
"vars": { "language": "French", "body": "Hello world" },
|
|
12
|
+
"assert": [],
|
|
13
|
+
"options": {},
|
|
14
|
+
"description": "Row #1"
|
|
15
|
+
},
|
|
16
|
+
{
|
|
17
|
+
"vars": { "language": "French", "body": "I'm hungry" },
|
|
18
|
+
"assert": [],
|
|
19
|
+
"options": {},
|
|
20
|
+
"description": "Row #2"
|
|
21
|
+
},
|
|
22
|
+
{
|
|
23
|
+
"vars": { "language": "Pirate", "body": "Hello world" },
|
|
24
|
+
"assert": [],
|
|
25
|
+
"options": {},
|
|
26
|
+
"description": "Row #3"
|
|
27
|
+
},
|
|
28
|
+
{
|
|
29
|
+
"vars": { "language": "Pirate", "body": "I'm hungry" },
|
|
30
|
+
"assert": [],
|
|
31
|
+
"options": {},
|
|
32
|
+
"description": "Row #4"
|
|
33
|
+
}
|
|
34
|
+
],
|
|
35
|
+
"scenarios": [],
|
|
36
|
+
"env": {},
|
|
37
|
+
"sharing": true,
|
|
38
|
+
"defaultTest": { "vars": {}, "assert": [], "options": {}, "metadata": {} },
|
|
39
|
+
"outputPath": [],
|
|
40
|
+
"extensions": [],
|
|
41
|
+
"metadata": {},
|
|
42
|
+
"redteam": { "plugins": [], "strategies": [] }
|
|
43
|
+
},
|
|
44
|
+
"results": [
|
|
45
|
+
{
|
|
46
|
+
"id": "14c141a7-be6f-460f-b3cf-711e0a64fe88",
|
|
47
|
+
"evalId": "eval-2024-10-02T21:37:20-kMQ",
|
|
48
|
+
"promptIdx": 0,
|
|
49
|
+
"testIdx": 0,
|
|
50
|
+
"testCase": {
|
|
51
|
+
"vars": { "language": "French", "body": "Hello world" },
|
|
52
|
+
"assert": [],
|
|
53
|
+
"options": {},
|
|
54
|
+
"description": "Row #1",
|
|
55
|
+
"metadata": {}
|
|
56
|
+
},
|
|
57
|
+
"prompt": {
|
|
58
|
+
"raw": "Rephrase this in French: Hello world",
|
|
59
|
+
"label": "examples/simple-csv/prompts.txt: Rephrase this in {{language}}: {{body}}"
|
|
60
|
+
},
|
|
61
|
+
"promptId": "ad447337160016cbd0a5c7f2626d0c1fe2ade0d4d653e20bed9250632bc56e03",
|
|
62
|
+
"error": null,
|
|
63
|
+
"score": 1,
|
|
64
|
+
"success": true,
|
|
65
|
+
"providerResponse": {
|
|
66
|
+
"output": "Bonjour le monde.",
|
|
67
|
+
"tokenUsage": { "cached": 19, "total": 19 },
|
|
68
|
+
"cached": true,
|
|
69
|
+
"cost": 0.0000046499999999999995
|
|
70
|
+
},
|
|
71
|
+
"gradingResult": {
|
|
72
|
+
"pass": true,
|
|
73
|
+
"score": 1,
|
|
74
|
+
"reason": "No assertions",
|
|
75
|
+
"tokensUsed": { "total": 0, "prompt": 0, "completion": 0, "cached": 0 },
|
|
76
|
+
"assertion": null
|
|
77
|
+
},
|
|
78
|
+
"namedScores": {},
|
|
79
|
+
"provider": {
|
|
80
|
+
"id": "openai:gpt-4o-mini",
|
|
81
|
+
"label": "",
|
|
82
|
+
"config": { "basePath": "examples/simple-csv" }
|
|
83
|
+
},
|
|
84
|
+
"latencyMs": 3,
|
|
85
|
+
"cost": 0.0000046499999999999995
|
|
86
|
+
},
|
|
87
|
+
{
|
|
88
|
+
"id": "013a4746-4051-4dc5-87d1-1c8414bc124f",
|
|
89
|
+
"evalId": "eval-2024-10-02T21:37:20-kMQ",
|
|
90
|
+
"promptIdx": 1,
|
|
91
|
+
"testIdx": 0,
|
|
92
|
+
"testCase": {
|
|
93
|
+
"vars": { "language": "French", "body": "Hello world" },
|
|
94
|
+
"assert": [],
|
|
95
|
+
"options": {},
|
|
96
|
+
"description": "Row #1",
|
|
97
|
+
"metadata": {}
|
|
98
|
+
},
|
|
99
|
+
"prompt": {
|
|
100
|
+
"raw": "Translate this to conversational French: Hello world",
|
|
101
|
+
"label": "examples/simple-csv/prompts.txt: Translate this to conversational {{language}}: {{body}}"
|
|
102
|
+
},
|
|
103
|
+
"promptId": "0ceccef5d40adcbf45bda7be485d2ee420e4d5943975932e7a38c5468d97c6ee",
|
|
104
|
+
"error": null,
|
|
105
|
+
"score": 1,
|
|
106
|
+
"success": true,
|
|
107
|
+
"providerResponse": {
|
|
108
|
+
"output": "Salut tout le monde !",
|
|
109
|
+
"tokenUsage": { "cached": 20, "total": 20 },
|
|
110
|
+
"cached": true,
|
|
111
|
+
"cost": 0.0000052500000000000006
|
|
112
|
+
},
|
|
113
|
+
"gradingResult": {
|
|
114
|
+
"pass": true,
|
|
115
|
+
"score": 1,
|
|
116
|
+
"reason": "No assertions",
|
|
117
|
+
"tokensUsed": { "total": 0, "prompt": 0, "completion": 0, "cached": 0 },
|
|
118
|
+
"assertion": null
|
|
119
|
+
},
|
|
120
|
+
"namedScores": {},
|
|
121
|
+
"provider": {
|
|
122
|
+
"id": "openai:gpt-4o-mini",
|
|
123
|
+
"label": "",
|
|
124
|
+
"config": { "basePath": "examples/simple-csv" }
|
|
125
|
+
},
|
|
126
|
+
"latencyMs": 7,
|
|
127
|
+
"cost": 0.0000052500000000000006
|
|
128
|
+
},
|
|
129
|
+
{
|
|
130
|
+
"id": "7791f530-38e0-431a-b8ef-fc2a64b14cb5",
|
|
131
|
+
"evalId": "eval-2024-10-02T21:37:20-kMQ",
|
|
132
|
+
"promptIdx": 1,
|
|
133
|
+
"testIdx": 1,
|
|
134
|
+
"testCase": {
|
|
135
|
+
"vars": { "language": "French", "body": "I'm hungry" },
|
|
136
|
+
"assert": [],
|
|
137
|
+
"options": {},
|
|
138
|
+
"description": "Row #2",
|
|
139
|
+
"metadata": {}
|
|
140
|
+
},
|
|
141
|
+
"prompt": {
|
|
142
|
+
"raw": "Translate this to conversational French: I'm hungry",
|
|
143
|
+
"label": "examples/simple-csv/prompts.txt: Translate this to conversational {{language}}: {{body}}"
|
|
144
|
+
},
|
|
145
|
+
"promptId": "0ceccef5d40adcbf45bda7be485d2ee420e4d5943975932e7a38c5468d97c6ee",
|
|
146
|
+
"error": null,
|
|
147
|
+
"score": 1,
|
|
148
|
+
"success": true,
|
|
149
|
+
"providerResponse": {
|
|
150
|
+
"output": "J'ai faim.",
|
|
151
|
+
"tokenUsage": { "cached": 19, "total": 19 },
|
|
152
|
+
"cached": true,
|
|
153
|
+
"cost": 0.0000046499999999999995
|
|
154
|
+
},
|
|
155
|
+
"gradingResult": {
|
|
156
|
+
"pass": true,
|
|
157
|
+
"score": 1,
|
|
158
|
+
"reason": "No assertions",
|
|
159
|
+
"tokensUsed": { "total": 0, "prompt": 0, "completion": 0, "cached": 0 },
|
|
160
|
+
"assertion": null
|
|
161
|
+
},
|
|
162
|
+
"namedScores": {},
|
|
163
|
+
"provider": {
|
|
164
|
+
"id": "openai:gpt-4o-mini",
|
|
165
|
+
"label": "",
|
|
166
|
+
"config": { "basePath": "examples/simple-csv" }
|
|
167
|
+
},
|
|
168
|
+
"latencyMs": 8,
|
|
169
|
+
"cost": 0.0000046499999999999995
|
|
170
|
+
},
|
|
171
|
+
{
|
|
172
|
+
"id": "1720b007-3e86-4920-a738-95fb0dcf2a3f",
|
|
173
|
+
"evalId": "eval-2024-10-02T21:37:20-kMQ",
|
|
174
|
+
"promptIdx": 0,
|
|
175
|
+
"testIdx": 1,
|
|
176
|
+
"testCase": {
|
|
177
|
+
"vars": { "language": "French", "body": "I'm hungry" },
|
|
178
|
+
"assert": [],
|
|
179
|
+
"options": {},
|
|
180
|
+
"description": "Row #2",
|
|
181
|
+
"metadata": {}
|
|
182
|
+
},
|
|
183
|
+
"prompt": {
|
|
184
|
+
"raw": "Rephrase this in French: I'm hungry",
|
|
185
|
+
"label": "examples/simple-csv/prompts.txt: Rephrase this in {{language}}: {{body}}"
|
|
186
|
+
},
|
|
187
|
+
"promptId": "ad447337160016cbd0a5c7f2626d0c1fe2ade0d4d653e20bed9250632bc56e03",
|
|
188
|
+
"error": null,
|
|
189
|
+
"score": 1,
|
|
190
|
+
"success": true,
|
|
191
|
+
"providerResponse": {
|
|
192
|
+
"output": "J'ai faim.",
|
|
193
|
+
"tokenUsage": { "cached": 19, "total": 19 },
|
|
194
|
+
"cached": true,
|
|
195
|
+
"cost": 0.0000046499999999999995
|
|
196
|
+
},
|
|
197
|
+
"gradingResult": {
|
|
198
|
+
"pass": true,
|
|
199
|
+
"score": 1,
|
|
200
|
+
"reason": "No assertions",
|
|
201
|
+
"tokensUsed": { "total": 0, "prompt": 0, "completion": 0, "cached": 0 },
|
|
202
|
+
"assertion": null
|
|
203
|
+
},
|
|
204
|
+
"namedScores": {},
|
|
205
|
+
"provider": {
|
|
206
|
+
"id": "openai:gpt-4o-mini",
|
|
207
|
+
"label": "",
|
|
208
|
+
"config": { "basePath": "examples/simple-csv" }
|
|
209
|
+
},
|
|
210
|
+
"latencyMs": 10,
|
|
211
|
+
"cost": 0.0000046499999999999995
|
|
212
|
+
},
|
|
213
|
+
{
|
|
214
|
+
"id": "4540d2da-9a5a-402e-9243-41be20f63b1f",
|
|
215
|
+
"evalId": "eval-2024-10-02T21:37:20-kMQ",
|
|
216
|
+
"promptIdx": 0,
|
|
217
|
+
"testIdx": 2,
|
|
218
|
+
"testCase": {
|
|
219
|
+
"vars": { "language": "Pirate", "body": "Hello world" },
|
|
220
|
+
"assert": [],
|
|
221
|
+
"options": {},
|
|
222
|
+
"description": "Row #3",
|
|
223
|
+
"metadata": {}
|
|
224
|
+
},
|
|
225
|
+
"prompt": {
|
|
226
|
+
"raw": "Rephrase this in Pirate: Hello world",
|
|
227
|
+
"label": "examples/simple-csv/prompts.txt: Rephrase this in {{language}}: {{body}}"
|
|
228
|
+
},
|
|
229
|
+
"promptId": "ad447337160016cbd0a5c7f2626d0c1fe2ade0d4d653e20bed9250632bc56e03",
|
|
230
|
+
"error": null,
|
|
231
|
+
"score": 1,
|
|
232
|
+
"success": true,
|
|
233
|
+
"providerResponse": {
|
|
234
|
+
"output": "Ahoy, me hearties!",
|
|
235
|
+
"tokenUsage": { "cached": 22, "total": 22 },
|
|
236
|
+
"cached": true,
|
|
237
|
+
"cost": 0.000006449999999999999
|
|
238
|
+
},
|
|
239
|
+
"gradingResult": {
|
|
240
|
+
"pass": true,
|
|
241
|
+
"score": 1,
|
|
242
|
+
"reason": "No assertions",
|
|
243
|
+
"tokensUsed": { "total": 0, "prompt": 0, "completion": 0, "cached": 0 },
|
|
244
|
+
"assertion": null
|
|
245
|
+
},
|
|
246
|
+
"namedScores": {},
|
|
247
|
+
"provider": {
|
|
248
|
+
"id": "openai:gpt-4o-mini",
|
|
249
|
+
"label": "",
|
|
250
|
+
"config": { "basePath": "examples/simple-csv" }
|
|
251
|
+
},
|
|
252
|
+
"latencyMs": 5,
|
|
253
|
+
"cost": 0.000006449999999999999
|
|
254
|
+
},
|
|
255
|
+
{
|
|
256
|
+
"id": "467d9521-22f8-45ae-a3c2-0777dd7b1bb0",
|
|
257
|
+
"evalId": "eval-2024-10-02T21:37:20-kMQ",
|
|
258
|
+
"promptIdx": 1,
|
|
259
|
+
"testIdx": 2,
|
|
260
|
+
"testCase": {
|
|
261
|
+
"vars": { "language": "Pirate", "body": "Hello world" },
|
|
262
|
+
"assert": [],
|
|
263
|
+
"options": {},
|
|
264
|
+
"description": "Row #3",
|
|
265
|
+
"metadata": {}
|
|
266
|
+
},
|
|
267
|
+
"prompt": {
|
|
268
|
+
"raw": "Translate this to conversational Pirate: Hello world",
|
|
269
|
+
"label": "examples/simple-csv/prompts.txt: Translate this to conversational {{language}}: {{body}}"
|
|
270
|
+
},
|
|
271
|
+
"promptId": "0ceccef5d40adcbf45bda7be485d2ee420e4d5943975932e7a38c5468d97c6ee",
|
|
272
|
+
"error": null,
|
|
273
|
+
"score": 1,
|
|
274
|
+
"success": true,
|
|
275
|
+
"providerResponse": {
|
|
276
|
+
"output": "Ahoy, me hearty! Greetings to ye, world!",
|
|
277
|
+
"tokenUsage": { "cached": 27, "total": 27 },
|
|
278
|
+
"cached": true,
|
|
279
|
+
"cost": 0.00000945
|
|
280
|
+
},
|
|
281
|
+
"gradingResult": {
|
|
282
|
+
"pass": true,
|
|
283
|
+
"score": 1,
|
|
284
|
+
"reason": "No assertions",
|
|
285
|
+
"tokensUsed": { "total": 0, "prompt": 0, "completion": 0, "cached": 0 },
|
|
286
|
+
"assertion": null
|
|
287
|
+
},
|
|
288
|
+
"namedScores": {},
|
|
289
|
+
"provider": {
|
|
290
|
+
"id": "openai:gpt-4o-mini",
|
|
291
|
+
"label": "",
|
|
292
|
+
"config": { "basePath": "examples/simple-csv" }
|
|
293
|
+
},
|
|
294
|
+
"latencyMs": 5,
|
|
295
|
+
"cost": 0.00000945
|
|
296
|
+
},
|
|
297
|
+
{
|
|
298
|
+
"id": "70b1ebd1-f748-4969-9d38-b6460ff6107f",
|
|
299
|
+
"evalId": "eval-2024-10-02T21:37:20-kMQ",
|
|
300
|
+
"promptIdx": 0,
|
|
301
|
+
"testIdx": 3,
|
|
302
|
+
"testCase": {
|
|
303
|
+
"vars": { "language": "Pirate", "body": "I'm hungry" },
|
|
304
|
+
"assert": [],
|
|
305
|
+
"options": {},
|
|
306
|
+
"description": "Row #4",
|
|
307
|
+
"metadata": {}
|
|
308
|
+
},
|
|
309
|
+
"prompt": {
|
|
310
|
+
"raw": "Rephrase this in Pirate: I'm hungry",
|
|
311
|
+
"label": "examples/simple-csv/prompts.txt: Rephrase this in {{language}}: {{body}}"
|
|
312
|
+
},
|
|
313
|
+
"promptId": "ad447337160016cbd0a5c7f2626d0c1fe2ade0d4d653e20bed9250632bc56e03",
|
|
314
|
+
"error": null,
|
|
315
|
+
"score": 1,
|
|
316
|
+
"success": true,
|
|
317
|
+
"providerResponse": {
|
|
318
|
+
"output": "Arrr, me belly be grumblin'!",
|
|
319
|
+
"tokenUsage": { "cached": 27, "total": 27 },
|
|
320
|
+
"cached": true,
|
|
321
|
+
"cost": 0.00000945
|
|
322
|
+
},
|
|
323
|
+
"gradingResult": {
|
|
324
|
+
"pass": true,
|
|
325
|
+
"score": 1,
|
|
326
|
+
"reason": "No assertions",
|
|
327
|
+
"tokensUsed": { "total": 0, "prompt": 0, "completion": 0, "cached": 0 },
|
|
328
|
+
"assertion": null
|
|
329
|
+
},
|
|
330
|
+
"namedScores": {},
|
|
331
|
+
"provider": {
|
|
332
|
+
"id": "openai:gpt-4o-mini",
|
|
333
|
+
"label": "",
|
|
334
|
+
"config": { "basePath": "examples/simple-csv" }
|
|
335
|
+
},
|
|
336
|
+
"latencyMs": 3,
|
|
337
|
+
"cost": 0.00000945
|
|
338
|
+
},
|
|
339
|
+
{
|
|
340
|
+
"id": "5b65ede9-f787-4637-b18d-9e0ff69dd778",
|
|
341
|
+
"evalId": "eval-2024-10-02T21:37:20-kMQ",
|
|
342
|
+
"promptIdx": 1,
|
|
343
|
+
"testIdx": 3,
|
|
344
|
+
"testCase": {
|
|
345
|
+
"vars": { "language": "Pirate", "body": "I'm hungry" },
|
|
346
|
+
"assert": [],
|
|
347
|
+
"options": {},
|
|
348
|
+
"description": "Row #4",
|
|
349
|
+
"metadata": {}
|
|
350
|
+
},
|
|
351
|
+
"prompt": {
|
|
352
|
+
"raw": "Translate this to conversational Pirate: I'm hungry",
|
|
353
|
+
"label": "examples/simple-csv/prompts.txt: Translate this to conversational {{language}}: {{body}}"
|
|
354
|
+
},
|
|
355
|
+
"promptId": "0ceccef5d40adcbf45bda7be485d2ee420e4d5943975932e7a38c5468d97c6ee",
|
|
356
|
+
"error": null,
|
|
357
|
+
"score": 1,
|
|
358
|
+
"success": true,
|
|
359
|
+
"providerResponse": {
|
|
360
|
+
"output": "Arrr, me belly be rumblin'! I be feelin' mighty hungry, matey!",
|
|
361
|
+
"tokenUsage": { "cached": 37, "total": 37 },
|
|
362
|
+
"cached": true,
|
|
363
|
+
"cost": 0.00001545
|
|
364
|
+
},
|
|
365
|
+
"gradingResult": {
|
|
366
|
+
"pass": true,
|
|
367
|
+
"score": 1,
|
|
368
|
+
"reason": "No assertions",
|
|
369
|
+
"tokensUsed": { "total": 0, "prompt": 0, "completion": 0, "cached": 0 },
|
|
370
|
+
"assertion": null
|
|
371
|
+
},
|
|
372
|
+
"namedScores": {},
|
|
373
|
+
"provider": {
|
|
374
|
+
"id": "openai:gpt-4o-mini",
|
|
375
|
+
"label": "",
|
|
376
|
+
"config": { "basePath": "examples/simple-csv" }
|
|
377
|
+
},
|
|
378
|
+
"latencyMs": 4,
|
|
379
|
+
"cost": 0.00001545
|
|
380
|
+
}
|
|
381
|
+
],
|
|
382
|
+
"prompts": [
|
|
383
|
+
{
|
|
384
|
+
"raw": "Rephrase this in {{language}}: {{body}}",
|
|
385
|
+
"label": "examples/simple-csv/prompts.txt: Rephrase this in {{language}}: {{body}}",
|
|
386
|
+
"id": "ad447337160016cbd0a5c7f2626d0c1fe2ade0d4d653e20bed9250632bc56e03",
|
|
387
|
+
"provider": "openai:gpt-4o-mini",
|
|
388
|
+
"metrics": {
|
|
389
|
+
"score": 4,
|
|
390
|
+
"testPassCount": 4,
|
|
391
|
+
"testFailCount": 0,
|
|
392
|
+
"assertPassCount": 0,
|
|
393
|
+
"assertFailCount": 0,
|
|
394
|
+
"totalLatencyMs": 21,
|
|
395
|
+
"tokenUsage": { "total": 87, "prompt": 0, "completion": 0, "cached": 87 },
|
|
396
|
+
"namedScores": {},
|
|
397
|
+
"namedScoresCount": {},
|
|
398
|
+
"cost": 0.000025199999999999996
|
|
399
|
+
}
|
|
400
|
+
},
|
|
401
|
+
{
|
|
402
|
+
"raw": "Translate this to conversational {{language}}: {{body}}",
|
|
403
|
+
"label": "examples/simple-csv/prompts.txt: Translate this to conversational {{language}}: {{body}}",
|
|
404
|
+
"id": "0ceccef5d40adcbf45bda7be485d2ee420e4d5943975932e7a38c5468d97c6ee",
|
|
405
|
+
"provider": "openai:gpt-4o-mini",
|
|
406
|
+
"metrics": {
|
|
407
|
+
"score": 4,
|
|
408
|
+
"testPassCount": 4,
|
|
409
|
+
"testFailCount": 0,
|
|
410
|
+
"assertPassCount": 0,
|
|
411
|
+
"assertFailCount": 0,
|
|
412
|
+
"totalLatencyMs": 24,
|
|
413
|
+
"tokenUsage": { "total": 103, "prompt": 0, "completion": 0, "cached": 103 },
|
|
414
|
+
"namedScores": {},
|
|
415
|
+
"namedScoresCount": {},
|
|
416
|
+
"cost": 0.0000348
|
|
417
|
+
}
|
|
418
|
+
}
|
|
419
|
+
],
|
|
420
|
+
"datasetId": "27018c2ea4e47d8af2fc1b2d69c131ed777f1d89e9069bf38622701b7e1e786a"
|
|
421
|
+
}
|
package/dist/test/types.test.js
CHANGED
|
@@ -162,11 +162,64 @@ describe('TestSuiteConfigSchema', () => {
|
|
|
162
162
|
message: "Exactly one of 'targets' or 'providers' must be provided, but not both",
|
|
163
163
|
})
|
|
164
164
|
.safeParse(config);
|
|
165
|
-
if (!result.success) {
|
|
166
|
-
console.error(`Validation failed for ${file}:`, result.error);
|
|
167
|
-
}
|
|
168
165
|
expect(result.success).toBe(true);
|
|
169
166
|
});
|
|
170
167
|
}
|
|
171
168
|
});
|
|
169
|
+
describe('TestSuiteSchema', () => {
|
|
170
|
+
const baseTestSuite = {
|
|
171
|
+
providers: [
|
|
172
|
+
{
|
|
173
|
+
id: () => 'mock-provider',
|
|
174
|
+
callApi: () => Promise.resolve({}),
|
|
175
|
+
},
|
|
176
|
+
],
|
|
177
|
+
prompts: [{ raw: 'Hello, world!', label: 'mock-prompt' }],
|
|
178
|
+
};
|
|
179
|
+
describe('extensions field', () => {
|
|
180
|
+
it('should accept valid Python extension paths', () => {
|
|
181
|
+
const validExtensions = [
|
|
182
|
+
'file://path/to/file.py:function_name',
|
|
183
|
+
'file://./relative/path.py:function_name',
|
|
184
|
+
'file:///absolute/path.py:function_name',
|
|
185
|
+
];
|
|
186
|
+
validExtensions.forEach((extension) => {
|
|
187
|
+
const result = types_1.TestSuiteSchema.safeParse({ ...baseTestSuite, extensions: [extension] });
|
|
188
|
+
expect(result.success).toBe(true);
|
|
189
|
+
});
|
|
190
|
+
});
|
|
191
|
+
it('should accept valid JavaScript extension paths', () => {
|
|
192
|
+
const validExtensions = [
|
|
193
|
+
'file://path/to/file.js:function_name',
|
|
194
|
+
'file://./relative/path.ts:function_name',
|
|
195
|
+
'file:///absolute/path.mjs:function_name',
|
|
196
|
+
'file://path/to/file.cjs:function_name',
|
|
197
|
+
];
|
|
198
|
+
validExtensions.forEach((extension) => {
|
|
199
|
+
const result = types_1.TestSuiteSchema.safeParse({ ...baseTestSuite, extensions: [extension] });
|
|
200
|
+
expect(result.success).toBe(true);
|
|
201
|
+
});
|
|
202
|
+
});
|
|
203
|
+
it.each([
|
|
204
|
+
['path/to/file.py:function_name', 'Missing file:// prefix'],
|
|
205
|
+
['file://path/to/file.txt:function_name', 'Invalid file extension'],
|
|
206
|
+
['file://path/to/file.py', 'Missing function name'],
|
|
207
|
+
['file://path/to/file.py:', 'Empty function name'],
|
|
208
|
+
['file://:function_name', 'Missing file path'],
|
|
209
|
+
['file://path/to/file.py:function_name:extra_arg', 'Extra argument'],
|
|
210
|
+
])('should reject invalid extension path: %s (%s)', (extension, reason) => {
|
|
211
|
+
const result = types_1.TestSuiteSchema.safeParse({ ...baseTestSuite, extensions: [extension] });
|
|
212
|
+
expect(result.success).toBe(false);
|
|
213
|
+
expect(result.error?.issues[0].message).toMatch(/Extension must/);
|
|
214
|
+
});
|
|
215
|
+
it('should allow extensions field to be optional', () => {
|
|
216
|
+
const result = types_1.TestSuiteSchema.safeParse({ ...baseTestSuite });
|
|
217
|
+
expect(result.success).toBe(true);
|
|
218
|
+
});
|
|
219
|
+
it('should allow an empty array of extensions', () => {
|
|
220
|
+
const result = types_1.TestSuiteSchema.safeParse({ ...baseTestSuite, extensions: [] });
|
|
221
|
+
expect(result.success).toBe(true);
|
|
222
|
+
});
|
|
223
|
+
});
|
|
224
|
+
});
|
|
172
225
|
//# sourceMappingURL=types.test.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.test.js","sourceRoot":"","sources":["../../test/types.test.ts"],"names":[],"mappings":";;;;;AAAA,4CAAoB;AACpB,+BAAgC;AAChC,sDAA2B;AAC3B,gDAAwB;AACxB,6BAAwB;
|
|
1
|
+
{"version":3,"file":"types.test.js","sourceRoot":"","sources":["../../test/types.test.ts"],"names":[],"mappings":";;;;;AAAA,4CAAoB;AACpB,+BAAgC;AAChC,sDAA2B;AAC3B,gDAAwB;AACxB,6BAAwB;AAExB,wCAOsB;AAEtB,QAAQ,CAAC,iBAAiB,EAAE,GAAG,EAAE;IAC/B,EAAE,CAAC,mCAAmC,EAAE,GAAG,EAAE;QAC3C,MAAM,cAAc,GAAG;YACrB,IAAI,EAAE,QAAQ;YACd,KAAK,EAAE,gBAAgB;SACxB,CAAC;QAEF,MAAM,MAAM,GAAG,uBAAe,CAAC,SAAS,CAAC,cAAc,CAAC,CAAC;QACzD,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACpC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,uDAAuD,EAAE,GAAG,EAAE;QAC/D,MAAM,aAAa,GAAG;YACpB,IAAI,EAAE,SAAS;YACf,KAAK,EAAE,gBAAgB;YACvB,SAAS,EAAE,GAAG;YACd,MAAM,EAAE,CAAC;YACT,QAAQ,EAAE,oBAAoB;YAC9B,YAAY,EAAE,sBAAsB;YACpC,MAAM,EAAE,kBAAkB;YAC1B,SAAS,EAAE,eAAe;SAC3B,CAAC;QAEF,MAAM,MAAM,GAAG,uBAAe,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC;QACxD,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACpC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,0CAA0C,EAAE,GAAG,EAAE;QAClD,MAAM,SAAS,GAAG,gCAAwB,CAAC,OAAO,CAAC;QAEnD,SAAS,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE;YACzB,MAAM,SAAS,GAAG;gBAChB,IAAI;gBACJ,KAAK,EAAE,YAAY;aACpB,CAAC;YAEF,MAAM,MAAM,GAAG,uBAAe,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC;YACpD,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACpC,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,iDAAiD,EAAE,GAAG,EAAE;QACzD,MAAM,oBAAoB,GAAG;YAC3B,IAAI,EAAE,cAAc;YACpB,KAAK,EAAE,gBAAgB;SACxB,CAAC;QAEF,MAAM,MAAM,GAAG,uBAAe,CAAC,SAAS,CAAC,oBAAoB,CAAC,CAAC;QAC/D,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACpC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,iDAAiD,EAAE,GAAG,EAAE;QACzD,MAAM,iBAAiB,GAAG;YACxB,IAAI,EAAE,QAAQ;YACd,KAAK,EAAE,CAAC,MAAc,EAAE,EAAE,CAAC,MAAM,KAAK,gBAAgB;SACvD,CAAC;QAEF,MAAM,MAAM,GAAG,uBAAe,CAAC,SAAS,CAAC,iBAAiB,CAAC,CAAC;QAC5D,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACpC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,8CAA8C,EAAE,GAAG,EAAE;QACtD,MAAM,cAAc,GAAG;YACrB,IAAI,EAAE,cAAc;YACpB,KAAK,EAAE,CAAC,QAAQ,EAAE,QAAQ,EAAE,QAAQ,CAAC;SACtC,CAAC;QAEF,MAAM,MAAM,GAAG,uBAAe,CAAC,SAAS,CAAC,cAAc,CAAC,CAAC;QACzD,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACpC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,qDAAqD,EAAE,GAAG,EAAE;QAC7D,MAAM,yBAAyB,GAAG;YAChC,IAAI,EAAE,cAAc;YACpB,KAAK,EAAE,gBAAgB;SACxB,CAAC;QAEF,MAAM,MAAM,GAAG,uBAAe,CAAC,SAAS,CAAC,yBAAyB,CAAC,CAAC;QACpE,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACpC,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,YAAY,EAAE,GAAG,EAAE;IAC1B,EAAE,CAAC,uDAAuD,EAAE,GAAG,EAAE;QAC/D,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;QACrB,MAAM,SAAS,GAAG;YAChB,EAAE,KAAK,EAAE,EAAE,GAAG,EAAE,cAAc,EAAE,EAAE,QAAQ,EAAE,EAAE,GAAG,EAAE,cAAc,EAAE,EAAE;YACrE,EAAE,KAAK,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE,EAAE,QAAQ,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE,EAAE;YAC/C,EAAE,KAAK,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE,EAAE,QAAQ,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,EAAE;YACnD,EAAE,KAAK,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,EAAE,QAAQ,EAAE,EAAE,GAAG,EAAE,OAAO,EAAE,EAAE;YACrD,EAAE,KAAK,EAAE,EAAE,GAAG,EAAE,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,EAAE,EAAE,QAAQ,EAAE,EAAE,GAAG,EAAE,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,EAAE,EAAE;YACvE,EAAE,KAAK,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,QAAQ,EAAE,EAAE,GAAG,EAAE,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,EAAE,EAAE;YACjE,EAAE,KAAK,EAAE,EAAE,GAAG,EAAE,CAAC,IAAI,EAAE,KAAK,CAAC,EAAE,EAAE,QAAQ,EAAE,EAAE,GAAG,EAAE,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,EAAE;YACvE,EAAE,KAAK,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,EAAE,EAAE,QAAQ,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,EAAE,EAAE;SACtF,CAAC;QAEF,SAAS,CAAC,OAAO,CAAC,CAAC,EAAE,KAAK,EAAE,QAAQ,EAAE,EAAE,EAAE;YACxC,MAAM,CAAC,kBAAU,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC,CAAC;QACjF,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,yCAAyC,EAAE,GAAG,EAAE;QACjD,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;QAErB,MAAM,YAAY,GAAG;YACnB,EAAE,GAAG,EAAE,IAAI,EAAE;YACb,EAAE,GAAG,EAAE,SAAS,EAAE;YAClB,EAAE,GAAG,EAAE,MAAM,CAAC,MAAM,CAAC,EAAE;YACvB,EAAE,GAAG,EAAE,GAAG,EAAE,GAAE,CAAC,EAAE;SAClB,CAAC;QAEF,YAAY,CAAC,OAAO,CAAC,CAAC,YAAY,EAAE,EAAE;YACpC,MAAM,CAAC,GAAG,EAAE,CAAC,kBAAU,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC,CAAC,OAAO,CAAC,OAAC,CAAC,QAAQ,CAAC,CAAC;QACnE,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,iBAAiB,EAAE,GAAG,EAAE;IAC/B,EAAE,CAAC,uDAAuD,EAAE,GAAG,EAAE;QAC/D,MAAM,YAAY,GAAG;YACnB,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE;YAC3C,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,QAAQ,EAAE,WAAW,EAAE,EAAE,QAAQ,EAAE,CAAC,EAAE,EAAE;YACzE,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,SAAS,EAAE,UAAU,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE;YACzE,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,gBAAgB,EAAE,EAAE,EAAE;YAC9D,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE;YACzD,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,WAAW,EAAE,OAAO,EAAE,YAAY,EAAE;SACrE,CAAC;QAEF,YAAY,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,EAAE;YAC9B,MAAM,CAAC,IAAA,uBAAe,EAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC7C,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,yDAAyD,EAAE,GAAG,EAAE;QACjE,MAAM,cAAc,GAAG;YACrB,EAAE;YACF,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,mBAAmB,EAAE;YACvD,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,oBAAoB,EAAE;YACxD,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE;YACpC,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,OAAO,EAAE,WAAW,EAAE,SAAS,EAAE;YACjE,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE;YAChE,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,OAAO,EAAE,gBAAgB,EAAE,SAAS,EAAE;YACtE,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE;YAC/D,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,EAAE,EAAE;SACvD,CAAC;QAEF,cAAc,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,EAAE;YAChC,MAAM,CAAC,IAAA,uBAAe,EAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC9C,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,uBAAuB,EAAE,GAAG,EAAE;IACrC,MAAM,OAAO,GAAG,cAAI,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;IAC3C,MAAM,WAAW,GAAG,IAAA,eAAQ,EAAC,GAAG,OAAO,8CAA8C,CAAC,CAAC;IAEvF,EAAE,CAAC,iCAAiC,EAAE,GAAG,EAAE;QACzC,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;IAChD,CAAC,CAAC,CAAC;IAEH,KAAK,MAAM,IAAI,IAAI,WAAW,EAAE,CAAC;QAC/B,EAAE,CAAC,mBAAmB,cAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,IAAI,CAAC,EAAE,EAAE,KAAK,IAAI,EAAE;YAC/D,MAAM,aAAa,GAAG,YAAE,CAAC,YAAY,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;YACpD,MAAM,MAAM,GAAG,iBAAI,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;YACxC,4DAA4D;YAC5D,iHAAiH;YACjH,qFAAqF;YACrF,MAAM,MAAM,GAAG,6BAAqB,CAAC,MAAM,CAAC;gBAC1C,OAAO,EAAE,OAAC,CAAC,KAAK,CAAC,CAAC,6BAAqB,CAAC,KAAK,CAAC,SAAS,EAAE,OAAC,CAAC,SAAS,EAAE,CAAC,CAAC;gBACxE,SAAS,EAAE,OAAC,CAAC,KAAK,CAAC,CAAC,6BAAqB,CAAC,KAAK,CAAC,SAAS,EAAE,OAAC,CAAC,SAAS,EAAE,CAAC,CAAC;aAC3E,CAAC;iBACC,MAAM,CACL,CAAC,IAAI,EAAE,EAAE;gBACP,MAAM,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBACzC,MAAM,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;gBAC7C,OAAO,CAAC,UAAU,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,UAAU,IAAI,YAAY,CAAC,CAAC;YACxE,CAAC,EACD;gBACE,OAAO,EAAE,wEAAwE;aAClF,CACF;iBACA,SAAS,CAAC,MAAM,CAAC,CAAC;YACrB,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACpC,CAAC,CAAC,CAAC;IACL,CAAC;AACH,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,iBAAiB,EAAE,GAAG,EAAE;IAC/B,MAAM,aAAa,GAAc;QAC/B,SAAS,EAAE;YACT;gBACE,EAAE,EAAE,GAAG,EAAE,CAAC,eAAe;gBACzB,OAAO,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;aACnC;SACF;QACD,OAAO,EAAE,CAAC,EAAE,GAAG,EAAE,eAAe,EAAE,KAAK,EAAE,aAAa,EAAE,CAAC;KAC1D,CAAC;IAEF,QAAQ,CAAC,kBAAkB,EAAE,GAAG,EAAE;QAChC,EAAE,CAAC,4CAA4C,EAAE,GAAG,EAAE;YACpD,MAAM,eAAe,GAAG;gBACtB,sCAAsC;gBACtC,yCAAyC;gBACzC,wCAAwC;aACzC,CAAC;YAEF,eAAe,CAAC,OAAO,CAAC,CAAC,SAAS,EAAE,EAAE;gBACpC,MAAM,MAAM,GAAG,uBAAe,CAAC,SAAS,CAAC,EAAE,GAAG,aAAa,EAAE,UAAU,EAAE,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;gBACxF,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACpC,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,gDAAgD,EAAE,GAAG,EAAE;YACxD,MAAM,eAAe,GAAG;gBACtB,sCAAsC;gBACtC,yCAAyC;gBACzC,yCAAyC;gBACzC,uCAAuC;aACxC,CAAC;YAEF,eAAe,CAAC,OAAO,CAAC,CAAC,SAAS,EAAE,EAAE;gBACpC,MAAM,MAAM,GAAG,uBAAe,CAAC,SAAS,CAAC,EAAE,GAAG,aAAa,EAAE,UAAU,EAAE,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;gBACxF,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACpC,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,IAAI,CAAC;YACN,CAAC,+BAA+B,EAAE,wBAAwB,CAAC;YAC3D,CAAC,uCAAuC,EAAE,wBAAwB,CAAC;YACnE,CAAC,wBAAwB,EAAE,uBAAuB,CAAC;YACnD,CAAC,yBAAyB,EAAE,qBAAqB,CAAC;YAClD,CAAC,uBAAuB,EAAE,mBAAmB,CAAC;YAC9C,CAAC,gDAAgD,EAAE,gBAAgB,CAAC;SACrE,CAAC,CAAC,+CAA+C,EAAE,CAAC,SAAS,EAAE,MAAM,EAAE,EAAE;YACxE,MAAM,MAAM,GAAG,uBAAe,CAAC,SAAS,CAAC,EAAE,GAAG,aAAa,EAAE,UAAU,EAAE,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;YACxF,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACnC,MAAM,CAAC,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,OAAO,CAAC,gBAAgB,CAAC,CAAC;QACpE,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,8CAA8C,EAAE,GAAG,EAAE;YACtD,MAAM,MAAM,GAAG,uBAAe,CAAC,SAAS,CAAC,EAAE,GAAG,aAAa,EAAE,CAAC,CAAC;YAC/D,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACpC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,2CAA2C,EAAE,GAAG,EAAE;YACnD,MAAM,MAAM,GAAG,uBAAe,CAAC,SAAS,CAAC,EAAE,GAAG,aAAa,EAAE,UAAU,EAAE,EAAE,EAAE,CAAC,CAAC;YAC/E,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACpC,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"util.file.test.d.ts","sourceRoot":"","sources":["../../test/util.file.test.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
const file_1 = require("../src/util/file");
|
|
4
|
+
describe('util', () => {
|
|
5
|
+
beforeEach(() => {
|
|
6
|
+
jest.clearAllMocks();
|
|
7
|
+
});
|
|
8
|
+
describe('isJavascriptFile', () => {
|
|
9
|
+
it('should return true for JavaScript files', () => {
|
|
10
|
+
expect((0, file_1.isJavascriptFile)('file.js')).toBe(true);
|
|
11
|
+
expect((0, file_1.isJavascriptFile)('file.cjs')).toBe(true);
|
|
12
|
+
expect((0, file_1.isJavascriptFile)('file.mjs')).toBe(true);
|
|
13
|
+
});
|
|
14
|
+
it('should return true for TypeScript files', () => {
|
|
15
|
+
expect((0, file_1.isJavascriptFile)('file.ts')).toBe(true);
|
|
16
|
+
expect((0, file_1.isJavascriptFile)('file.cts')).toBe(true);
|
|
17
|
+
expect((0, file_1.isJavascriptFile)('file.mts')).toBe(true);
|
|
18
|
+
});
|
|
19
|
+
it('should return false for non-JavaScript/TypeScript files', () => {
|
|
20
|
+
expect((0, file_1.isJavascriptFile)('file.txt')).toBe(false);
|
|
21
|
+
expect((0, file_1.isJavascriptFile)('file.py')).toBe(false);
|
|
22
|
+
expect((0, file_1.isJavascriptFile)('file.jsx')).toBe(false);
|
|
23
|
+
expect((0, file_1.isJavascriptFile)('file.tsx')).toBe(false);
|
|
24
|
+
});
|
|
25
|
+
it('should handle paths with directories', () => {
|
|
26
|
+
expect((0, file_1.isJavascriptFile)('/path/to/file.js')).toBe(true);
|
|
27
|
+
expect((0, file_1.isJavascriptFile)('C:\\path\\to\\file.ts')).toBe(true);
|
|
28
|
+
expect((0, file_1.isJavascriptFile)('/path/to/file.txt')).toBe(false);
|
|
29
|
+
});
|
|
30
|
+
});
|
|
31
|
+
});
|
|
32
|
+
//# sourceMappingURL=util.file.test.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"util.file.test.js","sourceRoot":"","sources":["../../test/util.file.test.ts"],"names":[],"mappings":";;AAAA,2CAAoD;AAEpD,QAAQ,CAAC,MAAM,EAAE,GAAG,EAAE;IACpB,UAAU,CAAC,GAAG,EAAE;QACd,IAAI,CAAC,aAAa,EAAE,CAAC;IACvB,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,kBAAkB,EAAE,GAAG,EAAE;QAChC,EAAE,CAAC,yCAAyC,EAAE,GAAG,EAAE;YACjD,MAAM,CAAC,IAAA,uBAAgB,EAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC/C,MAAM,CAAC,IAAA,uBAAgB,EAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAChD,MAAM,CAAC,IAAA,uBAAgB,EAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAClD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,yCAAyC,EAAE,GAAG,EAAE;YACjD,MAAM,CAAC,IAAA,uBAAgB,EAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC/C,MAAM,CAAC,IAAA,uBAAgB,EAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAChD,MAAM,CAAC,IAAA,uBAAgB,EAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAClD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,yDAAyD,EAAE,GAAG,EAAE;YACjE,MAAM,CAAC,IAAA,uBAAgB,EAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACjD,MAAM,CAAC,IAAA,uBAAgB,EAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAChD,MAAM,CAAC,IAAA,uBAAgB,EAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACjD,MAAM,CAAC,IAAA,uBAAgB,EAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACnD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,sCAAsC,EAAE,GAAG,EAAE;YAC9C,MAAM,CAAC,IAAA,uBAAgB,EAAC,kBAAkB,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACxD,MAAM,CAAC,IAAA,uBAAgB,EAAC,uBAAuB,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC7D,MAAM,CAAC,IAAA,uBAAgB,EAAC,mBAAmB,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC5D,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"util.listPrevious.test.d.ts","sourceRoot":"","sources":["../../test/util.listPrevious.test.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
const migrate_1 = require("../src/migrate");
|
|
7
|
+
const util_1 = require("../src/util");
|
|
8
|
+
const evalFactory_1 = __importDefault(require("./factories/evalFactory"));
|
|
9
|
+
describe('listPreviousResults', () => {
|
|
10
|
+
beforeAll(async () => {
|
|
11
|
+
await (0, migrate_1.runDbMigrations)();
|
|
12
|
+
});
|
|
13
|
+
it('returns previous results', async () => {
|
|
14
|
+
const eval1 = await evalFactory_1.default.create();
|
|
15
|
+
const eval2 = await evalFactory_1.default.create();
|
|
16
|
+
const evalOld = await evalFactory_1.default.createOldResult();
|
|
17
|
+
const results = await (0, util_1.listPreviousResults)();
|
|
18
|
+
expect(results).toHaveLength(3);
|
|
19
|
+
expect(results).toContainEqual(expect.objectContaining({
|
|
20
|
+
evalId: eval1.id,
|
|
21
|
+
createdAt: eval1.createdAt,
|
|
22
|
+
numTests: 2,
|
|
23
|
+
}));
|
|
24
|
+
expect(results).toContainEqual(expect.objectContaining({
|
|
25
|
+
evalId: eval2.id,
|
|
26
|
+
createdAt: eval2.createdAt,
|
|
27
|
+
description: eval2.description || null,
|
|
28
|
+
numTests: 2,
|
|
29
|
+
}));
|
|
30
|
+
expect(results).toContainEqual(expect.objectContaining({
|
|
31
|
+
evalId: evalOld.id,
|
|
32
|
+
createdAt: evalOld.createdAt,
|
|
33
|
+
numTests: 1,
|
|
34
|
+
}));
|
|
35
|
+
});
|
|
36
|
+
});
|
|
37
|
+
//# sourceMappingURL=util.listPrevious.test.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"util.listPrevious.test.js","sourceRoot":"","sources":["../../test/util.listPrevious.test.ts"],"names":[],"mappings":";;;;;AAAA,4CAAiD;AACjD,sCAAkD;AAClD,0EAAkD;AAElD,QAAQ,CAAC,qBAAqB,EAAE,GAAG,EAAE;IACnC,SAAS,CAAC,KAAK,IAAI,EAAE;QACnB,MAAM,IAAA,yBAAe,GAAE,CAAC;IAC1B,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,0BAA0B,EAAE,KAAK,IAAI,EAAE;QACxC,MAAM,KAAK,GAAG,MAAM,qBAAW,CAAC,MAAM,EAAE,CAAC;QACzC,MAAM,KAAK,GAAG,MAAM,qBAAW,CAAC,MAAM,EAAE,CAAC;QACzC,MAAM,OAAO,GAAQ,MAAM,qBAAW,CAAC,eAAe,EAAE,CAAC;QACzD,MAAM,OAAO,GAAG,MAAM,IAAA,0BAAmB,GAAE,CAAC;QAC5C,MAAM,CAAC,OAAO,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAChC,MAAM,CAAC,OAAO,CAAC,CAAC,cAAc,CAC5B,MAAM,CAAC,gBAAgB,CAAC;YACtB,MAAM,EAAE,KAAK,CAAC,EAAE;YAChB,SAAS,EAAE,KAAK,CAAC,SAAS;YAC1B,QAAQ,EAAE,CAAC;SACZ,CAAC,CACH,CAAC;QACF,MAAM,CAAC,OAAO,CAAC,CAAC,cAAc,CAC5B,MAAM,CAAC,gBAAgB,CAAC;YACtB,MAAM,EAAE,KAAK,CAAC,EAAE;YAChB,SAAS,EAAE,KAAK,CAAC,SAAS;YAC1B,WAAW,EAAE,KAAK,CAAC,WAAW,IAAI,IAAI;YACtC,QAAQ,EAAE,CAAC;SACZ,CAAC,CACH,CAAC;QACF,MAAM,CAAC,OAAO,CAAC,CAAC,cAAc,CAC5B,MAAM,CAAC,gBAAgB,CAAC;YACtB,MAAM,EAAE,OAAO,CAAC,EAAE;YAClB,SAAS,EAAE,OAAO,CAAC,SAAS;YAC5B,QAAQ,EAAE,CAAC;SACZ,CAAC,CACH,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|