promptfoo 0.9.0 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +50 -40
- package/dist/assertions.d.ts +2 -2
- package/dist/assertions.d.ts.map +1 -1
- package/dist/assertions.js +186 -44
- package/dist/assertions.js.map +1 -1
- package/dist/cache.js +9 -9
- package/dist/cache.js.map +1 -1
- package/dist/evaluator.d.ts +1 -1
- package/dist/evaluator.d.ts.map +1 -1
- package/dist/evaluator.js +30 -23
- package/dist/evaluator.js.map +1 -1
- package/dist/index.d.ts +10 -10
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +18 -14
- package/dist/index.js.map +1 -1
- package/dist/main.js +49 -44
- package/dist/main.js.map +1 -1
- package/dist/providers/localai.js +11 -11
- package/dist/providers/localai.js.map +1 -1
- package/dist/providers/openai.d.ts.map +1 -1
- package/dist/providers/openai.js +30 -21
- package/dist/providers/openai.js.map +1 -1
- package/dist/providers.d.ts +3 -3
- package/dist/providers.d.ts.map +1 -1
- package/dist/providers.js +15 -15
- package/dist/providers.js.map +1 -1
- package/dist/types.d.ts +7 -3
- package/dist/types.d.ts.map +1 -1
- package/dist/util.d.ts +4 -4
- package/dist/util.d.ts.map +1 -1
- package/dist/util.js +49 -18
- package/dist/util.js.map +1 -1
- package/dist/web/client/assets/index-15dfcd18.js +172 -0
- package/dist/web/client/assets/index-87905193.css +1 -0
- package/dist/web/client/index.html +2 -2
- package/dist/web/server.js +9 -9
- package/dist/web/server.js.map +1 -1
- package/package.json +3 -1
- package/src/assertions.ts +249 -38
- package/src/cache.ts +2 -2
- package/src/evaluator.ts +25 -18
- package/src/index.ts +13 -8
- package/src/main.ts +28 -15
- package/src/providers/localai.ts +3 -3
- package/src/providers/openai.ts +16 -8
- package/src/providers.ts +3 -3
- package/src/types.ts +24 -3
- package/src/util.ts +48 -17
- package/src/web/client/package-lock.json +5729 -0
- package/src/web/client/src/ResultsTable.css +35 -4
- package/src/web/client/src/ResultsTable.tsx +150 -70
- package/src/web/client/src/ResultsView.tsx +83 -18
- package/src/web/client/src/index.css +6 -0
- package/src/web/client/src/types.ts +2 -0
- package/src/web/server.ts +3 -3
- package/dist/web/client/assets/index-207192fc.css +0 -1
- package/dist/web/client/assets/index-8751749f.js +0 -172
package/dist/evaluator.js
CHANGED
|
@@ -31,9 +31,9 @@ const node_readline_1 = __importDefault(require("node:readline"));
|
|
|
31
31
|
const async_1 = __importDefault(require("async"));
|
|
32
32
|
const chalk_1 = __importDefault(require("chalk"));
|
|
33
33
|
const nunjucks_1 = __importDefault(require("nunjucks"));
|
|
34
|
-
const
|
|
35
|
-
const
|
|
36
|
-
const
|
|
34
|
+
const logger_1 = __importDefault(require("./logger"));
|
|
35
|
+
const assertions_1 = require("./assertions");
|
|
36
|
+
const suggestions_1 = require("./suggestions");
|
|
37
37
|
const DEFAULT_MAX_CONCURRENCY = 4;
|
|
38
38
|
function generateVarCombinations(vars) {
|
|
39
39
|
const keys = Object.keys(vars);
|
|
@@ -68,9 +68,12 @@ class Evaluator {
|
|
|
68
68
|
}
|
|
69
69
|
async runEval({ provider, prompt, test, includeProviderId, }) {
|
|
70
70
|
const vars = test.vars || {};
|
|
71
|
-
const renderedPrompt = nunjucks_1.default.renderString(prompt, vars);
|
|
71
|
+
const renderedPrompt = nunjucks_1.default.renderString(prompt.raw, vars);
|
|
72
72
|
// Note that we're using original prompt, not renderedPrompt
|
|
73
|
-
|
|
73
|
+
let promptDisplay = prompt.display;
|
|
74
|
+
if (includeProviderId) {
|
|
75
|
+
promptDisplay = `[${provider.id()}] ${promptDisplay}`;
|
|
76
|
+
}
|
|
74
77
|
const setup = {
|
|
75
78
|
prompt: {
|
|
76
79
|
raw: renderedPrompt,
|
|
@@ -89,7 +92,7 @@ class Evaluator {
|
|
|
89
92
|
ret.error = response.error;
|
|
90
93
|
}
|
|
91
94
|
else if (response.output) {
|
|
92
|
-
const checkResult = await (0,
|
|
95
|
+
const checkResult = await (0, assertions_1.runAssertions)(test, response.output);
|
|
93
96
|
if (!checkResult.pass) {
|
|
94
97
|
ret.error = checkResult.reason;
|
|
95
98
|
}
|
|
@@ -122,7 +125,7 @@ class Evaluator {
|
|
|
122
125
|
catch (err) {
|
|
123
126
|
return {
|
|
124
127
|
...setup,
|
|
125
|
-
error: String(err),
|
|
128
|
+
error: String(err) + '\n\n' + err.stack,
|
|
126
129
|
success: false,
|
|
127
130
|
};
|
|
128
131
|
}
|
|
@@ -132,17 +135,17 @@ class Evaluator {
|
|
|
132
135
|
const prompts = [];
|
|
133
136
|
if (options.generateSuggestions) {
|
|
134
137
|
// TODO(ian): Move this into its own command/file
|
|
135
|
-
|
|
136
|
-
const { prompts: newPrompts, error } = await (0,
|
|
138
|
+
logger_1.default.info(`Generating prompt variations...`);
|
|
139
|
+
const { prompts: newPrompts, error } = await (0, suggestions_1.generatePrompts)(testSuite.prompts[0].raw, 1);
|
|
137
140
|
if (error || !newPrompts) {
|
|
138
141
|
throw new Error(`Failed to generate prompts: ${error}`);
|
|
139
142
|
}
|
|
140
|
-
|
|
143
|
+
logger_1.default.info(chalk_1.default.blue('Generated prompts:'));
|
|
141
144
|
let numAdded = 0;
|
|
142
145
|
for (const prompt of newPrompts) {
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
+
logger_1.default.info('--------------------------------------------------------');
|
|
147
|
+
logger_1.default.info(`${prompt}`);
|
|
148
|
+
logger_1.default.info('--------------------------------------------------------');
|
|
146
149
|
// Ask the user if they want to continue
|
|
147
150
|
await new Promise((resolve) => {
|
|
148
151
|
const rl = node_readline_1.default.createInterface({
|
|
@@ -152,28 +155,28 @@ class Evaluator {
|
|
|
152
155
|
rl.question(`${chalk_1.default.blue('Do you want to test this prompt?')} (y/N): `, async (answer) => {
|
|
153
156
|
rl.close();
|
|
154
157
|
if (answer.toLowerCase().startsWith('y')) {
|
|
155
|
-
testSuite.prompts.push(prompt);
|
|
158
|
+
testSuite.prompts.push({ raw: prompt, display: prompt });
|
|
156
159
|
numAdded++;
|
|
157
160
|
}
|
|
158
161
|
else {
|
|
159
|
-
|
|
162
|
+
logger_1.default.info('Skipping this prompt.');
|
|
160
163
|
}
|
|
161
164
|
resolve(true);
|
|
162
165
|
});
|
|
163
166
|
});
|
|
164
167
|
}
|
|
165
168
|
if (numAdded < 1) {
|
|
166
|
-
|
|
169
|
+
logger_1.default.info(chalk_1.default.red('No prompts selected. Aborting.'));
|
|
167
170
|
process.exit(1);
|
|
168
171
|
}
|
|
169
172
|
}
|
|
170
173
|
// Split prompts by provider
|
|
171
|
-
for (const
|
|
174
|
+
for (const prompt of testSuite.prompts) {
|
|
172
175
|
for (const provider of testSuite.providers) {
|
|
173
|
-
const
|
|
176
|
+
const updatedDisplay = testSuite.providers.length > 1 ? `[${provider.id()}] ${prompt.display}` : prompt.display;
|
|
174
177
|
prompts.push({
|
|
175
|
-
|
|
176
|
-
display,
|
|
178
|
+
...prompt,
|
|
179
|
+
display: updatedDisplay,
|
|
177
180
|
});
|
|
178
181
|
}
|
|
179
182
|
}
|
|
@@ -211,6 +214,7 @@ class Evaluator {
|
|
|
211
214
|
// And progress bar...
|
|
212
215
|
let progressbar;
|
|
213
216
|
if (options.showProgressBar) {
|
|
217
|
+
// FIXME(ian): Add var combinations too
|
|
214
218
|
const totalNumRuns = testSuite.prompts.length * testSuite.providers.length * (tests.length || 1);
|
|
215
219
|
const cliProgress = await Promise.resolve().then(() => __importStar(require('cli-progress')));
|
|
216
220
|
progressbar = new cliProgress.SingleBar({
|
|
@@ -238,11 +242,14 @@ class Evaluator {
|
|
|
238
242
|
const varCombinations = generateVarCombinations(testCase.vars || {});
|
|
239
243
|
for (const vars of varCombinations) {
|
|
240
244
|
let colIndex = 0;
|
|
241
|
-
for (const
|
|
245
|
+
for (const prompt of testSuite.prompts) {
|
|
242
246
|
for (const provider of testSuite.providers) {
|
|
243
247
|
runEvalOptions.push({
|
|
244
248
|
provider,
|
|
245
|
-
prompt:
|
|
249
|
+
prompt: {
|
|
250
|
+
...prompt,
|
|
251
|
+
raw: prependToPrompt + prompt.raw + appendToPrompt,
|
|
252
|
+
},
|
|
246
253
|
test: { ...testCase, vars },
|
|
247
254
|
includeProviderId: testSuite.providers.length > 1,
|
|
248
255
|
rowIndex,
|
|
@@ -262,7 +269,7 @@ class Evaluator {
|
|
|
262
269
|
if (progressbar) {
|
|
263
270
|
progressbar.increment({
|
|
264
271
|
provider: options.provider.id(),
|
|
265
|
-
prompt: options.prompt.slice(0, 10),
|
|
272
|
+
prompt: options.prompt.raw.slice(0, 10),
|
|
266
273
|
vars: Object.entries(options.test.vars || {})
|
|
267
274
|
.map(([k, v]) => `${k}=${v}`)
|
|
268
275
|
.join(' ')
|
package/dist/evaluator.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"evaluator.js","sourceRoot":"","sources":["../src/evaluator.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,kEAAqC;AAErC,kDAA0B;AAC1B,kDAA0B;AAC1B,wDAAgC;AAEhC,
|
|
1
|
+
{"version":3,"file":"evaluator.js","sourceRoot":"","sources":["../src/evaluator.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,kEAAqC;AAErC,kDAA0B;AAC1B,kDAA0B;AAC1B,wDAAgC;AAEhC,sDAA8B;AAC9B,6CAA6C;AAe7C,+CAAgD;AAchD,MAAM,uBAAuB,GAAG,CAAC,CAAC;AAElC,SAAS,uBAAuB,CAC9B,IAAuC;IAEvC,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC/B,MAAM,YAAY,GAA6B,CAAC,EAAE,CAAC,CAAC;IAEpD,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE;QACtB,MAAM,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;QAClE,MAAM,eAAe,GAA6B,EAAE,CAAC;QAErD,KAAK,MAAM,WAAW,IAAI,YAAY,EAAE;YACtC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE;gBAC1B,eAAe,CAAC,IAAI,CAAC,EAAE,GAAG,WAAW,EAAE,CAAC,GAAG,CAAC,EAAE,KAAe,EAAE,CAAC,CAAC;aAClE;SACF;QAED,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC;QACxB,YAAY,CAAC,IAAI,CAAC,GAAG,eAAe,CAAC,CAAC;KACvC;IAED,OAAO,YAAY,CAAC;AACtB,CAAC;AAED,MAAM,SAAS;IAKb,YAAY,SAAoB,EAAE,OAAwB;QACxD,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,CAAC,KAAK,GAAG;YACX,SAAS,EAAE,CAAC;YACZ,QAAQ,EAAE,CAAC;YACX,UAAU,EAAE;gBACV,KAAK,EAAE,CAAC;gBACR,MAAM,EAAE,CAAC;gBACT,UAAU,EAAE,CAAC;gBACb,MAAM,EAAE,CAAC;aACV;SACF,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,EACZ,QAAQ,EACR,MAAM,EACN,IAAI,EACJ,iBAAiB,GACF;QACf,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;QAC7B,MAAM,cAAc,GAAG,kBAAQ,CAAC,YAAY,CAAC,MAAM,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;QAE/D,4DAA4D;QAC5D,IAAI,aAAa,GAAG,MAAM,CAAC,OAAO,CAAC;QACnC,IAAI,iBAAiB,EAAE;YACrB,aAAa,GAAG,IAAI,QAAQ,CAAC,EAAE,EAAE,KAAK,aAAa,EAAE,CAAC;SACvD;QAED,MAAM,KAAK,GAAG;YACZ,MAAM,EAAE;gBACN,GAAG,EAAE,cAAc;gBACnB,OAAO,EAAE,aAAa;aACvB;YACD,IAAI;SACL,CAAC;QAEF,IAAI;YACF,MAAM,QAAQ,GAAG,MAAM,QAAQ,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;YACxD,MAAM,GAAG,GAAmB;gBAC1B,GAAG,KAAK;gBACR,QAAQ;gBACR,OAAO,EAAE,KAAK;aACf,CAAC;YACF,IAAI,QAAQ,CAAC,KAAK,EAAE;gBAClB,GAAG,CAAC,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC;aAC5B;iBAAM,IAAI,QAAQ,CAAC,MAAM,EAAE;gBAC1B,MAAM,WAAW,GAAG,MAAM,IAAA,0BAAa,EAAC,IAAI,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;gBAC/D,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE;oBACrB,GAAG,CAAC,KAAK,GAAG,WAAW,CAAC,MAAM,CAAC;iBAChC;gBACD,GAAG,CAAC,OAAO,GAAG,WAAW,CAAC,IAAI,CAAC;gBAC/B,IAAI,WAAW,CAAC,UAAU,EAAE;oBAC1B,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,KAAK,IAAI,WAAW,CAAC,UAAU,CAAC,KAAK,CAAC;oBAC5D,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,MAAM,IAAI,WAAW,CAAC,UAAU,CAAC,MAAM,CAAC;oBAC9D,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,UAAU,IAAI,WAAW,CAAC,UAAU,CAAC,UAAU,CAAC;iBACvE;aACF;iBAAM;gBACL,GAAG,CAAC,OAAO,GAAG,KAAK,CAAC;gBACpB,GAAG,CAAC,KAAK,GAAG,WAAW,CAAC;aACzB;YAED,2BAA2B;YAC3B,IAAI,QAAQ,CAAC,UAAU,EAAE;gBACvB,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,KAAK,IAAI,QAAQ,CAAC,UAAU,CAAC,KAAK,IAAI,CAAC,CAAC;gBAC9D,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,MAAM,IAAI,QAAQ,CAAC,UAAU,CAAC,MAAM,IAAI,CAAC,CAAC;gBAChE,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,UAAU,IAAI,QAAQ,CAAC,UAAU,CAAC,UAAU,IAAI,CAAC,CAAC;gBACxE,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,MAAM,IAAI,QAAQ,CAAC,UAAU,CAAC,MAAM,IAAI,CAAC,CAAC;aACjE;YAED,IAAI,GAAG,CAAC,OAAO,EAAE;gBACf,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC;aACxB;iBAAM;gBACL,IAAI,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC;aACvB;YAED,OAAO,GAAG,CAAC;SACZ;QAAC,OAAO,GAAG,EAAE;YACZ,OAAO;gBACL,GAAG,KAAK;gBACR,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,GAAG,MAAM,GAAI,GAAa,CAAC,KAAK;gBAClD,OAAO,EAAE,KAAK;aACf,CAAC;SACH;IACH,CAAC;IAED,KAAK,CAAC,QAAQ;QACZ,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,GAAG,IAAI,CAAC;QACpC,MAAM,OAAO,GAAa,EAAE,CAAC;QAE7B,IAAI,OAAO,CAAC,mBAAmB,EAAE;YAC/B,iDAAiD;YACjD,gBAAM,CAAC,IAAI,CAAC,iCAAiC,CAAC,CAAC;YAC/C,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,KAAK,EAAE,GAAG,MAAM,IAAA,6BAAe,EAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;YAC1F,IAAI,KAAK,IAAI,CAAC,UAAU,EAAE;gBACxB,MAAM,IAAI,KAAK,CAAC,+BAA+B,KAAK,EAAE,CAAC,CAAC;aACzD;YAED,gBAAM,CAAC,IAAI,CAAC,eAAK,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC,CAAC;YAC9C,IAAI,QAAQ,GAAG,CAAC,CAAC;YACjB,KAAK,MAAM,MAAM,IAAI,UAAU,EAAE;gBAC/B,gBAAM,CAAC,IAAI,CAAC,0DAA0D,CAAC,CAAC;gBACxE,gBAAM,CAAC,IAAI,CAAC,GAAG,MAAM,EAAE,CAAC,CAAC;gBACzB,gBAAM,CAAC,IAAI,CAAC,0DAA0D,CAAC,CAAC;gBAExE,wCAAwC;gBACxC,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;oBAC5B,MAAM,EAAE,GAAG,uBAAQ,CAAC,eAAe,CAAC;wBAClC,KAAK,EAAE,OAAO,CAAC,KAAK;wBACpB,MAAM,EAAE,OAAO,CAAC,MAAM;qBACvB,CAAC,CAAC;oBACH,EAAE,CAAC,QAAQ,CACT,GAAG,eAAK,CAAC,IAAI,CAAC,kCAAkC,CAAC,UAAU,EAC3D,KAAK,EAAE,MAAM,EAAE,EAAE;wBACf,EAAE,CAAC,KAAK,EAAE,CAAC;wBACX,IAAI,MAAM,CAAC,WAAW,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE;4BACxC,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;4BACzD,QAAQ,EAAE,CAAC;yBACZ;6BAAM;4BACL,gBAAM,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;yBACtC;wBACD,OAAO,CAAC,IAAI,CAAC,CAAC;oBAChB,CAAC,CACF,CAAC;gBACJ,CAAC,CAAC,CAAC;aACJ;YAED,IAAI,QAAQ,GAAG,CAAC,EAAE;gBAChB,gBAAM,CAAC,IAAI,CAAC,eAAK,CAAC,GAAG,CAAC,gCAAgC,CAAC,CAAC,CAAC;gBACzD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;aACjB;SACF;QAED,4BAA4B;QAC5B,KAAK,MAAM,MAAM,IAAI,SAAS,CAAC,OAAO,EAAE;YACtC,KAAK,MAAM,QAAQ,IAAI,SAAS,CAAC,SAAS,EAAE;gBAC1C,MAAM,cAAc,GAClB,SAAS,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,QAAQ,CAAC,EAAE,EAAE,KAAK,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC;gBAC3F,OAAO,CAAC,IAAI,CAAC;oBACX,GAAG,MAAM;oBACT,OAAO,EAAE,cAAc;iBACxB,CAAC,CAAC;aACJ;SACF;QAED,uCAAuC;QAEvC,MAAM,KAAK,GAAG,CACZ,SAAS,CAAC,KAAK,IAAI;YACjB;YACE,8DAA8D;aAC/D;SACF,CACF,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;YACb,MAAM,aAAa,GAAa,MAAM,CAAC,MAAM,CAAC,EAAE,EAAE,SAAS,CAAC,WAAW,CAAC,CAAC;YACzE,OAAO,MAAM,CAAC,MAAM,CAAC,aAAa,EAAE,IAAI,CAAC,CAAC;QAC5C,CAAC,CAAC,CAAC;QAEH,MAAM,QAAQ,GAAgB,IAAI,GAAG,EAAE,CAAC;QACxC,MAAM,0BAA0B,GAAwC,EAAE,CAAC;QAC3E,KAAK,MAAM,QAAQ,IAAI,KAAK,EAAE;YAC5B,IAAI,QAAQ,CAAC,IAAI,EAAE;gBACjB,MAAM,yBAAyB,GAAsC,EAAE,CAAC;gBACxE,KAAK,MAAM,OAAO,IAAI,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE;oBAChD,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;oBACtB,yBAAyB,CAAC,OAAO,CAAC,GAAG,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;iBAC7D;gBACD,0BAA0B,CAAC,IAAI,CAAC,yBAAyB,CAAC,CAAC;aAC5D;SACF;QAED,kBAAkB;QAClB,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;QAE7C,MAAM,KAAK,GAAkB;YAC3B,IAAI,EAAE;gBACJ,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC;gBACtC,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,IAAI,EAAE;gBACjC,sCAAsC;aACvC;YACD,IAAI,EAAE,EAAE;SACT,CAAC;QAEF,sBAAsB;QACtB,IAAI,WAAkC,CAAC;QACvC,IAAI,OAAO,CAAC,eAAe,EAAE;YAC3B,uCAAuC;YACvC,MAAM,YAAY,GAChB,SAAS,CAAC,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,KAAK,CAAC,MAAM,IAAI,CAAC,CAAC,CAAC;YAC9E,MAAM,WAAW,GAAG,wDAAa,cAAc,GAAC,CAAC;YACjD,WAAW,GAAG,IAAI,WAAW,CAAC,SAAS,CACrC;gBACE,MAAM,EACJ,4FAA4F;aAC/F,EACD,WAAW,CAAC,OAAO,CAAC,cAAc,CACnC,CAAC;YACF,WAAW,CAAC,KAAK,CAAC,YAAY,EAAE,CAAC,EAAE;gBACjC,QAAQ,EAAE,EAAE;gBACZ,MAAM,EAAE,EAAE;gBACV,IAAI,EAAE,EAAE;aACT,CAAC,CAAC;SACJ;QAED,oBAAoB;QACpB,MAAM,cAAc,GAAqB,EAAE,CAAC;QAC5C,IAAI,QAAQ,GAAG,CAAC,CAAC;QACjB,KAAK,MAAM,QAAQ,IAAI,KAAK,EAAE;YAC5B,4BAA4B;YAC5B,QAAQ,CAAC,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,EAAE,EAAE,SAAS,CAAC,WAAW,EAAE,IAAI,EAAE,QAAQ,CAAC,IAAI,CAAC,CAAC;YAC9E,QAAQ,CAAC,MAAM,GAAG,CAAC,GAAG,CAAC,SAAS,CAAC,WAAW,EAAE,MAAM,IAAI,EAAE,CAAC,EAAE,GAAG,CAAC,QAAQ,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,CAAC;YACzF,QAAQ,CAAC,OAAO,GAAG,QAAQ,CAAC,OAAO,IAAI,EAAE,CAAC;YAC1C,QAAQ,CAAC,OAAO,CAAC,QAAQ;gBACvB,QAAQ,CAAC,OAAO,CAAC,QAAQ,IAAI,SAAS,CAAC,WAAW,EAAE,OAAO,EAAE,QAAQ,CAAC;YACxE,MAAM,eAAe,GACnB,QAAQ,CAAC,OAAO,EAAE,MAAM,IAAI,SAAS,CAAC,WAAW,EAAE,OAAO,EAAE,MAAM,IAAI,EAAE,CAAC;YAC3E,MAAM,cAAc,GAClB,QAAQ,CAAC,OAAO,EAAE,MAAM,IAAI,SAAS,CAAC,WAAW,EAAE,OAAO,EAAE,MAAM,IAAI,EAAE,CAAC;YAE3E,0BAA0B;YAC1B,MAAM,eAAe,GAAG,uBAAuB,CAAC,QAAQ,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC;YACrE,KAAK,MAAM,IAAI,IAAI,eAAe,EAAE;gBAClC,IAAI,QAAQ,GAAG,CAAC,CAAC;gBACjB,KAAK,MAAM,MAAM,IAAI,SAAS,CAAC,OAAO,EAAE;oBACtC,KAAK,MAAM,QAAQ,IAAI,SAAS,CAAC,SAAS,EAAE;wBAC1C,cAAc,CAAC,IAAI,CAAC;4BAClB,QAAQ;4BACR,MAAM,EAAE;gCACN,GAAG,MAAM;gCACT,GAAG,EAAE,eAAe,GAAG,MAAM,CAAC,GAAG,GAAG,cAAc;6BACnD;4BACD,IAAI,EAAE,EAAE,GAAG,QAAQ,EAAE,IAAI,EAAE;4BAC3B,iBAAiB,EAAE,SAAS,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC;4BACjD,QAAQ;4BACR,QAAQ;yBACT,CAAC,CAAC;wBACH,QAAQ,EAAE,CAAC;qBACZ;iBACF;gBACD,QAAQ,EAAE,CAAC;aACZ;SACF;QAED,wBAAwB;QACxB,MAAM,OAAO,GAAqB,EAAE,CAAC;QACrC,MAAM,eAAK,CAAC,cAAc,CACxB,cAAc,EACd,OAAO,CAAC,cAAc,IAAI,uBAAuB,EACjD,KAAK,EAAE,OAAuB,EAAE,KAAsB,EAAE,EAAE;YACxD,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;YAExC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAElB,IAAI,WAAW,EAAE;gBACf,WAAW,CAAC,SAAS,CAAC;oBACpB,QAAQ,EAAE,OAAO,CAAC,QAAQ,CAAC,EAAE,EAAE;oBAC/B,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC;oBACvC,IAAI,EAAE,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;yBAC1C,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;yBAC5B,IAAI,CAAC,GAAG,CAAC;yBACT,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC;iBAChB,CAAC,CAAC;aACJ;YAED,wBAAwB;YACxB,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE;gBAC7B,MAAM,IAAI,KAAK,CAAC,+BAA+B,CAAC,CAAC;aAClD;YAED,IAAI,UAA8B,CAAC;YACnC,IAAI,MAAM,EAAE;gBACV,IAAI,GAAG,CAAC,OAAO,EAAE;oBACf,UAAU,GAAG,UAAU,GAAG,CAAC,QAAQ,EAAE,MAAM,IAAI,GAAG,CAAC,KAAK,IAAI,EAAE,EAAE,CAAC;iBAClE;qBAAM;oBACL,UAAU,GAAG,UAAU,GAAG,CAAC,KAAK,UAAU,GAAG,CAAC,QAAQ,EAAE,MAAM,IAAI,GAAG,CAAC,KAAK,IAAI,EAAE,EAAE,CAAC;iBACrF;aACF;iBAAM,IAAI,GAAG,CAAC,KAAK,EAAE;gBACpB,UAAU,GAAG,UAAU,GAAG,CAAC,KAAK,EAAE,CAAC;aACpC;iBAAM;gBACL,UAAU,GAAG,GAAG,CAAC,QAAQ,EAAE,MAAM,IAAI,GAAG,CAAC,KAAK,IAAI,EAAE,CAAC;aACtD;YAED,sEAAsE;YACtE,uCAAuC;YACvC,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,GAAG,OAAO,CAAC;YACvC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE;gBACzB,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG;oBACrB,OAAO,EAAE,EAAE;oBACX,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE;iBAClF,CAAC;aACH;YACD,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG,UAAU,CAAC;QACtD,CAAC,CACF,CAAC;QAEF,IAAI,WAAW,EAAE;YACf,WAAW,CAAC,IAAI,EAAE,CAAC;SACpB;QAED,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,KAAK,EAAE,CAAC;IAC3D,CAAC;CACF;AAED,SAAgB,QAAQ,CAAC,SAAoB,EAAE,OAAwB;IACrE,MAAM,EAAE,GAAG,IAAI,SAAS,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;IAC7C,OAAO,EAAE,CAAC,QAAQ,EAAE,CAAC;AACvB,CAAC;AAHD,4BAGC"}
|
package/dist/index.d.ts
CHANGED
|
@@ -1,21 +1,21 @@
|
|
|
1
|
-
import type { EvaluateOptions, TestSuiteConfig } from './types
|
|
2
|
-
export * from './types
|
|
1
|
+
import type { EvaluateOptions, TestSuiteConfig } from './types';
|
|
2
|
+
export * from './types';
|
|
3
3
|
interface EvaluateTestSuite extends TestSuiteConfig {
|
|
4
4
|
prompts: string[];
|
|
5
5
|
}
|
|
6
|
-
declare function evaluate(testSuite: EvaluateTestSuite, options?: EvaluateOptions): Promise<import("./types
|
|
6
|
+
declare function evaluate(testSuite: EvaluateTestSuite, options?: EvaluateOptions): Promise<import("./types").EvaluateSummary>;
|
|
7
7
|
declare const _default: {
|
|
8
8
|
evaluate: typeof evaluate;
|
|
9
9
|
assertions: {
|
|
10
|
-
matchesSimilarity: typeof import("./assertions
|
|
11
|
-
matchesLlmRubric: typeof import("./assertions
|
|
10
|
+
matchesSimilarity: typeof import("./assertions").matchesSimilarity;
|
|
11
|
+
matchesLlmRubric: typeof import("./assertions").matchesLlmRubric;
|
|
12
12
|
};
|
|
13
13
|
providers: {
|
|
14
|
-
OpenAiCompletionProvider: typeof import("./providers/openai
|
|
15
|
-
OpenAiChatCompletionProvider: typeof import("./providers/openai
|
|
16
|
-
LocalAiCompletionProvider: typeof import("./providers/localai
|
|
17
|
-
LocalAiChatProvider: typeof import("./providers/localai
|
|
18
|
-
loadApiProvider: typeof import("./providers
|
|
14
|
+
OpenAiCompletionProvider: typeof import("./providers/openai").OpenAiCompletionProvider;
|
|
15
|
+
OpenAiChatCompletionProvider: typeof import("./providers/openai").OpenAiChatCompletionProvider;
|
|
16
|
+
LocalAiCompletionProvider: typeof import("./providers/localai").LocalAiCompletionProvider;
|
|
17
|
+
LocalAiChatProvider: typeof import("./providers/localai").LocalAiChatProvider;
|
|
18
|
+
loadApiProvider: typeof import("./providers").loadApiProvider;
|
|
19
19
|
};
|
|
20
20
|
};
|
|
21
21
|
export default _default;
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,eAAe,EAAa,eAAe,EAAE,MAAM,
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,eAAe,EAAa,eAAe,EAAE,MAAM,SAAS,CAAC;AAG3E,cAAc,SAAS,CAAC;AAExB,UAAU,iBAAkB,SAAQ,eAAe;IACjD,OAAO,EAAE,MAAM,EAAE,CAAC;CACnB;AAED,iBAAe,QAAQ,CAAC,SAAS,EAAE,iBAAiB,EAAE,OAAO,GAAE,eAAoB,8CAalF;;;;;;;;;;;;;;;AAQD,wBAIE"}
|
package/dist/index.js
CHANGED
|
@@ -17,29 +17,33 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
17
17
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
18
18
|
};
|
|
19
19
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
20
|
-
const
|
|
21
|
-
const
|
|
22
|
-
const
|
|
23
|
-
const
|
|
24
|
-
const
|
|
25
|
-
__exportStar(require("./types
|
|
20
|
+
const evaluator_1 = require("./evaluator");
|
|
21
|
+
const providers_1 = require("./providers");
|
|
22
|
+
const assertions_1 = __importDefault(require("./assertions"));
|
|
23
|
+
const providers_2 = __importDefault(require("./providers"));
|
|
24
|
+
const util_1 = require("./util");
|
|
25
|
+
__exportStar(require("./types"), exports);
|
|
26
26
|
async function evaluate(testSuite, options = {}) {
|
|
27
27
|
const constructedTestSuite = {
|
|
28
28
|
...testSuite,
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
29
|
+
providers: await (0, providers_1.loadApiProviders)(testSuite.providers),
|
|
30
|
+
tests: await (0, util_1.readTests)(testSuite.tests),
|
|
31
|
+
// Full prompts expected (not filepaths)
|
|
32
|
+
prompts: testSuite.prompts.map((promptContent) => ({
|
|
33
|
+
raw: promptContent,
|
|
34
|
+
display: promptContent,
|
|
35
|
+
})),
|
|
32
36
|
};
|
|
33
|
-
return (0,
|
|
37
|
+
return (0, evaluator_1.evaluate)(constructedTestSuite, options);
|
|
34
38
|
}
|
|
35
39
|
module.exports = {
|
|
36
40
|
evaluate,
|
|
37
|
-
assertions:
|
|
38
|
-
providers:
|
|
41
|
+
assertions: assertions_1.default,
|
|
42
|
+
providers: providers_2.default,
|
|
39
43
|
};
|
|
40
44
|
exports.default = {
|
|
41
45
|
evaluate,
|
|
42
|
-
assertions:
|
|
43
|
-
providers:
|
|
46
|
+
assertions: assertions_1.default,
|
|
47
|
+
providers: providers_2.default,
|
|
44
48
|
};
|
|
45
49
|
//# sourceMappingURL=index.js.map
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;AAAA,
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;AAAA,2CAAqD;AACrD,2CAA+C;AAC/C,8DAAsC;AACtC,4DAAoC;AAGpC,iCAAmC;AAEnC,0CAAwB;AAMxB,KAAK,UAAU,QAAQ,CAAC,SAA4B,EAAE,UAA2B,EAAE;IACjF,MAAM,oBAAoB,GAAc;QACtC,GAAG,SAAS;QACZ,SAAS,EAAE,MAAM,IAAA,4BAAgB,EAAC,SAAS,CAAC,SAAS,CAAC;QACtD,KAAK,EAAE,MAAM,IAAA,gBAAS,EAAC,SAAS,CAAC,KAAK,CAAC;QAEvC,wCAAwC;QACxC,OAAO,EAAE,SAAS,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,aAAa,EAAE,EAAE,CAAC,CAAC;YACjD,GAAG,EAAE,aAAa;YAClB,OAAO,EAAE,aAAa;SACvB,CAAC,CAAC;KACJ,CAAC;IACF,OAAO,IAAA,oBAAU,EAAC,oBAAoB,EAAE,OAAO,CAAC,CAAC;AACnD,CAAC;AAED,MAAM,CAAC,OAAO,GAAG;IACf,QAAQ;IACR,UAAU,EAAV,oBAAU;IACV,SAAS,EAAT,mBAAS;CACV,CAAC;AAEF,kBAAe;IACb,QAAQ;IACR,UAAU,EAAV,oBAAU;IACV,SAAS,EAAT,mBAAS;CACV,CAAC"}
|
package/dist/main.js
CHANGED
|
@@ -32,14 +32,14 @@ const path_1 = require("path");
|
|
|
32
32
|
const cli_table3_1 = __importDefault(require("cli-table3"));
|
|
33
33
|
const chalk_1 = __importDefault(require("chalk"));
|
|
34
34
|
const commander_1 = require("commander");
|
|
35
|
-
const
|
|
36
|
-
const
|
|
37
|
-
const
|
|
38
|
-
const
|
|
39
|
-
const
|
|
40
|
-
const
|
|
41
|
-
const
|
|
42
|
-
const
|
|
35
|
+
const logger_1 = __importStar(require("./logger"));
|
|
36
|
+
const providers_1 = require("./providers");
|
|
37
|
+
const evaluator_1 = require("./evaluator");
|
|
38
|
+
const util_1 = require("./util");
|
|
39
|
+
const esm_1 = require("./esm");
|
|
40
|
+
const server_1 = require("./web/server");
|
|
41
|
+
const cache_1 = require("./cache");
|
|
42
|
+
const onboarding_1 = require("./onboarding");
|
|
43
43
|
function createDummyFiles(directory) {
|
|
44
44
|
if (directory) {
|
|
45
45
|
// Make the directory if it doesn't exist
|
|
@@ -49,22 +49,22 @@ function createDummyFiles(directory) {
|
|
|
49
49
|
}
|
|
50
50
|
if (directory) {
|
|
51
51
|
if (!(0, fs_1.existsSync)(directory)) {
|
|
52
|
-
|
|
52
|
+
logger_1.default.info(`Creating directory ${directory} ...`);
|
|
53
53
|
(0, fs_1.mkdirSync)(directory);
|
|
54
54
|
}
|
|
55
55
|
}
|
|
56
56
|
else {
|
|
57
57
|
directory = '.';
|
|
58
58
|
}
|
|
59
|
-
(0, fs_1.writeFileSync)((0, path_1.join)(process.cwd(), directory, 'prompts.txt'),
|
|
60
|
-
(0, fs_1.writeFileSync)((0, path_1.join)(process.cwd(), directory, 'promptfooconfig.yaml'),
|
|
61
|
-
(0, fs_1.writeFileSync)((0, path_1.join)(process.cwd(), directory, 'README.md'),
|
|
59
|
+
(0, fs_1.writeFileSync)((0, path_1.join)(process.cwd(), directory, 'prompts.txt'), onboarding_1.DEFAULT_PROMPTS);
|
|
60
|
+
(0, fs_1.writeFileSync)((0, path_1.join)(process.cwd(), directory, 'promptfooconfig.yaml'), onboarding_1.DEFAULT_YAML_CONFIG);
|
|
61
|
+
(0, fs_1.writeFileSync)((0, path_1.join)(process.cwd(), directory, 'README.md'), onboarding_1.DEFAULT_README);
|
|
62
62
|
if (directory === '.') {
|
|
63
|
-
|
|
63
|
+
logger_1.default.info('Wrote prompts.txt and promptfooconfig.js. Open README.md to get started!');
|
|
64
64
|
}
|
|
65
65
|
else {
|
|
66
|
-
|
|
67
|
-
|
|
66
|
+
logger_1.default.info(`Wrote prompts.txt and promptfooconfig.js to ./${directory}`);
|
|
67
|
+
logger_1.default.info(`\`cd ${directory}\` and open README.md to get started!`);
|
|
68
68
|
}
|
|
69
69
|
}
|
|
70
70
|
async function main() {
|
|
@@ -76,7 +76,7 @@ async function main() {
|
|
|
76
76
|
];
|
|
77
77
|
let config = {};
|
|
78
78
|
for (const path of potentialPaths) {
|
|
79
|
-
const maybeConfig = (0,
|
|
79
|
+
const maybeConfig = await (0, util_1.maybeReadConfig)(path);
|
|
80
80
|
if (maybeConfig) {
|
|
81
81
|
config = maybeConfig;
|
|
82
82
|
break;
|
|
@@ -90,7 +90,7 @@ async function main() {
|
|
|
90
90
|
}
|
|
91
91
|
const program = new commander_1.Command();
|
|
92
92
|
program.option('--version', 'Print version', () => {
|
|
93
|
-
const packageJson = JSON.parse((0, fs_1.readFileSync)((0, path_1.join)((0,
|
|
93
|
+
const packageJson = JSON.parse((0, fs_1.readFileSync)((0, path_1.join)((0, esm_1.getDirectory)(), '../package.json'), 'utf8'));
|
|
94
94
|
console.log(packageJson.version);
|
|
95
95
|
process.exit(0);
|
|
96
96
|
});
|
|
@@ -105,7 +105,7 @@ async function main() {
|
|
|
105
105
|
.description('Start browser ui')
|
|
106
106
|
.option('-p, --port <number>', 'Port number', '15500')
|
|
107
107
|
.action((cmdObj) => {
|
|
108
|
-
(0,
|
|
108
|
+
(0, server_1.init)(cmdObj.port);
|
|
109
109
|
});
|
|
110
110
|
program
|
|
111
111
|
.command('eval')
|
|
@@ -125,47 +125,48 @@ async function main() {
|
|
|
125
125
|
.option('--suggest-prompts <number>', 'Generate N new prompts and append them to the prompt list')
|
|
126
126
|
.option('--prompt-prefix <path>', 'This prefix is prepended to every prompt', config.defaultTest?.options?.prefix)
|
|
127
127
|
.option('--prompt-suffix <path>', 'This suffix is append to every prompt', config.defaultTest?.options?.suffix)
|
|
128
|
-
.option('--no-write', 'Do not write results to promptfoo directory')
|
|
129
|
-
.option('--no-cache', 'Do not read or write results to disk cache')
|
|
128
|
+
.option('--no-write', 'Do not write results to promptfoo directory', config?.commandLineOptions?.write)
|
|
129
|
+
.option('--no-cache', 'Do not read or write results to disk cache', config?.commandLineOptions?.cache)
|
|
130
130
|
.option('--grader', 'Model that will grade outputs', config?.commandLineOptions?.grader)
|
|
131
131
|
.option('--verbose', 'Show debug logs', config?.commandLineOptions?.verbose)
|
|
132
132
|
.option('--view [port]', 'View in browser ui')
|
|
133
133
|
.action(async (cmdObj) => {
|
|
134
134
|
// Misc settings
|
|
135
135
|
if (cmdObj.verbose) {
|
|
136
|
-
(0,
|
|
136
|
+
(0, logger_1.setLogLevel)('debug');
|
|
137
137
|
}
|
|
138
138
|
if (!cmdObj.cache) {
|
|
139
|
-
(0,
|
|
139
|
+
(0, cache_1.disableCache)();
|
|
140
140
|
}
|
|
141
141
|
// Config parsing
|
|
142
142
|
const maxConcurrency = parseInt(cmdObj.maxConcurrency || '', 10);
|
|
143
143
|
const configPath = cmdObj.config;
|
|
144
144
|
if (configPath) {
|
|
145
|
-
config = (0,
|
|
145
|
+
config = await (0, util_1.readConfig)(configPath);
|
|
146
146
|
}
|
|
147
147
|
else {
|
|
148
148
|
config = {
|
|
149
149
|
prompts: cmdObj.prompts || config.prompts,
|
|
150
150
|
providers: cmdObj.providers || config.providers,
|
|
151
151
|
tests: cmdObj.tests || cmdObj.vars || config.tests,
|
|
152
|
+
defaultTest: config.defaultTest,
|
|
152
153
|
};
|
|
153
154
|
}
|
|
154
155
|
// Validation
|
|
155
156
|
if (!config.prompts || config.prompts.length === 0) {
|
|
156
|
-
|
|
157
|
+
logger_1.default.error(chalk_1.default.red('You must provide at least 1 prompt file'));
|
|
157
158
|
process.exit(1);
|
|
158
159
|
}
|
|
159
160
|
if (!config.providers || config.providers.length === 0) {
|
|
160
|
-
|
|
161
|
+
logger_1.default.error(chalk_1.default.red('You must specify at least 1 provider (for example, openai:gpt-3.5-turbo)'));
|
|
161
162
|
process.exit(1);
|
|
162
163
|
}
|
|
163
164
|
// Parse prompts, providers, and tests
|
|
164
|
-
const parsedPrompts = (0,
|
|
165
|
-
const parsedProviders = await (0,
|
|
166
|
-
const parsedTests = await (0,
|
|
165
|
+
const parsedPrompts = (0, util_1.readPrompts)(config.prompts);
|
|
166
|
+
const parsedProviders = await (0, providers_1.loadApiProviders)(config.providers);
|
|
167
|
+
const parsedTests = await (0, util_1.readTests)(config.tests);
|
|
167
168
|
if (parsedPrompts.length === 0) {
|
|
168
|
-
|
|
169
|
+
logger_1.default.error(chalk_1.default.red('No prompts found'));
|
|
169
170
|
process.exit(1);
|
|
170
171
|
}
|
|
171
172
|
const defaultTest = {
|
|
@@ -191,15 +192,15 @@ async function main() {
|
|
|
191
192
|
};
|
|
192
193
|
if (cmdObj.grader && testSuite.defaultTest) {
|
|
193
194
|
testSuite.defaultTest.options = testSuite.defaultTest.options || {};
|
|
194
|
-
testSuite.defaultTest.options.provider = await (0,
|
|
195
|
+
testSuite.defaultTest.options.provider = await (0, providers_1.loadApiProvider)(cmdObj.grader);
|
|
195
196
|
}
|
|
196
197
|
if (cmdObj.generateSuggestions) {
|
|
197
198
|
options.generateSuggestions = true;
|
|
198
199
|
}
|
|
199
|
-
const summary = await (0,
|
|
200
|
+
const summary = await (0, evaluator_1.evaluate)(testSuite, options);
|
|
200
201
|
if (cmdObj.output) {
|
|
201
|
-
|
|
202
|
-
(0,
|
|
202
|
+
logger_1.default.info(chalk_1.default.yellow(`Writing output to ${cmdObj.output}`));
|
|
203
|
+
(0, util_1.writeOutput)(cmdObj.output, summary);
|
|
203
204
|
}
|
|
204
205
|
else {
|
|
205
206
|
// Output table by default
|
|
@@ -216,8 +217,9 @@ async function main() {
|
|
|
216
217
|
},
|
|
217
218
|
});
|
|
218
219
|
// Skip first row (header) and add the rest. Color PASS/FAIL
|
|
219
|
-
for (const row of summary.table.body) {
|
|
220
|
+
for (const row of summary.table.body.slice(0, 25)) {
|
|
220
221
|
table.push([
|
|
222
|
+
...row.vars,
|
|
221
223
|
...row.outputs.map((col) => {
|
|
222
224
|
const tableCellMaxLength = parseInt(cmdObj.tableCellMaxLength || '', 10);
|
|
223
225
|
if (!isNaN(tableCellMaxLength) && col.length > tableCellMaxLength) {
|
|
@@ -236,24 +238,27 @@ async function main() {
|
|
|
236
238
|
}
|
|
237
239
|
return col;
|
|
238
240
|
}),
|
|
239
|
-
...row.vars,
|
|
240
241
|
]);
|
|
241
242
|
}
|
|
242
|
-
|
|
243
|
+
logger_1.default.info('\n' + table.toString());
|
|
244
|
+
if (summary.table.body.length > 25) {
|
|
245
|
+
const rowsLeft = summary.table.body.length - 25;
|
|
246
|
+
logger_1.default.info(`... ${rowsLeft} more row${rowsLeft === 1 ? '' : 's'} not shown ...\n`);
|
|
247
|
+
}
|
|
243
248
|
}
|
|
244
249
|
if (cmdObj.view || !cmdObj.write) {
|
|
245
|
-
|
|
250
|
+
logger_1.default.info('Evaluation complete');
|
|
246
251
|
}
|
|
247
252
|
else {
|
|
248
|
-
(0,
|
|
249
|
-
|
|
253
|
+
(0, util_1.writeLatestResults)(summary);
|
|
254
|
+
logger_1.default.info(`Evaluation complete. To use web viewer, run ${chalk_1.default.green('promptfoo view')}`);
|
|
250
255
|
}
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
256
|
+
logger_1.default.info(chalk_1.default.green.bold(`Successes: ${summary.stats.successes}`));
|
|
257
|
+
logger_1.default.info(chalk_1.default.red.bold(`Failures: ${summary.stats.failures}`));
|
|
258
|
+
logger_1.default.info(`Token usage: Total ${summary.stats.tokenUsage.total}, Prompt ${summary.stats.tokenUsage.prompt}, Completion ${summary.stats.tokenUsage.completion}, Cached ${summary.stats.tokenUsage.cached}`);
|
|
259
|
+
logger_1.default.info('Done.');
|
|
255
260
|
if (cmdObj.view) {
|
|
256
|
-
(0,
|
|
261
|
+
(0, server_1.init)(parseInt(cmdObj.view, 10) || 15500);
|
|
257
262
|
}
|
|
258
263
|
});
|
|
259
264
|
program.parse(process.argv);
|
package/dist/main.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"main.js","sourceRoot":"","sources":["../src/main.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;AACA,2BAAwE;AACxE,+BAAwC;AAExC,4DAA+B;AAC/B,kDAA0B;AAC1B,yCAAoC;AAEpC,
|
|
1
|
+
{"version":3,"file":"main.js","sourceRoot":"","sources":["../src/main.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;AACA,2BAAwE;AACxE,+BAAwC;AAExC,4DAA+B;AAC/B,kDAA0B;AAC1B,yCAAoC;AAEpC,mDAA+C;AAC/C,2CAAgE;AAChE,2CAAuC;AACvC,iCAOgB;AAChB,+BAAqC;AACrC,yCAAoC;AACpC,mCAAuC;AASvC,6CAAoF;AAEpF,SAAS,gBAAgB,CAAC,SAAwB;IAChD,IAAI,SAAS,EAAE;QACb,yCAAyC;QACzC,IAAI,CAAC,IAAA,eAAU,EAAC,SAAS,CAAC,EAAE;YAC1B,IAAA,cAAS,EAAC,SAAS,CAAC,CAAC;SACtB;KACF;IAED,IAAI,SAAS,EAAE;QACb,IAAI,CAAC,IAAA,eAAU,EAAC,SAAS,CAAC,EAAE;YAC1B,gBAAM,CAAC,IAAI,CAAC,sBAAsB,SAAS,MAAM,CAAC,CAAC;YACnD,IAAA,cAAS,EAAC,SAAS,CAAC,CAAC;SACtB;KACF;SAAM;QACL,SAAS,GAAG,GAAG,CAAC;KACjB;IAED,IAAA,kBAAa,EAAC,IAAA,WAAQ,EAAC,OAAO,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,aAAa,CAAC,EAAE,4BAAe,CAAC,CAAC;IAClF,IAAA,kBAAa,EAAC,IAAA,WAAQ,EAAC,OAAO,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,sBAAsB,CAAC,EAAE,gCAAmB,CAAC,CAAC;IAC/F,IAAA,kBAAa,EAAC,IAAA,WAAQ,EAAC,OAAO,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,WAAW,CAAC,EAAE,2BAAc,CAAC,CAAC;IAE/E,IAAI,SAAS,KAAK,GAAG,EAAE;QACrB,gBAAM,CAAC,IAAI,CAAC,0EAA0E,CAAC,CAAC;KACzF;SAAM;QACL,gBAAM,CAAC,IAAI,CAAC,iDAAiD,SAAS,EAAE,CAAC,CAAC;QAC1E,gBAAM,CAAC,IAAI,CAAC,QAAQ,SAAS,uCAAuC,CAAC,CAAC;KACvE;AACH,CAAC;AAED,KAAK,UAAU,IAAI;IACjB,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC;IAC1B,MAAM,cAAc,GAAG;QACrB,IAAA,WAAQ,EAAC,GAAG,EAAE,oBAAoB,CAAC;QACnC,IAAA,WAAQ,EAAC,GAAG,EAAE,sBAAsB,CAAC;QACrC,IAAA,WAAQ,EAAC,GAAG,EAAE,sBAAsB,CAAC;KACtC,CAAC;IACF,IAAI,MAAM,GAA2B,EAAE,CAAC;IACxC,KAAK,MAAM,IAAI,IAAI,cAAc,EAAE;QACjC,MAAM,WAAW,GAAG,MAAM,IAAA,sBAAe,EAAC,IAAI,CAAC,CAAC;QAChD,IAAI,WAAW,EAAE;YACf,MAAM,GAAG,WAAW,CAAC;YACrB,MAAM;SACP;KACF;IAED,IAAI,eAAe,GAAoB,EAAE,CAAC;IAC1C,IAAI,MAAM,CAAC,eAAe,EAAE;QAC1B,eAAe,CAAC,mBAAmB,GAAG,MAAM,CAAC,eAAe,CAAC,mBAAmB,CAAC;QACjF,eAAe,CAAC,cAAc,GAAG,MAAM,CAAC,eAAe,CAAC,cAAc,CAAC;QACvE,eAAe,CAAC,eAAe,GAAG,MAAM,CAAC,eAAe,CAAC,eAAe,CAAC;KAC1E;IAED,MAAM,OAAO,GAAG,IAAI,mBAAO,EAAE,CAAC;IAE9B,OAAO,CAAC,MAAM,CAAC,WAAW,EAAE,eAAe,EAAE,GAAG,EAAE;QAChD,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAC5B,IAAA,iBAAY,EAAC,IAAA,WAAQ,EAAC,IAAA,kBAAY,GAAE,EAAE,iBAAiB,CAAC,EAAE,MAAM,CAAC,CAClE,CAAC;QACF,OAAO,CAAC,GAAG,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;QACjC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,CAAC;IAEH,OAAO;SACJ,OAAO,CAAC,kBAAkB,CAAC;SAC3B,WAAW,CAAC,qCAAqC,CAAC;SAClD,MAAM,CAAC,CAAC,SAAwB,EAAE,EAAE;QACnC,gBAAgB,CAAC,SAAS,CAAC,CAAC;IAC9B,CAAC,CAAC,CAAC;IAEL,OAAO;SACJ,OAAO,CAAC,MAAM,CAAC;SACf,WAAW,CAAC,kBAAkB,CAAC;SAC/B,MAAM,CAAC,qBAAqB,EAAE,aAAa,EAAE,OAAO,CAAC;SACrD,MAAM,CAAC,CAAC,MAAkC,EAAE,EAAE;QAC7C,IAAA,aAAI,EAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACpB,CAAC,CAAC,CAAC;IAEL,OAAO;SACJ,OAAO,CAAC,MAAM,CAAC;SACf,WAAW,CAAC,kBAAkB,CAAC;SAC/B,cAAc,CAAC,0BAA0B,EAAE,8BAA8B,EAAE,MAAM,CAAC,OAAO,CAAC;SAC1F,cAAc,CACb,mCAAmC,EACnC,kGAAkG,EAClG,MAAM,EAAE,SAAS,CAClB;SACA,MAAM,CACL,qBAAqB,EACrB,8EAA8E,CAC/E;SACA,MAAM;IACL,kCAAkC;IAClC,gCAAgC,EAChC,6BAA6B,EAC7B,MAAM,EAAE,kBAAkB,EAAE,IAAI,CACjC;SACA,MAAM,CAAC,oBAAoB,EAAE,6BAA6B,EAAE,MAAM,EAAE,kBAAkB,EAAE,KAAK,CAAC;SAC9F,MAAM,CAAC,qBAAqB,EAAE,6CAA6C,EAAE,MAAM,CAAC,UAAU,CAAC;SAC/F,MAAM,CACL,gCAAgC,EAChC,wCAAwC,EACxC,MAAM,CAAC,eAAe,EAAE,cAAc;QACpC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,eAAe,CAAC,cAAc,CAAC;QAC/C,CAAC,CAAC,SAAS,CACd;SACA,MAAM,CACL,kCAAkC,EAClC,6CAA6C,EAC7C,KAAK,CACN;SACA,MAAM,CACL,4BAA4B,EAC5B,2DAA2D,CAC5D;SACA,MAAM,CACL,wBAAwB,EACxB,0CAA0C,EAC1C,MAAM,CAAC,WAAW,EAAE,OAAO,EAAE,MAAM,CACpC;SACA,MAAM,CACL,wBAAwB,EACxB,uCAAuC,EACvC,MAAM,CAAC,WAAW,EAAE,OAAO,EAAE,MAAM,CACpC;SACA,MAAM,CACL,YAAY,EACZ,6CAA6C,EAC7C,MAAM,EAAE,kBAAkB,EAAE,KAAK,CAClC;SACA,MAAM,CACL,YAAY,EACZ,4CAA4C,EAC5C,MAAM,EAAE,kBAAkB,EAAE,KAAK,CAClC;SACA,MAAM,CAAC,UAAU,EAAE,+BAA+B,EAAE,MAAM,EAAE,kBAAkB,EAAE,MAAM,CAAC;SACvF,MAAM,CAAC,WAAW,EAAE,iBAAiB,EAAE,MAAM,EAAE,kBAAkB,EAAE,OAAO,CAAC;SAC3E,MAAM,CAAC,eAAe,EAAE,oBAAoB,CAAC;SAC7C,MAAM,CAAC,KAAK,EAAE,MAAoC,EAAE,EAAE;QACrD,gBAAgB;QAChB,IAAI,MAAM,CAAC,OAAO,EAAE;YAClB,IAAA,oBAAW,EAAC,OAAO,CAAC,CAAC;SACtB;QACD,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE;YACjB,IAAA,oBAAY,GAAE,CAAC;SAChB;QAED,iBAAiB;QACjB,MAAM,cAAc,GAAG,QAAQ,CAAC,MAAM,CAAC,cAAc,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC;QACjE,MAAM,UAAU,GAAG,MAAM,CAAC,MAAM,CAAC;QACjC,IAAI,UAAU,EAAE;YACd,MAAM,GAAG,MAAM,IAAA,iBAAU,EAAC,UAAU,CAAC,CAAC;SACvC;aAAM;YACL,MAAM,GAAG;gBACP,OAAO,EAAE,MAAM,CAAC,OAAO,IAAI,MAAM,CAAC,OAAO;gBACzC,SAAS,EAAE,MAAM,CAAC,SAAS,IAAI,MAAM,CAAC,SAAS;gBAC/C,KAAK,EAAE,MAAM,CAAC,KAAK,IAAI,MAAM,CAAC,IAAI,IAAI,MAAM,CAAC,KAAK;gBAClD,WAAW,EAAE,MAAM,CAAC,WAAW;aAChC,CAAC;SACH;QAED,aAAa;QACb,IAAI,CAAC,MAAM,CAAC,OAAO,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE;YAClD,gBAAM,CAAC,KAAK,CAAC,eAAK,CAAC,GAAG,CAAC,yCAAyC,CAAC,CAAC,CAAC;YACnE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;SACjB;QACD,IAAI,CAAC,MAAM,CAAC,SAAS,IAAI,MAAM,CAAC,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE;YACtD,gBAAM,CAAC,KAAK,CACV,eAAK,CAAC,GAAG,CAAC,0EAA0E,CAAC,CACtF,CAAC;YACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;SACjB;QAED,sCAAsC;QACtC,MAAM,aAAa,GAAG,IAAA,kBAAW,EAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QAClD,MAAM,eAAe,GAAG,MAAM,IAAA,4BAAgB,EAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QACjE,MAAM,WAAW,GAAe,MAAM,IAAA,gBAAS,EAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAE9D,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,EAAE;YAC9B,gBAAM,CAAC,KAAK,CAAC,eAAK,CAAC,GAAG,CAAC,kBAAkB,CAAC,CAAC,CAAC;YAC5C,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;SACjB;QAED,MAAM,WAAW,GAAa;YAC5B,OAAO,EAAE;gBACP,MAAM,EAAE,MAAM,CAAC,YAAY;gBAC3B,MAAM,EAAE,MAAM,CAAC,YAAY;gBAC3B,QAAQ,EAAE,MAAM,CAAC,MAAM;gBACvB,gBAAgB;aACjB;YACD,GAAG,MAAM,CAAC,WAAW;SACtB,CAAC;QAEF,MAAM,SAAS,GAAc;YAC3B,WAAW,EAAE,MAAM,CAAC,WAAW;YAC/B,OAAO,EAAE,aAAa;YACtB,SAAS,EAAE,eAAe;YAC1B,KAAK,EAAE,WAAW;YAClB,WAAW;SACZ,CAAC;QAEF,MAAM,OAAO,GAAoB;YAC/B,eAAe,EAAE,IAAI;YACrB,cAAc,EAAE,CAAC,KAAK,CAAC,cAAc,CAAC,IAAI,cAAc,GAAG,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,SAAS;YACzF,GAAG,eAAe;SACnB,CAAC;QAEF,IAAI,MAAM,CAAC,MAAM,IAAI,SAAS,CAAC,WAAW,EAAE;YAC1C,SAAS,CAAC,WAAW,CAAC,OAAO,GAAG,SAAS,CAAC,WAAW,CAAC,OAAO,IAAI,EAAE,CAAC;YACpE,SAAS,CAAC,WAAW,CAAC,OAAO,CAAC,QAAQ,GAAG,MAAM,IAAA,2BAAe,EAAC,MAAM,CAAC,MAAM,CAAC,CAAC;SAC/E;QACD,IAAI,MAAM,CAAC,mBAAmB,EAAE;YAC9B,OAAO,CAAC,mBAAmB,GAAG,IAAI,CAAC;SACpC;QAED,MAAM,OAAO,GAAG,MAAM,IAAA,oBAAQ,EAAC,SAAS,EAAE,OAAO,CAAC,CAAC;QAEnD,IAAI,MAAM,CAAC,MAAM,EAAE;YACjB,gBAAM,CAAC,IAAI,CAAC,eAAK,CAAC,MAAM,CAAC,qBAAqB,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;YAChE,IAAA,kBAAW,EAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;SACrC;aAAM;YACL,0BAA0B;YAC1B,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,GAAG,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC;YAC5E,MAAM,IAAI,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC;YAChC,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC;YAC1D,MAAM,KAAK,GAAG,IAAI,oBAAK,CAAC;gBACtB,IAAI,EAAE,CAAC,GAAG,IAAI,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,IAAI,CAAC;gBACrC,SAAS,EAAE,KAAK,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,GAAG,UAAU,CAAC,CAAC;gBACpE,QAAQ,EAAE,IAAI;gBACd,kBAAkB,EAAE,KAAK;gBACzB,KAAK,EAAE;oBACL,IAAI,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC;iBACvB;aACF,CAAC,CAAC;YACH,4DAA4D;YAC5D,KAAK,MAAM,GAAG,IAAI,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE;gBACjD,KAAK,CAAC,IAAI,CAAC;oBACT,GAAG,GAAG,CAAC,IAAI;oBACX,GAAG,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;wBACzB,MAAM,kBAAkB,GAAG,QAAQ,CAAC,MAAM,CAAC,kBAAkB,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC;wBACzE,IAAI,CAAC,KAAK,CAAC,kBAAkB,CAAC,IAAI,GAAG,CAAC,MAAM,GAAG,kBAAkB,EAAE;4BACjE,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,kBAAkB,CAAC,GAAG,KAAK,CAAC;yBAChD;wBACD,IAAI,GAAG,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE;4BAC5B,uBAAuB;4BACvB,OAAO,eAAK,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;yBACzD;6BAAM,IAAI,GAAG,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE;4BACnC,sCAAsC;4BACtC,OAAO,GAAG;iCACP,KAAK,CAAC,KAAK,CAAC;iCACZ,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,eAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;iCACpD,IAAI,CAAC,KAAK,CAAC,CAAC;yBAChB;wBACD,OAAO,GAAG,CAAC;oBACb,CAAC,CAAC;iBACH,CAAC,CAAC;aACJ;YAED,gBAAM,CAAC,IAAI,CAAC,IAAI,GAAG,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC;YACrC,IAAI,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,GAAG,EAAE,EAAE;gBAClC,MAAM,QAAQ,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,GAAG,EAAE,CAAC;gBAChD,gBAAM,CAAC,IAAI,CAAC,OAAO,QAAQ,YAAY,QAAQ,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,kBAAkB,CAAC,CAAC;aACrF;SACF;QACD,IAAI,MAAM,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE;YAChC,gBAAM,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;SACpC;aAAM;YACL,IAAA,yBAAkB,EAAC,OAAO,CAAC,CAAC;YAC5B,gBAAM,CAAC,IAAI,CAAC,+CAA+C,eAAK,CAAC,KAAK,CAAC,gBAAgB,CAAC,EAAE,CAAC,CAAC;SAC7F;QACD,gBAAM,CAAC,IAAI,CAAC,eAAK,CAAC,KAAK,CAAC,IAAI,CAAC,cAAc,OAAO,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC;QACvE,gBAAM,CAAC,IAAI,CAAC,eAAK,CAAC,GAAG,CAAC,IAAI,CAAC,aAAa,OAAO,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC;QACnE,gBAAM,CAAC,IAAI,CACT,sBAAsB,OAAO,CAAC,KAAK,CAAC,UAAU,CAAC,KAAK,YAAY,OAAO,CAAC,KAAK,CAAC,UAAU,CAAC,MAAM,gBAAgB,OAAO,CAAC,KAAK,CAAC,UAAU,CAAC,UAAU,YAAY,OAAO,CAAC,KAAK,CAAC,UAAU,CAAC,MAAM,EAAE,CAChM,CAAC;QACF,gBAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAErB,IAAI,MAAM,CAAC,IAAI,EAAE;YACf,IAAA,aAAI,EAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,KAAK,CAAC,CAAC;SAC1C;IACH,CAAC,CAAC,CAAC;IAEL,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IAE5B,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE;QACjC,OAAO,CAAC,UAAU,EAAE,CAAC;KACtB;AACH,CAAC;AAED,IAAI,EAAE,CAAC"}
|
|
@@ -4,9 +4,9 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
4
4
|
};
|
|
5
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
6
|
exports.LocalAiCompletionProvider = exports.LocalAiChatProvider = void 0;
|
|
7
|
-
const
|
|
8
|
-
const
|
|
9
|
-
const
|
|
7
|
+
const logger_1 = __importDefault(require("../logger"));
|
|
8
|
+
const cache_1 = require("../cache");
|
|
9
|
+
const shared_1 = require("./shared");
|
|
10
10
|
class LocalAiGenericProvider {
|
|
11
11
|
constructor(modelName) {
|
|
12
12
|
this.modelName = modelName;
|
|
@@ -30,23 +30,23 @@ class LocalAiChatProvider extends LocalAiGenericProvider {
|
|
|
30
30
|
prompt,
|
|
31
31
|
temperature: process.env.LOCALAI_TEMPERATURE || 0.7,
|
|
32
32
|
};
|
|
33
|
-
|
|
33
|
+
logger_1.default.debug(`Calling LocalAI API: ${JSON.stringify(body)}`);
|
|
34
34
|
let data, cached = false;
|
|
35
35
|
try {
|
|
36
|
-
({ data, cached } = (await (0,
|
|
36
|
+
({ data, cached } = (await (0, cache_1.fetchJsonWithCache)(`${this.apiBaseUrl}/chat/completions`, {
|
|
37
37
|
method: 'POST',
|
|
38
38
|
headers: {
|
|
39
39
|
'Content-Type': 'application/json',
|
|
40
40
|
},
|
|
41
41
|
body: JSON.stringify(body),
|
|
42
|
-
},
|
|
42
|
+
}, shared_1.REQUEST_TIMEOUT_MS)));
|
|
43
43
|
}
|
|
44
44
|
catch (err) {
|
|
45
45
|
return {
|
|
46
46
|
error: `API call error: ${String(err)}`,
|
|
47
47
|
};
|
|
48
48
|
}
|
|
49
|
-
|
|
49
|
+
logger_1.default.debug(` LocalAI API response: ${JSON.stringify(data)}`);
|
|
50
50
|
try {
|
|
51
51
|
return {
|
|
52
52
|
output: data.choices[0].text,
|
|
@@ -67,23 +67,23 @@ class LocalAiCompletionProvider extends LocalAiGenericProvider {
|
|
|
67
67
|
prompt,
|
|
68
68
|
temperature: process.env.LOCALAI_TEMPERATURE || 0.7,
|
|
69
69
|
};
|
|
70
|
-
|
|
70
|
+
logger_1.default.debug(`Calling LocalAI API: ${JSON.stringify(body)}`);
|
|
71
71
|
let data, cached = false;
|
|
72
72
|
try {
|
|
73
|
-
({ data, cached } = (await (0,
|
|
73
|
+
({ data, cached } = (await (0, cache_1.fetchJsonWithCache)(`${this.apiBaseUrl}/completions`, {
|
|
74
74
|
method: 'POST',
|
|
75
75
|
headers: {
|
|
76
76
|
'Content-Type': 'application/json',
|
|
77
77
|
},
|
|
78
78
|
body: JSON.stringify(body),
|
|
79
|
-
},
|
|
79
|
+
}, shared_1.REQUEST_TIMEOUT_MS)));
|
|
80
80
|
}
|
|
81
81
|
catch (err) {
|
|
82
82
|
return {
|
|
83
83
|
error: `API call error: ${String(err)}`,
|
|
84
84
|
};
|
|
85
85
|
}
|
|
86
|
-
|
|
86
|
+
logger_1.default.debug(` LocalAI API response: ${JSON.stringify(data)}`);
|
|
87
87
|
try {
|
|
88
88
|
return {
|
|
89
89
|
output: data.choices[0].text,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"localai.js","sourceRoot":"","sources":["../../src/providers/localai.ts"],"names":[],"mappings":";;;;;;AAAA,
|
|
1
|
+
{"version":3,"file":"localai.js","sourceRoot":"","sources":["../../src/providers/localai.ts"],"names":[],"mappings":";;;;;;AAAA,uDAA+B;AAC/B,oCAA8C;AAC9C,qCAA8C;AAI9C,MAAM,sBAAsB;IAI1B,YAAY,SAAiB;QAC3B,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,UAAU,GAAG,OAAO,CAAC,GAAG,CAAC,gBAAgB,IAAI,0BAA0B,CAAC;IAC/E,CAAC;IAED,EAAE;QACA,OAAO,WAAW,IAAI,CAAC,SAAS,EAAE,CAAC;IACrC,CAAC;IAED,QAAQ;QACN,OAAO,qBAAqB,IAAI,CAAC,SAAS,GAAG,CAAC;IAChD,CAAC;IAED,wDAAwD;IACxD,KAAK,CAAC,OAAO,CAAC,MAAc;QAC1B,MAAM,IAAI,KAAK,CAAC,iBAAiB,CAAC,CAAC;IACrC,CAAC;CACF;AAED,MAAa,mBAAoB,SAAQ,sBAAsB;IAC7D,KAAK,CAAC,OAAO,CAAC,MAAc;QAC1B,MAAM,IAAI,GAAG;YACX,KAAK,EAAE,IAAI,CAAC,SAAS;YACrB,MAAM;YACN,WAAW,EAAE,OAAO,CAAC,GAAG,CAAC,mBAAmB,IAAI,GAAG;SACpD,CAAC;QACF,gBAAM,CAAC,KAAK,CAAC,wBAAwB,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAE7D,IAAI,IAAI,EACN,MAAM,GAAG,KAAK,CAAC;QACjB,IAAI;YACF,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,IAAA,0BAAkB,EAC3C,GAAG,IAAI,CAAC,UAAU,mBAAmB,EACrC;gBACE,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE;oBACP,cAAc,EAAE,kBAAkB;iBACnC;gBACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;aAC3B,EACD,2BAAkB,CACnB,CAAmB,CAAC,CAAC;SACvB;QAAC,OAAO,GAAG,EAAE;YACZ,OAAO;gBACL,KAAK,EAAE,mBAAmB,MAAM,CAAC,GAAG,CAAC,EAAE;aACxC,CAAC;SACH;QACD,gBAAM,CAAC,KAAK,CAAC,0BAA0B,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC/D,IAAI;YACF,OAAO;gBACL,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI;aAC7B,CAAC;SACH;QAAC,OAAO,GAAG,EAAE;YACZ,OAAO;gBACL,KAAK,EAAE,uBAAuB,MAAM,CAAC,GAAG,CAAC,KAAK,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE;aACrE,CAAC;SACH;IACH,CAAC;CACF;AAvCD,kDAuCC;AAED,MAAa,yBAA0B,SAAQ,sBAAsB;IACnE,KAAK,CAAC,OAAO,CAAC,MAAc;QAC1B,MAAM,IAAI,GAAG;YACX,KAAK,EAAE,IAAI,CAAC,SAAS;YACrB,MAAM;YACN,WAAW,EAAE,OAAO,CAAC,GAAG,CAAC,mBAAmB,IAAI,GAAG;SACpD,CAAC;QACF,gBAAM,CAAC,KAAK,CAAC,wBAAwB,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAE7D,IAAI,IAAI,EACN,MAAM,GAAG,KAAK,CAAC;QACjB,IAAI;YACF,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,IAAA,0BAAkB,EAC3C,GAAG,IAAI,CAAC,UAAU,cAAc,EAChC;gBACE,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE;oBACP,cAAc,EAAE,kBAAkB;iBACnC;gBACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;aAC3B,EACD,2BAAkB,CACnB,CAAmB,CAAC,CAAC;SACvB;QAAC,OAAO,GAAG,EAAE;YACZ,OAAO;gBACL,KAAK,EAAE,mBAAmB,MAAM,CAAC,GAAG,CAAC,EAAE;aACxC,CAAC;SACH;QACD,gBAAM,CAAC,KAAK,CAAC,0BAA0B,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC/D,IAAI;YACF,OAAO;gBACL,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI;aAC7B,CAAC;SACH;QAAC,OAAO,GAAG,EAAE;YACZ,OAAO;gBACL,KAAK,EAAE,uBAAuB,MAAM,CAAC,GAAG,CAAC,KAAK,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE;aACrE,CAAC;SACH;IACH,CAAC;CACF;AAvCD,8DAuCC"}
|