promptfoo 0.2.1 → 0.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/evaluator.d.ts.map +1 -1
- package/dist/evaluator.js +18 -6
- package/dist/evaluator.js.map +1 -1
- package/dist/main.js +6 -0
- package/dist/main.js.map +1 -1
- package/dist/providers.d.ts.map +1 -1
- package/dist/providers.js +65 -35
- package/dist/providers.js.map +1 -1
- package/dist/types.d.ts +2 -1
- package/dist/types.d.ts.map +1 -1
- package/package.json +1 -1
- package/src/evaluator.ts +16 -6
- package/src/main.ts +9 -0
- package/src/providers.ts +64 -37
- package/src/types.ts +2 -1
package/dist/evaluator.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"evaluator.d.ts","sourceRoot":"","sources":["../src/evaluator.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,eAAe,EAAE,eAAe,EAAuC,MAAM,YAAY,CAAC;
|
|
1
|
+
{"version":3,"file":"evaluator.d.ts","sourceRoot":"","sources":["../src/evaluator.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,eAAe,EAAE,eAAe,EAAuC,MAAM,YAAY,CAAC;AA2EnG,wBAAsB,QAAQ,CAAC,OAAO,EAAE,eAAe,GAAG,OAAO,CAAC,eAAe,CAAC,CAqIjF"}
|
package/dist/evaluator.js
CHANGED
|
@@ -29,14 +29,26 @@ async function runEval({ provider, prompt, vars, includeProviderId, }) {
|
|
|
29
29
|
};
|
|
30
30
|
try {
|
|
31
31
|
const response = await provider.callApi(renderedPrompt);
|
|
32
|
-
const success = vars.__expected ? checkExpectedValue(vars.__expected, response.output) : true;
|
|
33
32
|
const ret = {
|
|
34
33
|
...setup,
|
|
35
34
|
response,
|
|
36
|
-
success,
|
|
35
|
+
success: false,
|
|
37
36
|
};
|
|
38
|
-
if (
|
|
39
|
-
ret.error =
|
|
37
|
+
if (response.error) {
|
|
38
|
+
ret.error = response.error;
|
|
39
|
+
}
|
|
40
|
+
else if (response.output) {
|
|
41
|
+
const matchesExpected = vars.__expected
|
|
42
|
+
? checkExpectedValue(vars.__expected, response.output)
|
|
43
|
+
: true;
|
|
44
|
+
if (!matchesExpected) {
|
|
45
|
+
ret.error = `Expected ${vars.__expected}, got "${response.output}"`;
|
|
46
|
+
}
|
|
47
|
+
ret.success = matchesExpected;
|
|
48
|
+
}
|
|
49
|
+
else {
|
|
50
|
+
ret.success = false;
|
|
51
|
+
ret.error = 'No output';
|
|
40
52
|
}
|
|
41
53
|
return ret;
|
|
42
54
|
}
|
|
@@ -112,7 +124,7 @@ export async function evaluate(options) {
|
|
|
112
124
|
const combinedOutputs = new Array(vars.length).fill(null).map(() => []);
|
|
113
125
|
await async.forEachOfLimit(runEvalOptions, options.maxConcurrency || DEFAULT_MAX_CONCURRENCY, async (options, index) => {
|
|
114
126
|
const row = await runEval(options);
|
|
115
|
-
results
|
|
127
|
+
results[index] = row;
|
|
116
128
|
if (row.error) {
|
|
117
129
|
stats.failures++;
|
|
118
130
|
}
|
|
@@ -142,7 +154,7 @@ export async function evaluate(options) {
|
|
|
142
154
|
throw new Error('Expected index to be a number');
|
|
143
155
|
}
|
|
144
156
|
const combinedOutputIndex = Math.floor(index / prompts.length);
|
|
145
|
-
combinedOutputs[combinedOutputIndex].push(row.response?.output || '');
|
|
157
|
+
combinedOutputs[combinedOutputIndex].push(row.response?.output || row.error || '');
|
|
146
158
|
});
|
|
147
159
|
if (progressbar) {
|
|
148
160
|
progressbar.stop();
|
package/dist/evaluator.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"evaluator.js","sourceRoot":"","sources":["../src/evaluator.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,QAAQ,MAAM,UAAU,CAAC;AAahC,MAAM,uBAAuB,GAAG,CAAC,CAAC;AAElC,SAAS,kBAAkB,CAAC,QAAgB,EAAE,MAAc;IAC1D,IAAI,QAAQ,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE;QAChC,MAAM,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QACnC,MAAM,YAAY,GAAG,IAAI,QAAQ,CAAC,QAAQ,EAAE,UAAU,QAAQ,EAAE,CAAC,CAAC;QAClE,OAAO,YAAY,CAAC,MAAM,CAAC,CAAC;KAC7B;SAAM,IAAI,QAAQ,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE;QACxC,MAAM;QACN,OAAO,KAAK,CAAC;KACd;SAAM;QACL,OAAO,QAAQ,KAAK,MAAM,CAAC;KAC5B;AACH,CAAC;AAED,KAAK,UAAU,OAAO,CAAC,EACrB,QAAQ,EACR,MAAM,EACN,IAAI,EACJ,iBAAiB,GACF;IACf,IAAI,GAAG,IAAI,IAAI,EAAE,CAAC;IAClB,MAAM,cAAc,GAAG,QAAQ,CAAC,YAAY,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;IAE3D,4DAA4D;IAC5D,MAAM,aAAa,GAAG,iBAAiB,CAAC,CAAC,CAAC,IAAI,QAAQ,CAAC,EAAE,EAAE,KAAK,MAAM,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;IAElF,MAAM,KAAK,GAAG;QACZ,MAAM,EAAE;YACN,GAAG,EAAE,cAAc;YACnB,OAAO,EAAE,aAAa;SACvB;QACD,IAAI;KACL,CAAC;IAEF,IAAI;QACF,MAAM,QAAQ,GAAG,MAAM,QAAQ,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;QACxD,MAAM,
|
|
1
|
+
{"version":3,"file":"evaluator.js","sourceRoot":"","sources":["../src/evaluator.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,QAAQ,MAAM,UAAU,CAAC;AAahC,MAAM,uBAAuB,GAAG,CAAC,CAAC;AAElC,SAAS,kBAAkB,CAAC,QAAgB,EAAE,MAAc;IAC1D,IAAI,QAAQ,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE;QAChC,MAAM,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QACnC,MAAM,YAAY,GAAG,IAAI,QAAQ,CAAC,QAAQ,EAAE,UAAU,QAAQ,EAAE,CAAC,CAAC;QAClE,OAAO,YAAY,CAAC,MAAM,CAAC,CAAC;KAC7B;SAAM,IAAI,QAAQ,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE;QACxC,MAAM;QACN,OAAO,KAAK,CAAC;KACd;SAAM;QACL,OAAO,QAAQ,KAAK,MAAM,CAAC;KAC5B;AACH,CAAC;AAED,KAAK,UAAU,OAAO,CAAC,EACrB,QAAQ,EACR,MAAM,EACN,IAAI,EACJ,iBAAiB,GACF;IACf,IAAI,GAAG,IAAI,IAAI,EAAE,CAAC;IAClB,MAAM,cAAc,GAAG,QAAQ,CAAC,YAAY,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;IAE3D,4DAA4D;IAC5D,MAAM,aAAa,GAAG,iBAAiB,CAAC,CAAC,CAAC,IAAI,QAAQ,CAAC,EAAE,EAAE,KAAK,MAAM,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;IAElF,MAAM,KAAK,GAAG;QACZ,MAAM,EAAE;YACN,GAAG,EAAE,cAAc;YACnB,OAAO,EAAE,aAAa;SACvB;QACD,IAAI;KACL,CAAC;IAEF,IAAI;QACF,MAAM,QAAQ,GAAG,MAAM,QAAQ,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;QACxD,MAAM,GAAG,GAAmB;YAC1B,GAAG,KAAK;YACR,QAAQ;YACR,OAAO,EAAE,KAAK;SACf,CAAC;QACF,IAAI,QAAQ,CAAC,KAAK,EAAE;YAClB,GAAG,CAAC,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC;SAC5B;aAAM,IAAI,QAAQ,CAAC,MAAM,EAAE;YAC1B,MAAM,eAAe,GAAG,IAAI,CAAC,UAAU;gBACrC,CAAC,CAAC,kBAAkB,CAAC,IAAI,CAAC,UAAU,EAAE,QAAQ,CAAC,MAAM,CAAC;gBACtD,CAAC,CAAC,IAAI,CAAC;YACT,IAAI,CAAC,eAAe,EAAE;gBACpB,GAAG,CAAC,KAAK,GAAG,YAAY,IAAI,CAAC,UAAU,UAAU,QAAQ,CAAC,MAAM,GAAG,CAAC;aACrE;YACD,GAAG,CAAC,OAAO,GAAG,eAAe,CAAC;SAC/B;aAAM;YACL,GAAG,CAAC,OAAO,GAAG,KAAK,CAAC;YACpB,GAAG,CAAC,KAAK,GAAG,WAAW,CAAC;SACzB;QACD,OAAO,GAAG,CAAC;KACZ;IAAC,OAAO,GAAG,EAAE;QACZ,OAAO;YACL,GAAG,KAAK;YACR,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC;YAClB,OAAO,EAAE,KAAK;SACf,CAAC;KACH;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,OAAwB;IACrD,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,MAAM,OAAO,GAAqB,EAAE,CAAC;IAErC,KAAK,MAAM,aAAa,IAAI,OAAO,CAAC,OAAO,EAAE;QAC3C,KAAK,MAAM,QAAQ,IAAI,OAAO,CAAC,SAAS,EAAE;YACxC,OAAO,CAAC,IAAI,CAAC;gBACX,GAAG,EAAE,aAAa;gBAClB,OAAO,EACL,OAAO,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,QAAQ,CAAC,EAAE,EAAE,KAAK,aAAa,EAAE,CAAC,CAAC,CAAC,aAAa;aACvF,CAAC,CAAC;SACJ;KACF;IAED,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IAC3E,MAAM,0BAA0B,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QAChD,MAAM,GAAG,GAAG,EAAE,GAAG,CAAC,EAAE,CAAC;QACrB,OAAO,GAAG,CAAC,UAAU,CAAC;QACtB,OAAO,GAAG,CAAC;IACb,CAAC,CAAC,CAAC;IACH,MAAM,MAAM,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC;IAClC,MAAM,KAAK,GAAe;QACxB,MAAM;YACJ,CAAC,CAAC;gBACE,QAAQ;gBACR,CAAC,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,GAAG,MAAM,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC,CAAC,CAAC,CAAC;aAClF,CAAC,IAAI,EAAE;YACV,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,GAAG,MAAM,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC,CAAC,CAAC,CAAC;KACtF,CAAC;IAEF,MAAM,KAAK,GAAG;QACZ,SAAS,EAAE,CAAC;QACZ,QAAQ,EAAE,CAAC;QACX,UAAU,EAAE;YACV,KAAK,EAAE,CAAC;YACR,MAAM,EAAE,CAAC;YACT,UAAU,EAAE,CAAC;SACd;KACF,CAAC;IAEF,IAAI,WAAkC,CAAC;IACvC,IAAI,OAAO,CAAC,eAAe,EAAE;QAC3B,MAAM,YAAY,GAChB,OAAO,CAAC,OAAO,CAAC,MAAM,GAAG,OAAO,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,MAAM,IAAI,CAAC,CAAC,CAAC;QAClF,MAAM,WAAW,GAAG,MAAM,MAAM,CAAC,cAAc,CAAC,CAAC;QACjD,WAAW,GAAG,IAAI,WAAW,CAAC,SAAS,CACrC;YACE,MAAM,EACJ,4FAA4F;SAC/F,EACD,WAAW,CAAC,OAAO,CAAC,cAAc,CACnC,CAAC;QACF,WAAW,CAAC,KAAK,CAAC,YAAY,EAAE,CAAC,EAAE;YACjC,QAAQ,EAAE,EAAE;YACZ,MAAM,EAAE,EAAE;YACV,IAAI,EAAE,EAAE;SACT,CAAC,CAAC;KACJ;IAED,MAAM,cAAc,GAAqB,EAAE,CAAC;IAC5C,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE;QACtB,KAAK,MAAM,aAAa,IAAI,OAAO,CAAC,OAAO,EAAE;YAC3C,KAAK,MAAM,QAAQ,IAAI,OAAO,CAAC,SAAS,EAAE;gBACxC,cAAc,CAAC,IAAI,CAAC;oBAClB,QAAQ;oBACR,MAAM,EAAE,aAAa;oBACrB,IAAI,EAAE,GAAG;oBACT,iBAAiB,EAAE,OAAO,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC;iBAChD,CAAC,CAAC;aACJ;SACF;KACF;IAED,MAAM,eAAe,GAAe,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;IACpF,MAAM,KAAK,CAAC,cAAc,CACxB,cAAc,EACd,OAAO,CAAC,cAAc,IAAI,uBAAuB,EACjD,KAAK,EAAE,OAAuB,EAAE,KAAsB,EAAE,EAAE;QACxD,MAAM,GAAG,GAAG,MAAM,OAAO,CAAC,OAAO,CAAC,CAAC;QACnC,OAAO,CAAC,KAAe,CAAC,GAAG,GAAG,CAAC;QAC/B,IAAI,GAAG,CAAC,KAAK,EAAE;YACb,KAAK,CAAC,QAAQ,EAAE,CAAC;SAClB;aAAM;YACL,IAAI,GAAG,CAAC,OAAO,EAAE;gBACf,KAAK,CAAC,SAAS,EAAE,CAAC;aACnB;iBAAM;gBACL,KAAK,CAAC,QAAQ,EAAE,CAAC;aAClB;YACD,KAAK,CAAC,UAAU,CAAC,KAAK,IAAI,GAAG,CAAC,QAAQ,EAAE,UAAU,EAAE,KAAK,IAAI,CAAC,CAAC;YAC/D,KAAK,CAAC,UAAU,CAAC,MAAM,IAAI,GAAG,CAAC,QAAQ,EAAE,UAAU,EAAE,MAAM,IAAI,CAAC,CAAC;YACjE,KAAK,CAAC,UAAU,CAAC,UAAU,IAAI,GAAG,CAAC,QAAQ,EAAE,UAAU,EAAE,UAAU,IAAI,CAAC,CAAC;SAC1E;QAED,IAAI,WAAW,EAAE;YACf,WAAW,CAAC,SAAS,CAAC;gBACpB,QAAQ,EAAE,OAAO,CAAC,QAAQ,CAAC,EAAE,EAAE;gBAC/B,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC;gBACnC,IAAI,EAAE,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,IAAI,EAAE,CAAC;qBACrC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;qBAC5B,IAAI,CAAC,GAAG,CAAC;qBACT,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC;aAChB,CAAC,CAAC;SACJ;QAED,wBAAwB;QACxB,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE;YAC7B,MAAM,IAAI,KAAK,CAAC,+BAA+B,CAAC,CAAC;SAClD;QACD,MAAM,mBAAmB,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;QAC/D,eAAe,CAAC,mBAAmB,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,MAAM,IAAI,GAAG,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC;IACrF,CAAC,CACF,CAAC;IAEF,IAAI,WAAW,EAAE;QACf,WAAW,CAAC,IAAI,EAAE,CAAC;KACpB;IAED,yCAAyC;IACzC,IAAI,MAAM,EAAE;QACV,KAAK,CAAC,IAAI,CACR,GAAG,eAAe,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC;YACxC,OAAO,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,OAAO,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE;YACjE,GAAG,MAAM;YACT,GAAG,MAAM,CAAC,MAAM,CAAC,0BAA0B,CAAC,KAAK,CAAC,CAAC;SACpD,CAAC,CACH,CAAC;KACH;SAAM;QACL,KAAK,CAAC,IAAI,CACR,GAAG,eAAe,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,GAAG,MAAM,EAAE,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CACtF,CAAC;KACH;IAED,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC;AACnC,CAAC"}
|
package/dist/main.js
CHANGED
|
@@ -8,6 +8,7 @@ import logger, { setLogLevel } from './logger.js';
|
|
|
8
8
|
import { loadApiProvider } from './providers.js';
|
|
9
9
|
import { evaluate } from './evaluator.js';
|
|
10
10
|
import { readPrompts, readVars, writeOutput } from './util.js';
|
|
11
|
+
import { getDirectory } from './esm.js';
|
|
11
12
|
function createDummyFiles(directory) {
|
|
12
13
|
if (directory) {
|
|
13
14
|
// Make the directory if it doesn't exist
|
|
@@ -68,6 +69,11 @@ async function main() {
|
|
|
68
69
|
logger.info('Loaded default config from promptfooconfig.js');
|
|
69
70
|
}
|
|
70
71
|
const program = new Command();
|
|
72
|
+
program.option('--version', 'Print version', () => {
|
|
73
|
+
const packageJson = JSON.parse(readFileSync(pathJoin(getDirectory(), '../package.json'), 'utf8'));
|
|
74
|
+
console.log(packageJson.version);
|
|
75
|
+
process.exit(0);
|
|
76
|
+
});
|
|
71
77
|
program
|
|
72
78
|
.command('init [directory]')
|
|
73
79
|
.description('Initialize project with dummy files')
|
package/dist/main.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"main.js","sourceRoot":"","sources":["../src/main.ts"],"names":[],"mappings":";AACA,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,IAAI,CAAC;AACxE,OAAO,EAAE,KAAK,EAAE,IAAI,IAAI,QAAQ,EAAE,MAAM,MAAM,CAAC;AAE/C,OAAO,KAAK,MAAM,YAAY,CAAC;AAC/B,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAEpC,OAAO,MAAM,EAAE,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAClD,OAAO,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AACjD,OAAO,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAC1C,OAAO,EAAE,WAAW,EAAE,QAAQ,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;
|
|
1
|
+
{"version":3,"file":"main.js","sourceRoot":"","sources":["../src/main.ts"],"names":[],"mappings":";AACA,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,IAAI,CAAC;AACxE,OAAO,EAAE,KAAK,EAAE,IAAI,IAAI,QAAQ,EAAE,MAAM,MAAM,CAAC;AAE/C,OAAO,KAAK,MAAM,YAAY,CAAC;AAC/B,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAEpC,OAAO,MAAM,EAAE,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAClD,OAAO,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AACjD,OAAO,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAC1C,OAAO,EAAE,WAAW,EAAE,QAAQ,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AAC/D,OAAO,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC;AAIxC,SAAS,gBAAgB,CAAC,SAAwB;IAChD,IAAI,SAAS,EAAE;QACb,yCAAyC;QACzC,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE;YAC1B,SAAS,CAAC,SAAS,CAAC,CAAC;SACtB;KACF;IACD,MAAM,YAAY,GAAG;;;;;;;;;YASX,CAAC;IACX,MAAM,SAAS,GACb,oFAAoF,CAAC;IACvF,MAAM,WAAW,GAAG;;;;;GAKnB,CAAC;IACF,MAAM,MAAM,GAAG;;;;;;CAMhB,CAAC;IAEA,IAAI,SAAS,EAAE;QACb,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE;YAC1B,MAAM,CAAC,IAAI,CAAC,sBAAsB,SAAS,MAAM,CAAC,CAAC;YACnD,SAAS,CAAC,SAAS,CAAC,CAAC;SACtB;KACF;SAAM;QACL,SAAS,GAAG,GAAG,CAAC;KACjB;IAED,aAAa,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,aAAa,CAAC,EAAE,YAAY,CAAC,CAAC;IAC/E,aAAa,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,UAAU,CAAC,EAAE,SAAS,CAAC,CAAC;IACzE,aAAa,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,oBAAoB,CAAC,EAAE,WAAW,CAAC,CAAC;IACrF,aAAa,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,WAAW,CAAC,EAAE,MAAM,CAAC,CAAC;IAEvE,IAAI,SAAS,KAAK,GAAG,EAAE;QACrB,MAAM,CAAC,IAAI,CACT,qFAAqF,CACtF,CAAC;KACH;SAAM;QACL,MAAM,CAAC,IAAI,CAAC,0DAA0D,SAAS,EAAE,CAAC,CAAC;QACnF,MAAM,CAAC,IAAI,CAAC,QAAQ,SAAS,uCAAuC,CAAC,CAAC;KACvE;AACH,CAAC;AAED,KAAK,UAAU,IAAI;IACjB,IAAI,aAAa,GAAgC,EAAE,CAAC;IACpD,IAAI,UAAU,CAAC,oBAAoB,CAAC,EAAE;QACpC,aAAa;QACb,aAAa,GAAG,CAAC,MAAM,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,sBAAsB,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC;QACxF,MAAM,CAAC,IAAI,CAAC,+CAA+C,CAAC,CAAC;KAC9D;IAED,MAAM,OAAO,GAAG,IAAI,OAAO,EAAE,CAAC;IAE9B,OAAO,CAAC,MAAM,CAAC,WAAW,EAAE,eAAe,EAAE,GAAG,EAAE;QAChD,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAC5B,YAAY,CAAC,QAAQ,CAAC,YAAY,EAAE,EAAE,iBAAiB,CAAC,EAAE,MAAM,CAAC,CAClE,CAAC;QACF,OAAO,CAAC,GAAG,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;QACjC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,CAAC;IAEH,OAAO;SACJ,OAAO,CAAC,kBAAkB,CAAC;SAC3B,WAAW,CAAC,qCAAqC,CAAC;SAClD,MAAM,CAAC,CAAC,SAAwB,EAAE,EAAE;QACnC,gBAAgB,CAAC,SAAS,CAAC,CAAC;IAC9B,CAAC,CAAC,CAAC;IAEL,OAAO;SACJ,OAAO,CAAC,MAAM,CAAC;SACf,WAAW,CAAC,kBAAkB,CAAC;SAC/B,cAAc,CACb,0BAA0B,EAC1B,8BAA8B,EAC9B,aAAa,CAAC,OAAO,CACtB;SACA,cAAc,CACb,mCAAmC,EACnC,kGAAkG,EAClG,aAAa,CAAC,SAAS,CACxB;SACA,MAAM,CACL,qBAAqB,EACrB,6CAA6C,EAC7C,aAAa,CAAC,MAAM,CACrB;SACA,MAAM,CACL,mBAAmB,EACnB,sDAAsD,EACtD,aAAa,CAAC,IAAI,CACnB;SACA,MAAM,CACL,qBAAqB,EACrB,oEAAoE,EACpE,aAAa,CAAC,MAAM,CACrB;SACA,MAAM,CACL,gCAAgC,EAChC,wCAAwC,EACxC,MAAM,CAAC,aAAa,CAAC,cAAc,CAAC,CACrC;SACA,MAAM,CAAC,WAAW,EAAE,iBAAiB,EAAE,aAAa,CAAC,OAAO,CAAC;SAC7D,MAAM,CAAC,KAAK,EAAE,MAAoC,EAAE,EAAE;QACrD,IAAI,MAAM,CAAC,OAAO,EAAE;YAClB,WAAW,CAAC,OAAO,CAAC,CAAC;SACtB;QAED,MAAM,UAAU,GAAG,MAAM,CAAC,MAAM,CAAC;QACjC,IAAI,MAAM,GAAG,EAAE,CAAC;QAChB,IAAI,UAAU,EAAE;YACd,MAAM,GAAG,GAAG,KAAK,CAAC,UAAU,CAAC,CAAC,GAAG,CAAC;YAClC,QAAQ,GAAG,EAAE;gBACX,KAAK,OAAO;oBACV,MAAM,OAAO,GAAG,YAAY,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;oBAClD,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;oBAC7B,MAAM;gBACR,KAAK,KAAK;oBACR,MAAM,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;oBAC7B,MAAM;gBACR;oBACE,MAAM,IAAI,KAAK,CAAC,0CAA0C,GAAG,EAAE,CAAC,CAAC;aACpE;SACF;QAED,IAAI,IAAI,GAAiB,EAAE,CAAC;QAC5B,IAAI,MAAM,CAAC,IAAI,EAAE;YACf,IAAI,GAAG,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;SAC9B;QAED,MAAM,SAAS,GAAG,MAAM,OAAO,CAAC,GAAG,CACjC,MAAM,CAAC,SAAS,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC,MAAM,eAAe,CAAC,CAAC,CAAC,CAAC,CAC5D,CAAC;QACF,MAAM,OAAO,GAAoB;YAC/B,OAAO,EAAE,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC;YACpC,IAAI;YACJ,SAAS;YACT,eAAe,EAAE,IAAI;YACrB,cAAc,EACZ,MAAM,CAAC,cAAc,IAAI,MAAM,CAAC,cAAc,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,CAAC,SAAS;YACxF,GAAG,MAAM;SACV,CAAC;QAEF,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,OAAO,CAAC,CAAC;QAExC,IAAI,MAAM,CAAC,MAAM,EAAE;YACjB,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,qBAAqB,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;YAChE,WAAW,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;SACrC;aAAM;YACL,0BAA0B;YAC1B,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,GAAG,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC;YAC5E,MAAM,IAAI,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;YAC9B,MAAM,KAAK,GAAG,IAAI,KAAK,CAAC;gBACtB,IAAI;gBACJ,SAAS,EAAE,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC;gBACtE,QAAQ,EAAE,IAAI;gBACd,kBAAkB,EAAE,IAAI;gBACxB,KAAK,EAAE;oBACL,IAAI,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC;iBACvB;aACF,CAAC,CAAC;YACH,mHAAmH;YACnH,KAAK,MAAM,GAAG,IAAI,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE;gBACxC,MAAM,KAAK,GAAG,GAAG,CAAC,CAAC,CAAC,KAAK,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC;gBAC1F,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;aAC/E;YAED,MAAM,CAAC,IAAI,CAAC,IAAI,GAAG,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC;SACtC;QACD,MAAM,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;QACnC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,cAAc,OAAO,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC;QACvE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,aAAa,OAAO,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC;QACnE,MAAM,CAAC,IAAI,CACT,sBAAsB,OAAO,CAAC,KAAK,CAAC,UAAU,CAAC,KAAK,WAAW,OAAO,CAAC,KAAK,CAAC,UAAU,CAAC,MAAM,eAAe,OAAO,CAAC,KAAK,CAAC,UAAU,CAAC,UAAU,EAAE,CACnJ,CAAC;QACF,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACvB,CAAC,CAAC,CAAC;IAEL,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IAE5B,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE;QACjC,OAAO,CAAC,UAAU,EAAE,CAAC;KACtB;AACH,CAAC;AAED,IAAI,EAAE,CAAC"}
|
package/dist/providers.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"providers.d.ts","sourceRoot":"","sources":["../src/providers.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,WAAW,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AAG3D,qBAAa,qBAAsB,YAAW,WAAW;IACvD,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;gBAEH,SAAS,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM;IAY9C,EAAE,IAAI,MAAM;IAIZ,QAAQ,IAAI,MAAM;IAKZ,OAAO,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,CAAC;CAGzD;AAED,qBAAa,wBAAyB,SAAQ,qBAAqB;IACjE,MAAM,CAAC,wBAAwB,WAM7B;gBAEU,SAAS,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM;IAOxC,OAAO,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,CAAC;
|
|
1
|
+
{"version":3,"file":"providers.d.ts","sourceRoot":"","sources":["../src/providers.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,WAAW,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AAG3D,qBAAa,qBAAsB,YAAW,WAAW;IACvD,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;gBAEH,SAAS,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM;IAY9C,EAAE,IAAI,MAAM;IAIZ,QAAQ,IAAI,MAAM;IAKZ,OAAO,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,CAAC;CAGzD;AAED,qBAAa,wBAAyB,SAAQ,qBAAqB;IACjE,MAAM,CAAC,wBAAwB,WAM7B;gBAEU,SAAS,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM;IAOxC,OAAO,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,CAAC;CAyCzD;AAED,qBAAa,4BAA6B,SAAQ,qBAAqB;IACrE,MAAM,CAAC,kBAAkB,WAOvB;gBAEU,SAAS,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM;IAOxC,OAAO,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,CAAC;CAkDzD;AAED,wBAAsB,eAAe,CAAC,YAAY,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC,CAyBhF"}
|
package/dist/providers.js
CHANGED
|
@@ -36,24 +36,39 @@ class OpenAiCompletionProvider extends OpenAiGenericProvider {
|
|
|
36
36
|
temperature: process.env.OPENAI_TEMPERATURE || 0,
|
|
37
37
|
};
|
|
38
38
|
logger.debug(`Calling OpenAI API: ${JSON.stringify(body)}`);
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
39
|
+
let response, data;
|
|
40
|
+
try {
|
|
41
|
+
response = await fetch('https://api.openai.com/v1/completions', {
|
|
42
|
+
method: 'POST',
|
|
43
|
+
headers: {
|
|
44
|
+
'Content-Type': 'application/json',
|
|
45
|
+
Authorization: `Bearer ${this.apiKey}`,
|
|
46
|
+
},
|
|
47
|
+
body: JSON.stringify(body),
|
|
48
|
+
});
|
|
49
|
+
data = (await response.json());
|
|
50
|
+
}
|
|
51
|
+
catch (err) {
|
|
52
|
+
return {
|
|
53
|
+
error: `API call error: ${String(err)}`,
|
|
54
|
+
};
|
|
55
|
+
}
|
|
48
56
|
logger.debug(`\tOpenAI API response: ${JSON.stringify(data)}`);
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
+
try {
|
|
58
|
+
return {
|
|
59
|
+
output: data.choices[0].text,
|
|
60
|
+
tokenUsage: {
|
|
61
|
+
total: data.usage.total_tokens,
|
|
62
|
+
prompt: data.usage.prompt_tokens,
|
|
63
|
+
completion: data.usage.completion_tokens,
|
|
64
|
+
},
|
|
65
|
+
};
|
|
66
|
+
}
|
|
67
|
+
catch (err) {
|
|
68
|
+
return {
|
|
69
|
+
error: `API response error: ${String(err)}: ${JSON.stringify(data)}`,
|
|
70
|
+
};
|
|
71
|
+
}
|
|
57
72
|
}
|
|
58
73
|
}
|
|
59
74
|
OpenAiCompletionProvider.OPENAI_COMPLETION_MODELS = [
|
|
@@ -78,7 +93,7 @@ class OpenAiChatCompletionProvider extends OpenAiGenericProvider {
|
|
|
78
93
|
// string prompt into a `messages` array.
|
|
79
94
|
messages = JSON.parse(prompt);
|
|
80
95
|
}
|
|
81
|
-
catch (
|
|
96
|
+
catch (err) {
|
|
82
97
|
messages = [{ role: 'user', content: prompt }];
|
|
83
98
|
}
|
|
84
99
|
const body = {
|
|
@@ -88,24 +103,39 @@ class OpenAiChatCompletionProvider extends OpenAiGenericProvider {
|
|
|
88
103
|
temperature: process.env.OPENAI_MAX_TEMPERATURE || 0,
|
|
89
104
|
};
|
|
90
105
|
logger.debug(`Calling OpenAI API: ${JSON.stringify(body)}`);
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
106
|
+
let response, data;
|
|
107
|
+
try {
|
|
108
|
+
response = await fetch('https://api.openai.com/v1/chat/completions', {
|
|
109
|
+
method: 'POST',
|
|
110
|
+
headers: {
|
|
111
|
+
'Content-Type': 'application/json',
|
|
112
|
+
Authorization: `Bearer ${this.apiKey}`,
|
|
113
|
+
},
|
|
114
|
+
body: JSON.stringify(body),
|
|
115
|
+
});
|
|
116
|
+
data = (await response.json());
|
|
117
|
+
}
|
|
118
|
+
catch (err) {
|
|
119
|
+
return {
|
|
120
|
+
error: `API call error: ${String(err)}`,
|
|
121
|
+
};
|
|
122
|
+
}
|
|
100
123
|
logger.debug(`\tOpenAI API response: ${JSON.stringify(data)}`);
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
124
|
+
try {
|
|
125
|
+
return {
|
|
126
|
+
output: data.choices[0].message.content,
|
|
127
|
+
tokenUsage: {
|
|
128
|
+
total: data.usage.total_tokens,
|
|
129
|
+
prompt: data.usage.prompt_tokens,
|
|
130
|
+
completion: data.usage.completion_tokens,
|
|
131
|
+
},
|
|
132
|
+
};
|
|
133
|
+
}
|
|
134
|
+
catch (err) {
|
|
135
|
+
return {
|
|
136
|
+
error: `API response error: ${String(err)}: ${JSON.stringify(data)}`,
|
|
137
|
+
};
|
|
138
|
+
}
|
|
109
139
|
}
|
|
110
140
|
}
|
|
111
141
|
OpenAiChatCompletionProvider.OPENAI_CHAT_MODELS = [
|
package/dist/providers.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"providers.js","sourceRoot":"","sources":["../src/providers.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,YAAY,CAAC;AAC/B,OAAO,IAAI,MAAM,WAAW,CAAC;AAG7B,OAAO,MAAM,MAAM,aAAa,CAAC;AAEjC,MAAM,OAAO,qBAAqB;IAIhC,YAAY,SAAiB,EAAE,MAAe;QAC5C,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAE3B,MAAM,GAAG,GAAG,MAAM,IAAI,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC;QACjD,IAAI,CAAC,GAAG,EAAE;YACR,MAAM,IAAI,KAAK,CACb,kHAAkH,CACnH,CAAC;SACH;QACD,IAAI,CAAC,MAAM,GAAG,GAAG,CAAC;IACpB,CAAC;IAED,EAAE;QACA,OAAO,UAAU,IAAI,CAAC,SAAS,EAAE,CAAC;IACpC,CAAC;IAED,QAAQ;QACN,OAAO,oBAAoB,IAAI,CAAC,SAAS,GAAG,CAAC;IAC/C,CAAC;IAED,wDAAwD;IACxD,KAAK,CAAC,OAAO,CAAC,MAAc;QAC1B,MAAM,IAAI,KAAK,CAAC,iBAAiB,CAAC,CAAC;IACrC,CAAC;CACF;AAED,MAAa,wBAAyB,SAAQ,qBAAqB;IASjE,YAAY,SAAiB,EAAE,MAAe;QAC5C,IAAI,CAAC,wBAAwB,CAAC,wBAAwB,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE;YAC1E,MAAM,CAAC,IAAI,CAAC,0CAA0C,SAAS,EAAE,CAAC,CAAC;SACpE;QACD,KAAK,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;IAC3B,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,MAAc;QAC1B,MAAM,IAAI,GAAG;YACX,KAAK,EAAE,IAAI,CAAC,SAAS;YACrB,MAAM;YACN,UAAU,EAAE,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,IAAI;YACjD,WAAW,EAAE,OAAO,CAAC,GAAG,CAAC,kBAAkB,IAAI,CAAC;SACjD,CAAC;QACF,MAAM,CAAC,KAAK,CAAC,uBAAuB,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC5D,
|
|
1
|
+
{"version":3,"file":"providers.js","sourceRoot":"","sources":["../src/providers.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,YAAY,CAAC;AAC/B,OAAO,IAAI,MAAM,WAAW,CAAC;AAG7B,OAAO,MAAM,MAAM,aAAa,CAAC;AAEjC,MAAM,OAAO,qBAAqB;IAIhC,YAAY,SAAiB,EAAE,MAAe;QAC5C,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAE3B,MAAM,GAAG,GAAG,MAAM,IAAI,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC;QACjD,IAAI,CAAC,GAAG,EAAE;YACR,MAAM,IAAI,KAAK,CACb,kHAAkH,CACnH,CAAC;SACH;QACD,IAAI,CAAC,MAAM,GAAG,GAAG,CAAC;IACpB,CAAC;IAED,EAAE;QACA,OAAO,UAAU,IAAI,CAAC,SAAS,EAAE,CAAC;IACpC,CAAC;IAED,QAAQ;QACN,OAAO,oBAAoB,IAAI,CAAC,SAAS,GAAG,CAAC;IAC/C,CAAC;IAED,wDAAwD;IACxD,KAAK,CAAC,OAAO,CAAC,MAAc;QAC1B,MAAM,IAAI,KAAK,CAAC,iBAAiB,CAAC,CAAC;IACrC,CAAC;CACF;AAED,MAAa,wBAAyB,SAAQ,qBAAqB;IASjE,YAAY,SAAiB,EAAE,MAAe;QAC5C,IAAI,CAAC,wBAAwB,CAAC,wBAAwB,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE;YAC1E,MAAM,CAAC,IAAI,CAAC,0CAA0C,SAAS,EAAE,CAAC,CAAC;SACpE;QACD,KAAK,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;IAC3B,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,MAAc;QAC1B,MAAM,IAAI,GAAG;YACX,KAAK,EAAE,IAAI,CAAC,SAAS;YACrB,MAAM;YACN,UAAU,EAAE,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,IAAI;YACjD,WAAW,EAAE,OAAO,CAAC,GAAG,CAAC,kBAAkB,IAAI,CAAC;SACjD,CAAC;QACF,MAAM,CAAC,KAAK,CAAC,uBAAuB,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC5D,IAAI,QAAQ,EAAE,IAAI,CAAC;QACnB,IAAI;YACF,QAAQ,GAAG,MAAM,KAAK,CAAC,uCAAuC,EAAE;gBAC9D,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE;oBACP,cAAc,EAAE,kBAAkB;oBAClC,aAAa,EAAE,UAAU,IAAI,CAAC,MAAM,EAAE;iBACvC;gBACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;aAC3B,CAAC,CAAC;YAEH,IAAI,GAAG,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAmB,CAAC;SAClD;QAAC,OAAO,GAAG,EAAE;YACZ,OAAO;gBACL,KAAK,EAAE,mBAAmB,MAAM,CAAC,GAAG,CAAC,EAAE;aACxC,CAAC;SACH;QACD,MAAM,CAAC,KAAK,CAAC,0BAA0B,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC/D,IAAI;YACF,OAAO;gBACL,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI;gBAC5B,UAAU,EAAE;oBACV,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,YAAY;oBAC9B,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,aAAa;oBAChC,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,iBAAiB;iBACzC;aACF,CAAC;SACH;QAAC,OAAO,GAAG,EAAE;YACZ,OAAO;gBACL,KAAK,EAAE,uBAAuB,MAAM,CAAC,GAAG,CAAC,KAAK,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE;aACrE,CAAC;SACH;IACH,CAAC;;AAvDM,iDAAwB,GAAG;IAChC,kBAAkB;IAClB,kBAAkB;IAClB,gBAAgB;IAChB,kBAAkB;IAClB,cAAc;CACf,CAAC;SAPS,wBAAwB;AA2DrC,MAAa,4BAA6B,SAAQ,qBAAqB;IAUrE,YAAY,SAAiB,EAAE,MAAe;QAC5C,IAAI,CAAC,4BAA4B,CAAC,kBAAkB,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE;YACxE,MAAM,CAAC,IAAI,CAAC,oCAAoC,SAAS,EAAE,CAAC,CAAC;SAC9D;QACD,KAAK,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;IAC3B,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,MAAc;QAC1B,IAAI,QAA6C,CAAC;QAClD,IAAI;YACF,qEAAqE;YACrE,yCAAyC;YACzC,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;SAC/B;QAAC,OAAO,GAAG,EAAE;YACZ,QAAQ,GAAG,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;SAChD;QACD,MAAM,IAAI,GAAG;YACX,KAAK,EAAE,IAAI,CAAC,SAAS;YACrB,QAAQ,EAAE,QAAQ;YAClB,UAAU,EAAE,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,IAAI;YACjD,WAAW,EAAE,OAAO,CAAC,GAAG,CAAC,sBAAsB,IAAI,CAAC;SACrD,CAAC;QACF,MAAM,CAAC,KAAK,CAAC,uBAAuB,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAE5D,IAAI,QAAQ,EAAE,IAAI,CAAC;QACnB,IAAI;YACF,QAAQ,GAAG,MAAM,KAAK,CAAC,4CAA4C,EAAE;gBACnE,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE;oBACP,cAAc,EAAE,kBAAkB;oBAClC,aAAa,EAAE,UAAU,IAAI,CAAC,MAAM,EAAE;iBACvC;gBACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;aAC3B,CAAC,CAAC;YACH,IAAI,GAAG,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAmB,CAAC;SAClD;QAAC,OAAO,GAAG,EAAE;YACZ,OAAO;gBACL,KAAK,EAAE,mBAAmB,MAAM,CAAC,GAAG,CAAC,EAAE;aACxC,CAAC;SACH;QAED,MAAM,CAAC,KAAK,CAAC,0BAA0B,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC/D,IAAI;YACF,OAAO;gBACL,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO;gBACvC,UAAU,EAAE;oBACV,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,YAAY;oBAC9B,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,aAAa;oBAChC,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,iBAAiB;iBACzC;aACF,CAAC;SACH;QAAC,OAAO,GAAG,EAAE;YACZ,OAAO;gBACL,KAAK,EAAE,uBAAuB,MAAM,CAAC,GAAG,CAAC,KAAK,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE;aACrE,CAAC;SACH;IACH,CAAC;;AAjEM,+CAAkB,GAAG;IAC1B,OAAO;IACP,YAAY;IACZ,WAAW;IACX,gBAAgB;IAChB,eAAe;IACf,oBAAoB;CACrB,CAAC;SARS,4BAA4B;AAqEzC,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,YAAoB;IACxD,IAAI,YAAY,EAAE,UAAU,CAAC,SAAS,CAAC,EAAE;QACvC,qBAAqB;QACrB,MAAM,OAAO,GAAG,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACxC,MAAM,SAAS,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;QAC7B,MAAM,SAAS,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;QAE7B,IAAI,SAAS,KAAK,MAAM,EAAE;YACxB,OAAO,IAAI,4BAA4B,CAAC,SAAS,IAAI,eAAe,CAAC,CAAC;SACvE;aAAM,IAAI,SAAS,KAAK,YAAY,EAAE;YACrC,OAAO,IAAI,wBAAwB,CAAC,SAAS,IAAI,kBAAkB,CAAC,CAAC;SACtE;aAAM,IAAI,4BAA4B,CAAC,kBAAkB,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE;YAC9E,OAAO,IAAI,4BAA4B,CAAC,SAAS,CAAC,CAAC;SACpD;aAAM,IAAI,wBAAwB,CAAC,wBAAwB,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE;YAChF,OAAO,IAAI,wBAAwB,CAAC,SAAS,CAAC,CAAC;SAChD;aAAM;YACL,MAAM,IAAI,KAAK,CACb,8BAA8B,SAAS,gGAAgG,CACxI,CAAC;SACH;KACF;IAED,qBAAqB;IACrB,MAAM,iBAAiB,GAAG,CAAC,MAAM,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,YAAY,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC;IACzF,OAAO,IAAI,iBAAiB,EAAE,CAAC;AACjC,CAAC"}
|
package/dist/types.d.ts
CHANGED
package/dist/types.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,kBAAkB;IACjC,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,WAAW;IAC1B,EAAE,EAAE,MAAM,MAAM,CAAC;IACjB,OAAO,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,OAAO,CAAC,gBAAgB,CAAC,CAAC;CACxD;AAED,UAAU,UAAU;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,gBAAgB;IAC/B,
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,kBAAkB;IACjC,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,WAAW;IAC1B,EAAE,EAAE,MAAM,MAAM,CAAC;IACjB,OAAO,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,OAAO,CAAC,gBAAgB,CAAC,CAAC;CACxD;AAED,UAAU,UAAU;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,gBAAgB;IAC/B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,UAAU,CAAC,EAAE,UAAU,CAAC;CACzB;AAED,MAAM,WAAW,MAAM;IACrB,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAC;CACvB;AAED,MAAM,MAAM,UAAU,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;AAEhD,MAAM,WAAW,eAAe;IAC9B,SAAS,EAAE,WAAW,EAAE,CAAC;IACzB,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,IAAI,CAAC,EAAE,UAAU,EAAE,CAAC;IAEpB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,eAAe,CAAC,EAAE,OAAO,CAAC;CAC3B;AAED,MAAM,WAAW,MAAM;IACrB,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC7B,QAAQ,CAAC,EAAE,gBAAgB,CAAC;IAC5B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,OAAO,CAAC;CAClB;AAED,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,cAAc,EAAE,CAAC;IAC1B,KAAK,EAAE,MAAM,EAAE,EAAE,CAAC;IAClB,KAAK,EAAE;QACL,SAAS,EAAE,MAAM,CAAC;QAClB,QAAQ,EAAE,MAAM,CAAC;QACjB,UAAU,EAAE,UAAU,CAAC;KACxB,CAAC;CACH"}
|
package/package.json
CHANGED
package/src/evaluator.ts
CHANGED
|
@@ -49,14 +49,24 @@ async function runEval({
|
|
|
49
49
|
|
|
50
50
|
try {
|
|
51
51
|
const response = await provider.callApi(renderedPrompt);
|
|
52
|
-
const success = vars.__expected ? checkExpectedValue(vars.__expected, response.output) : true;
|
|
53
52
|
const ret: EvaluateResult = {
|
|
54
53
|
...setup,
|
|
55
54
|
response,
|
|
56
|
-
success,
|
|
55
|
+
success: false,
|
|
57
56
|
};
|
|
58
|
-
if (
|
|
59
|
-
ret.error =
|
|
57
|
+
if (response.error) {
|
|
58
|
+
ret.error = response.error;
|
|
59
|
+
} else if (response.output) {
|
|
60
|
+
const matchesExpected = vars.__expected
|
|
61
|
+
? checkExpectedValue(vars.__expected, response.output)
|
|
62
|
+
: true;
|
|
63
|
+
if (!matchesExpected) {
|
|
64
|
+
ret.error = `Expected ${vars.__expected}, got "${response.output}"`;
|
|
65
|
+
}
|
|
66
|
+
ret.success = matchesExpected;
|
|
67
|
+
} else {
|
|
68
|
+
ret.success = false;
|
|
69
|
+
ret.error = 'No output';
|
|
60
70
|
}
|
|
61
71
|
return ret;
|
|
62
72
|
} catch (err) {
|
|
@@ -147,7 +157,7 @@ export async function evaluate(options: EvaluateOptions): Promise<EvaluateSummar
|
|
|
147
157
|
options.maxConcurrency || DEFAULT_MAX_CONCURRENCY,
|
|
148
158
|
async (options: RunEvalOptions, index: number | string) => {
|
|
149
159
|
const row = await runEval(options);
|
|
150
|
-
results
|
|
160
|
+
results[index as number] = row;
|
|
151
161
|
if (row.error) {
|
|
152
162
|
stats.failures++;
|
|
153
163
|
} else {
|
|
@@ -177,7 +187,7 @@ export async function evaluate(options: EvaluateOptions): Promise<EvaluateSummar
|
|
|
177
187
|
throw new Error('Expected index to be a number');
|
|
178
188
|
}
|
|
179
189
|
const combinedOutputIndex = Math.floor(index / prompts.length);
|
|
180
|
-
combinedOutputs[combinedOutputIndex].push(row.response?.output || '');
|
|
190
|
+
combinedOutputs[combinedOutputIndex].push(row.response?.output || row.error || '');
|
|
181
191
|
},
|
|
182
192
|
);
|
|
183
193
|
|
package/src/main.ts
CHANGED
|
@@ -10,6 +10,7 @@ import logger, { setLogLevel } from './logger.js';
|
|
|
10
10
|
import { loadApiProvider } from './providers.js';
|
|
11
11
|
import { evaluate } from './evaluator.js';
|
|
12
12
|
import { readPrompts, readVars, writeOutput } from './util.js';
|
|
13
|
+
import { getDirectory } from './esm.js';
|
|
13
14
|
|
|
14
15
|
import type { CommandLineOptions, EvaluateOptions, VarMapping } from './types.js';
|
|
15
16
|
|
|
@@ -80,6 +81,14 @@ async function main() {
|
|
|
80
81
|
|
|
81
82
|
const program = new Command();
|
|
82
83
|
|
|
84
|
+
program.option('--version', 'Print version', () => {
|
|
85
|
+
const packageJson = JSON.parse(
|
|
86
|
+
readFileSync(pathJoin(getDirectory(), '../package.json'), 'utf8'),
|
|
87
|
+
);
|
|
88
|
+
console.log(packageJson.version);
|
|
89
|
+
process.exit(0);
|
|
90
|
+
});
|
|
91
|
+
|
|
83
92
|
program
|
|
84
93
|
.command('init [directory]')
|
|
85
94
|
.description('Initialize project with dummy files')
|
package/src/providers.ts
CHANGED
|
@@ -58,25 +58,38 @@ export class OpenAiCompletionProvider extends OpenAiGenericProvider {
|
|
|
58
58
|
temperature: process.env.OPENAI_TEMPERATURE || 0,
|
|
59
59
|
};
|
|
60
60
|
logger.debug(`Calling OpenAI API: ${JSON.stringify(body)}`);
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
61
|
+
let response, data;
|
|
62
|
+
try {
|
|
63
|
+
response = await fetch('https://api.openai.com/v1/completions', {
|
|
64
|
+
method: 'POST',
|
|
65
|
+
headers: {
|
|
66
|
+
'Content-Type': 'application/json',
|
|
67
|
+
Authorization: `Bearer ${this.apiKey}`,
|
|
68
|
+
},
|
|
69
|
+
body: JSON.stringify(body),
|
|
70
|
+
});
|
|
71
|
+
|
|
72
|
+
data = (await response.json()) as unknown as any;
|
|
73
|
+
} catch (err) {
|
|
74
|
+
return {
|
|
75
|
+
error: `API call error: ${String(err)}`,
|
|
76
|
+
};
|
|
77
|
+
}
|
|
71
78
|
logger.debug(`\tOpenAI API response: ${JSON.stringify(data)}`);
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
79
|
+
try {
|
|
80
|
+
return {
|
|
81
|
+
output: data.choices[0].text,
|
|
82
|
+
tokenUsage: {
|
|
83
|
+
total: data.usage.total_tokens,
|
|
84
|
+
prompt: data.usage.prompt_tokens,
|
|
85
|
+
completion: data.usage.completion_tokens,
|
|
86
|
+
},
|
|
87
|
+
};
|
|
88
|
+
} catch (err) {
|
|
89
|
+
return {
|
|
90
|
+
error: `API response error: ${String(err)}: ${JSON.stringify(data)}`,
|
|
91
|
+
};
|
|
92
|
+
}
|
|
80
93
|
}
|
|
81
94
|
}
|
|
82
95
|
|
|
@@ -103,7 +116,7 @@ export class OpenAiChatCompletionProvider extends OpenAiGenericProvider {
|
|
|
103
116
|
// User can specify `messages` payload as JSON, or we'll just put the
|
|
104
117
|
// string prompt into a `messages` array.
|
|
105
118
|
messages = JSON.parse(prompt);
|
|
106
|
-
} catch (
|
|
119
|
+
} catch (err) {
|
|
107
120
|
messages = [{ role: 'user', content: prompt }];
|
|
108
121
|
}
|
|
109
122
|
const body = {
|
|
@@ -113,25 +126,39 @@ export class OpenAiChatCompletionProvider extends OpenAiGenericProvider {
|
|
|
113
126
|
temperature: process.env.OPENAI_MAX_TEMPERATURE || 0,
|
|
114
127
|
};
|
|
115
128
|
logger.debug(`Calling OpenAI API: ${JSON.stringify(body)}`);
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
129
|
+
|
|
130
|
+
let response, data;
|
|
131
|
+
try {
|
|
132
|
+
response = await fetch('https://api.openai.com/v1/chat/completions', {
|
|
133
|
+
method: 'POST',
|
|
134
|
+
headers: {
|
|
135
|
+
'Content-Type': 'application/json',
|
|
136
|
+
Authorization: `Bearer ${this.apiKey}`,
|
|
137
|
+
},
|
|
138
|
+
body: JSON.stringify(body),
|
|
139
|
+
});
|
|
140
|
+
data = (await response.json()) as unknown as any;
|
|
141
|
+
} catch (err) {
|
|
142
|
+
return {
|
|
143
|
+
error: `API call error: ${String(err)}`,
|
|
144
|
+
};
|
|
145
|
+
}
|
|
146
|
+
|
|
126
147
|
logger.debug(`\tOpenAI API response: ${JSON.stringify(data)}`);
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
148
|
+
try {
|
|
149
|
+
return {
|
|
150
|
+
output: data.choices[0].message.content,
|
|
151
|
+
tokenUsage: {
|
|
152
|
+
total: data.usage.total_tokens,
|
|
153
|
+
prompt: data.usage.prompt_tokens,
|
|
154
|
+
completion: data.usage.completion_tokens,
|
|
155
|
+
},
|
|
156
|
+
};
|
|
157
|
+
} catch (err) {
|
|
158
|
+
return {
|
|
159
|
+
error: `API response error: ${String(err)}: ${JSON.stringify(data)}`,
|
|
160
|
+
};
|
|
161
|
+
}
|
|
135
162
|
}
|
|
136
163
|
}
|
|
137
164
|
|