promptfoo 0.6.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +15 -2
- package/dist/cache.d.ts +8 -0
- package/dist/cache.d.ts.map +1 -0
- package/dist/cache.js +78 -0
- package/dist/cache.js.map +1 -0
- package/dist/evaluator.d.ts.map +1 -1
- package/dist/evaluator.js +2 -0
- package/dist/evaluator.js.map +1 -1
- package/dist/main.js +10 -5
- package/dist/main.js.map +1 -1
- package/dist/providers/localai.d.ts.map +1 -1
- package/dist/providers/localai.js +7 -9
- package/dist/providers/localai.js.map +1 -1
- package/dist/providers/openai.d.ts.map +1 -1
- package/dist/providers/openai.js +31 -38
- package/dist/providers/openai.js.map +1 -1
- package/dist/types.d.ts +6 -4
- package/dist/types.d.ts.map +1 -1
- package/dist/util.d.ts.map +1 -1
- package/dist/util.js +0 -1
- package/dist/util.js.map +1 -1
- package/package.json +5 -2
- package/src/cache.ts +91 -0
- package/src/evaluator.ts +2 -0
- package/src/main.ts +11 -6
- package/src/providers/localai.ts +9 -11
- package/src/providers/openai.ts +34 -42
- package/src/types.ts +6 -4
- package/src/util.ts +0 -1
package/README.md
CHANGED
|
@@ -9,22 +9,35 @@ With promptfoo, you can:
|
|
|
9
9
|
|
|
10
10
|
- **Test multiple prompts** against predefined test cases
|
|
11
11
|
- **Evaluate quality and catch regressions** by comparing LLM outputs side-by-side
|
|
12
|
-
- **Speed up evaluations**
|
|
12
|
+
- **Speed up evaluations** with caching and concurrent tests
|
|
13
13
|
- **Flag bad outputs automatically** by setting "expectations"
|
|
14
14
|
- Use as a command line tool, or integrate into your workflow as a library
|
|
15
15
|
- Use OpenAI models, open-source models like Llama and Vicuna, or integrate custom API providers for any LLM API
|
|
16
16
|
|
|
17
|
+
The goal: **test-driven prompt engineering**, rather than trial-and-error.
|
|
18
|
+
|
|
17
19
|
# [» View full documentation «](https://promptfoo.dev/docs/intro)
|
|
18
20
|
|
|
19
|
-
promptfoo produces matrix views that
|
|
21
|
+
promptfoo produces matrix views that let you quickly evaluate outputs across many prompts.
|
|
20
22
|
|
|
21
23
|
Here's an example of a side-by-side comparison of multiple prompts and inputs:
|
|
22
24
|
|
|
23
25
|
|
|
24
26
|
|
|
25
27
|
It works on the command line too:
|
|
28
|
+
|
|
26
29
|
|
|
27
30
|
|
|
31
|
+
## Workflow
|
|
32
|
+
|
|
33
|
+
Start by establishing a handful of test cases - core use cases and failure cases that you want to ensure your prompt can handle.
|
|
34
|
+
|
|
35
|
+
As you explore modifications to the prompt, use `promptfoo eval` to rate all outputs. This ensures the prompt is actually improving overall.
|
|
36
|
+
|
|
37
|
+
As you collect more examples and establish a user feedback loop, continue to build the pool of test cases.
|
|
38
|
+
|
|
39
|
+
<img width="772" alt="LLM ops" src="https://github.com/typpo/promptfoo/assets/310310/cf0461a7-2832-4362-9fbb-4ebd911d06ff">
|
|
40
|
+
|
|
28
41
|
## Usage (command line & web viewer)
|
|
29
42
|
|
|
30
43
|
To get started, run the following command:
|
package/dist/cache.d.ts
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import type { RequestInfo, RequestInit } from 'node-fetch';
|
|
2
|
+
export declare function fetchJsonWithCache(url: RequestInfo, options: RequestInit | undefined, timeout: number): Promise<{
|
|
3
|
+
data: any;
|
|
4
|
+
cached: boolean;
|
|
5
|
+
}>;
|
|
6
|
+
export declare function enableCache(): void;
|
|
7
|
+
export declare function disableCache(): void;
|
|
8
|
+
//# sourceMappingURL=cache.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cache.d.ts","sourceRoot":"","sources":["../src/cache.ts"],"names":[],"mappings":"AAUA,OAAO,KAAK,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AA4B3D,wBAAsB,kBAAkB,CACtC,GAAG,EAAE,WAAW,EAChB,OAAO,yBAAkB,EACzB,OAAO,EAAE,MAAM,GACd,OAAO,CAAC;IAAE,IAAI,EAAE,GAAG,CAAC;IAAC,MAAM,EAAE,OAAO,CAAA;CAAE,CAAC,CAuCzC;AAED,wBAAgB,WAAW,SAE1B;AAED,wBAAgB,YAAY,SAG3B"}
|
package/dist/cache.js
ADDED
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.disableCache = exports.enableCache = exports.fetchJsonWithCache = void 0;
|
|
7
|
+
const node_path_1 = __importDefault(require("node:path"));
|
|
8
|
+
const cache_manager_1 = __importDefault(require("cache-manager"));
|
|
9
|
+
const cache_manager_fs_hash_1 = __importDefault(require("cache-manager-fs-hash"));
|
|
10
|
+
const logger_js_1 = __importDefault(require("./logger.js"));
|
|
11
|
+
const util_js_1 = require("./util.js");
|
|
12
|
+
let cacheInstance;
|
|
13
|
+
let enabled = typeof process.env.PROMPTFOO_CACHE_ENABLED === 'undefined'
|
|
14
|
+
? true
|
|
15
|
+
: Boolean(process.env.PROMPTFOO_CACHE_ENABLED);
|
|
16
|
+
const cacheType = process.env.PROMPTFOO_CACHE_TYPE || (process.env.NODE_ENV === 'test' ? 'memory' : 'disk');
|
|
17
|
+
function getCache() {
|
|
18
|
+
if (!cacheInstance) {
|
|
19
|
+
cacheInstance = cache_manager_1.default.caching({
|
|
20
|
+
store: cacheType === 'disk' ? cache_manager_fs_hash_1.default : 'memory',
|
|
21
|
+
options: {
|
|
22
|
+
max: process.env.PROMPTFOO_CACHE_MAX_FILE_COUNT || 10000,
|
|
23
|
+
path: process.env.PROMPTFOO_CACHE_PATH || node_path_1.default.join((0, util_js_1.getConfigDirectoryPath)(), 'cache'),
|
|
24
|
+
ttl: process.env.PROMPTFOO_CACHE_TTL || 60 * 60 * 24 * 14,
|
|
25
|
+
maxsize: process.env.PROMPTFOO_CACHE_MAX_SIZE || 1e7, // in bytes, 10mb
|
|
26
|
+
//zip: true, // whether to use gzip compression
|
|
27
|
+
},
|
|
28
|
+
});
|
|
29
|
+
}
|
|
30
|
+
return cacheInstance;
|
|
31
|
+
}
|
|
32
|
+
async function fetchJsonWithCache(url, options = {}, timeout) {
|
|
33
|
+
if (!enabled) {
|
|
34
|
+
const resp = await (0, util_js_1.fetchWithTimeout)(url, options, timeout);
|
|
35
|
+
return {
|
|
36
|
+
cached: false,
|
|
37
|
+
data: await resp.json(),
|
|
38
|
+
};
|
|
39
|
+
}
|
|
40
|
+
const cache = await getCache();
|
|
41
|
+
const copy = Object.assign({}, options);
|
|
42
|
+
delete copy.headers;
|
|
43
|
+
const cacheKey = `fetch:${url}:${JSON.stringify(copy)}`;
|
|
44
|
+
// Try to get the cached response
|
|
45
|
+
const cachedResponse = await cache.get(cacheKey);
|
|
46
|
+
if (cachedResponse) {
|
|
47
|
+
logger_js_1.default.debug(`Returning cached response for ${url}: ${cachedResponse}`);
|
|
48
|
+
return {
|
|
49
|
+
cached: true,
|
|
50
|
+
data: JSON.parse(cachedResponse),
|
|
51
|
+
};
|
|
52
|
+
}
|
|
53
|
+
// Fetch the actual data and store it in the cache
|
|
54
|
+
const response = await (0, util_js_1.fetchWithTimeout)(url, options, timeout);
|
|
55
|
+
try {
|
|
56
|
+
const data = await response.json();
|
|
57
|
+
logger_js_1.default.debug(`Storing ${url} response in cache: ${data}`);
|
|
58
|
+
await cache.set(cacheKey, JSON.stringify(data));
|
|
59
|
+
return {
|
|
60
|
+
cached: false,
|
|
61
|
+
data,
|
|
62
|
+
};
|
|
63
|
+
}
|
|
64
|
+
catch (err) {
|
|
65
|
+
throw new Error(`Error parsing response from ${url}: ${err}`);
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
exports.fetchJsonWithCache = fetchJsonWithCache;
|
|
69
|
+
function enableCache() {
|
|
70
|
+
enabled = true;
|
|
71
|
+
}
|
|
72
|
+
exports.enableCache = enableCache;
|
|
73
|
+
function disableCache() {
|
|
74
|
+
logger_js_1.default.info('Cache is disabled.');
|
|
75
|
+
enabled = false;
|
|
76
|
+
}
|
|
77
|
+
exports.disableCache = disableCache;
|
|
78
|
+
//# sourceMappingURL=cache.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cache.js","sourceRoot":"","sources":["../src/cache.ts"],"names":[],"mappings":";;;;;;AAAA,0DAA6B;AAE7B,kEAAyC;AACzC,kFAA4C;AAG5C,4DAAiC;AACjC,uCAAqE;AAKrE,IAAI,aAAgC,CAAC;AAErC,IAAI,OAAO,GACT,OAAO,OAAO,CAAC,GAAG,CAAC,uBAAuB,KAAK,WAAW;IACxD,CAAC,CAAC,IAAI;IACN,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,uBAAuB,CAAC,CAAC;AAEnD,MAAM,SAAS,GACb,OAAO,CAAC,GAAG,CAAC,oBAAoB,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,KAAK,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;AAE5F,SAAS,QAAQ;IACf,IAAI,CAAC,aAAa,EAAE;QAClB,aAAa,GAAG,uBAAY,CAAC,OAAO,CAAC;YACnC,KAAK,EAAE,SAAS,KAAK,MAAM,CAAC,CAAC,CAAC,+BAAO,CAAC,CAAC,CAAC,QAAQ;YAChD,OAAO,EAAE;gBACP,GAAG,EAAE,OAAO,CAAC,GAAG,CAAC,8BAA8B,IAAI,KAAM;gBACzD,IAAI,EAAE,OAAO,CAAC,GAAG,CAAC,oBAAoB,IAAI,mBAAI,CAAC,IAAI,CAAC,IAAA,gCAAsB,GAAE,EAAE,OAAO,CAAC;gBACtF,GAAG,EAAE,OAAO,CAAC,GAAG,CAAC,mBAAmB,IAAI,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE;gBACzD,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,wBAAwB,IAAI,GAAG,EAAE,iBAAiB;gBACvE,+CAA+C;aAChD;SACF,CAAC,CAAC;KACJ;IACD,OAAO,aAAa,CAAC;AACvB,CAAC;AAEM,KAAK,UAAU,kBAAkB,CACtC,GAAgB,EAChB,UAAuB,EAAE,EACzB,OAAe;IAEf,IAAI,CAAC,OAAO,EAAE;QACZ,MAAM,IAAI,GAAG,MAAM,IAAA,0BAAgB,EAAC,GAAG,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;QAC3D,OAAO;YACL,MAAM,EAAE,KAAK;YACb,IAAI,EAAE,MAAM,IAAI,CAAC,IAAI,EAAE;SACxB,CAAC;KACH;IAED,MAAM,KAAK,GAAG,MAAM,QAAQ,EAAE,CAAC;IAE/B,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC;IACxC,OAAO,IAAI,CAAC,OAAO,CAAC;IACpB,MAAM,QAAQ,GAAG,SAAS,GAAG,IAAI,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC;IAExD,iCAAiC;IACjC,MAAM,cAAc,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IAEjD,IAAI,cAAc,EAAE;QAClB,mBAAM,CAAC,KAAK,CAAC,iCAAiC,GAAG,KAAK,cAAc,EAAE,CAAC,CAAC;QACxE,OAAO;YACL,MAAM,EAAE,IAAI;YACZ,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,cAAwB,CAAC;SAC3C,CAAC;KACH;IAED,kDAAkD;IAClD,MAAM,QAAQ,GAAG,MAAM,IAAA,0BAAgB,EAAC,GAAG,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;IAC/D,IAAI;QACF,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;QACnC,mBAAM,CAAC,KAAK,CAAC,WAAW,GAAG,uBAAuB,IAAI,EAAE,CAAC,CAAC;QAC1D,MAAM,KAAK,CAAC,GAAG,CAAC,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC;QAChD,OAAO;YACL,MAAM,EAAE,KAAK;YACb,IAAI;SACL,CAAC;KACH;IAAC,OAAO,GAAG,EAAE;QACZ,MAAM,IAAI,KAAK,CAAC,+BAA+B,GAAG,KAAK,GAAG,EAAE,CAAC,CAAC;KAC/D;AACH,CAAC;AA3CD,gDA2CC;AAED,SAAgB,WAAW;IACzB,OAAO,GAAG,IAAI,CAAC;AACjB,CAAC;AAFD,kCAEC;AAED,SAAgB,YAAY;IAC1B,mBAAM,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;IAClC,OAAO,GAAG,KAAK,CAAC;AAClB,CAAC;AAHD,oCAGC"}
|
package/dist/evaluator.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"evaluator.d.ts","sourceRoot":"","sources":["../src/evaluator.ts"],"names":[],"mappings":"AAUA,OAAO,KAAK,EAEV,eAAe,EAGf,eAAe,EAGhB,MAAM,YAAY,CAAC;
|
|
1
|
+
{"version":3,"file":"evaluator.d.ts","sourceRoot":"","sources":["../src/evaluator.ts"],"names":[],"mappings":"AAUA,OAAO,KAAK,EAEV,eAAe,EAGf,eAAe,EAGhB,MAAM,YAAY,CAAC;AAmRpB,wBAAgB,QAAQ,CAAC,OAAO,EAAE,eAAe,4BAGhD"}
|
package/dist/evaluator.js
CHANGED
|
@@ -45,6 +45,7 @@ class Evaluator {
|
|
|
45
45
|
total: 0,
|
|
46
46
|
prompt: 0,
|
|
47
47
|
completion: 0,
|
|
48
|
+
cached: 0,
|
|
48
49
|
},
|
|
49
50
|
};
|
|
50
51
|
}
|
|
@@ -87,6 +88,7 @@ class Evaluator {
|
|
|
87
88
|
this.stats.tokenUsage.total += response.tokenUsage?.total || 0;
|
|
88
89
|
this.stats.tokenUsage.prompt += response.tokenUsage?.prompt || 0;
|
|
89
90
|
this.stats.tokenUsage.completion += response.tokenUsage?.completion || 0;
|
|
91
|
+
this.stats.tokenUsage.cached += response.tokenUsage?.cached || 0;
|
|
90
92
|
if (ret.success) {
|
|
91
93
|
this.stats.successes++;
|
|
92
94
|
}
|
package/dist/evaluator.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"evaluator.js","sourceRoot":"","sources":["../src/evaluator.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,kEAAqC;AAErC,kDAA0B;AAC1B,kDAA0B;AAC1B,wDAAgC;AAEhC,4DAAiC;AACjC,mDAAuD;AAYvD,qDAAmD;AAYnD,MAAM,uBAAuB,GAAG,CAAC,CAAC;AAElC,MAAM,SAAS;IAIb,YAAY,OAAwB;QAClC,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,CAAC,KAAK,GAAG;YACX,SAAS,EAAE,CAAC;YACZ,QAAQ,EAAE,CAAC;YACX,UAAU,EAAE;gBACV,KAAK,EAAE,CAAC;gBACR,MAAM,EAAE,CAAC;gBACT,UAAU,EAAE,CAAC;
|
|
1
|
+
{"version":3,"file":"evaluator.js","sourceRoot":"","sources":["../src/evaluator.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,kEAAqC;AAErC,kDAA0B;AAC1B,kDAA0B;AAC1B,wDAAgC;AAEhC,4DAAiC;AACjC,mDAAuD;AAYvD,qDAAmD;AAYnD,MAAM,uBAAuB,GAAG,CAAC,CAAC;AAElC,MAAM,SAAS;IAIb,YAAY,OAAwB;QAClC,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,CAAC,KAAK,GAAG;YACX,SAAS,EAAE,CAAC;YACZ,QAAQ,EAAE,CAAC;YACX,UAAU,EAAE;gBACV,KAAK,EAAE,CAAC;gBACR,MAAM,EAAE,CAAC;gBACT,UAAU,EAAE,CAAC;gBACb,MAAM,EAAE,CAAC;aACV;SACF,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,EACZ,QAAQ,EACR,MAAM,EACN,IAAI,EACJ,iBAAiB,GACF;QACf,IAAI,GAAG,IAAI,IAAI,EAAE,CAAC;QAClB,MAAM,cAAc,GAAG,kBAAQ,CAAC,YAAY,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;QAE3D,4DAA4D;QAC5D,MAAM,aAAa,GAAG,iBAAiB,CAAC,CAAC,CAAC,IAAI,QAAQ,CAAC,EAAE,EAAE,KAAK,MAAM,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;QAElF,MAAM,KAAK,GAAG;YACZ,MAAM,EAAE;gBACN,GAAG,EAAE,cAAc;gBACnB,OAAO,EAAE,aAAa;aACvB;YACD,IAAI;SACL,CAAC;QAEF,IAAI;YACF,MAAM,QAAQ,GAAG,MAAM,QAAQ,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;YACxD,MAAM,GAAG,GAAmB;gBAC1B,GAAG,KAAK;gBACR,QAAQ;gBACR,OAAO,EAAE,KAAK;aACf,CAAC;YACF,IAAI,QAAQ,CAAC,KAAK,EAAE;gBAClB,GAAG,CAAC,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC;aAC5B;iBAAM,IAAI,QAAQ,CAAC,MAAM,EAAE;gBAC1B,MAAM,WAAW,GAAG,IAAI,CAAC,UAAU;oBACjC,CAAC,CAAC,MAAM,IAAA,oCAAoB,EAAC,IAAI,CAAC,UAAU,EAAE,QAAQ,CAAC,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC;oBAC5E,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;gBACnB,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE;oBACrB,GAAG,CAAC,KAAK,GAAG,WAAW,CAAC,MAAM,IAAI,aAAa,IAAI,CAAC,UAAU,EAAE,CAAC;iBAClE;gBACD,GAAG,CAAC,OAAO,GAAG,WAAW,CAAC,IAAI,CAAC;aAChC;iBAAM;gBACL,GAAG,CAAC,OAAO,GAAG,KAAK,CAAC;gBACpB,GAAG,CAAC,KAAK,GAAG,WAAW,CAAC;aACzB;YAED,2BAA2B;YAC3B,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,KAAK,IAAI,QAAQ,CAAC,UAAU,EAAE,KAAK,IAAI,CAAC,CAAC;YAC/D,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,MAAM,IAAI,QAAQ,CAAC,UAAU,EAAE,MAAM,IAAI,CAAC,CAAC;YACjE,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,UAAU,IAAI,QAAQ,CAAC,UAAU,EAAE,UAAU,IAAI,CAAC,CAAC;YACzE,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,MAAM,IAAI,QAAQ,CAAC,UAAU,EAAE,MAAM,IAAI,CAAC,CAAC;YAEjE,IAAI,GAAG,CAAC,OAAO,EAAE;gBACf,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC;aACxB;iBAAM;gBACL,IAAI,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC;aACvB;YAED,OAAO,GAAG,CAAC;SACZ;QAAC,OAAO,GAAG,EAAE;YACZ,OAAO;gBACL,GAAG,KAAK;gBACR,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC;gBAClB,OAAO,EAAE,KAAK;aACf,CAAC;SACH;IACH,CAAC;IAED,KAAK,CAAC,QAAQ;QACZ,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;QAC7B,MAAM,OAAO,GAAa,EAAE,CAAC;QAE7B,IAAI,OAAO,CAAC,MAAM,EAAE,mBAAmB,EAAE;YACvC,mBAAM,CAAC,IAAI,CAAC,iCAAiC,CAAC,CAAC;YAC/C,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,KAAK,EAAE,GAAG,MAAM,IAAA,gCAAe,EAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;YACpF,IAAI,KAAK,IAAI,CAAC,UAAU,EAAE;gBACxB,MAAM,IAAI,KAAK,CAAC,+BAA+B,KAAK,EAAE,CAAC,CAAC;aACzD;YAED,mBAAM,CAAC,IAAI,CAAC,eAAK,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC,CAAC;YAC9C,IAAI,QAAQ,GAAG,CAAC,CAAC;YACjB,KAAK,MAAM,MAAM,IAAI,UAAU,EAAE;gBAC/B,mBAAM,CAAC,IAAI,CAAC,0DAA0D,CAAC,CAAC;gBACxE,mBAAM,CAAC,IAAI,CAAC,GAAG,MAAM,EAAE,CAAC,CAAC;gBACzB,mBAAM,CAAC,IAAI,CAAC,0DAA0D,CAAC,CAAC;gBAExE,wCAAwC;gBACxC,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;oBAC5B,MAAM,EAAE,GAAG,uBAAQ,CAAC,eAAe,CAAC;wBAClC,KAAK,EAAE,OAAO,CAAC,KAAK;wBACpB,MAAM,EAAE,OAAO,CAAC,MAAM;qBACvB,CAAC,CAAC;oBACH,EAAE,CAAC,QAAQ,CACT,GAAG,eAAK,CAAC,IAAI,CAAC,kCAAkC,CAAC,UAAU,EAC3D,KAAK,EAAE,MAAM,EAAE,EAAE;wBACf,EAAE,CAAC,KAAK,EAAE,CAAC;wBACX,IAAI,MAAM,CAAC,WAAW,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE;4BACxC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;4BAC7B,QAAQ,EAAE,CAAC;yBACZ;6BAAM;4BACL,mBAAM,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;yBACtC;wBACD,OAAO,CAAC,IAAI,CAAC,CAAC;oBAChB,CAAC,CACF,CAAC;gBACJ,CAAC,CAAC,CAAC;aACJ;YAED,IAAI,QAAQ,GAAG,CAAC,EAAE;gBAChB,mBAAM,CAAC,IAAI,CAAC,eAAK,CAAC,GAAG,CAAC,gCAAgC,CAAC,CAAC,CAAC;gBACzD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;aACjB;SACF;QAED,KAAK,MAAM,aAAa,IAAI,OAAO,CAAC,OAAO,EAAE;YAC3C,KAAK,MAAM,QAAQ,IAAI,OAAO,CAAC,SAAS,EAAE;gBACxC,MAAM,OAAO,GACX,OAAO,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,QAAQ,CAAC,EAAE,EAAE,KAAK,aAAa,EAAE,CAAC,CAAC,CAAC,aAAa,CAAC;gBACvF,OAAO,CAAC,IAAI,CAAC;oBACX,GAAG,EAAE,aAAa;oBAClB,OAAO;iBACR,CAAC,CAAC;aACJ;SACF;QAED,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QAC3E,MAAM,0BAA0B,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;YAChD,MAAM,GAAG,GAAG,EAAE,GAAG,CAAC,EAAE,CAAC;YACrB,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE;gBAC/B,IAAI,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE;oBACxB,OAAO,GAAG,CAAC,GAAG,CAAC,CAAC;iBACjB;YACH,CAAC,CAAC,CAAC;YACH,OAAO,GAAG,CAAC;QACb,CAAC,CAAC,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC;QAClC,MAAM,KAAK,GAAkB;YAC3B,IAAI,EAAE;gBACJ,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC;gBACtC,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC,CAAC,CAAC;aACjD;YACD,IAAI,EAAE,EAAE;SACT,CAAC;QAEF,IAAI,WAAkC,CAAC;QACvC,IAAI,OAAO,CAAC,eAAe,EAAE;YAC3B,MAAM,YAAY,GAChB,OAAO,CAAC,OAAO,CAAC,MAAM,GAAG,OAAO,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,MAAM,IAAI,CAAC,CAAC,CAAC;YAClF,MAAM,WAAW,GAAG,wDAAa,cAAc,GAAC,CAAC;YACjD,WAAW,GAAG,IAAI,WAAW,CAAC,SAAS,CACrC;gBACE,MAAM,EACJ,4FAA4F;aAC/F,EACD,WAAW,CAAC,OAAO,CAAC,cAAc,CACnC,CAAC;YACF,WAAW,CAAC,KAAK,CAAC,YAAY,EAAE,CAAC,EAAE;gBACjC,QAAQ,EAAE,EAAE;gBACZ,MAAM,EAAE,EAAE;gBACV,IAAI,EAAE,EAAE;aACT,CAAC,CAAC;SACJ;QAED,MAAM,cAAc,GAAqB,EAAE,CAAC;QAC5C,IAAI,QAAQ,GAAG,CAAC,CAAC;QACjB,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE;YACtB,IAAI,QAAQ,GAAG,CAAC,CAAC;YAEjB,MAAM,eAAe,GAAG,GAAG,CAAC,QAAQ,IAAI,OAAO,CAAC,MAAM,EAAE,MAAM,IAAI,EAAE,CAAC;YACrE,MAAM,cAAc,GAAG,GAAG,CAAC,QAAQ,IAAI,OAAO,CAAC,MAAM,EAAE,MAAM,IAAI,EAAE,CAAC;YAEpE,KAAK,MAAM,aAAa,IAAI,OAAO,CAAC,OAAO,EAAE;gBAC3C,KAAK,MAAM,QAAQ,IAAI,OAAO,CAAC,SAAS,EAAE;oBACxC,cAAc,CAAC,IAAI,CAAC;wBAClB,QAAQ;wBACR,MAAM,EAAE,eAAe,GAAG,aAAa,GAAG,cAAc;wBACxD,IAAI,EAAE,GAAG;wBACT,iBAAiB,EAAE,OAAO,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC;wBAC/C,QAAQ;wBACR,QAAQ;qBACT,CAAC,CAAC;oBACH,QAAQ,EAAE,CAAC;iBACZ;aACF;YACD,QAAQ,EAAE,CAAC;SACZ;QAED,MAAM,OAAO,GAAqB,EAAE,CAAC;QACrC,MAAM,eAAK,CAAC,cAAc,CACxB,cAAc,EACd,OAAO,CAAC,cAAc,IAAI,uBAAuB,EACjD,KAAK,EAAE,OAAuB,EAAE,KAAsB,EAAE,EAAE;YACxD,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;YAExC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAElB,IAAI,WAAW,EAAE;gBACf,WAAW,CAAC,SAAS,CAAC;oBACpB,QAAQ,EAAE,OAAO,CAAC,QAAQ,CAAC,EAAE,EAAE;oBAC/B,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC;oBACnC,IAAI,EAAE,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,IAAI,EAAE,CAAC;yBACrC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;yBAC5B,IAAI,CAAC,GAAG,CAAC;yBACT,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC;iBAChB,CAAC,CAAC;aACJ;YAED,wBAAwB;YACxB,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE;gBAC7B,MAAM,IAAI,KAAK,CAAC,+BAA+B,CAAC,CAAC;aAClD;YAED,IAAI,UAA8B,CAAC;YACnC,IAAI,MAAM,EAAE;gBACV,IAAI,GAAG,CAAC,OAAO,EAAE;oBACf,UAAU,GAAG,UAAU,GAAG,CAAC,QAAQ,EAAE,MAAM,IAAI,GAAG,CAAC,KAAK,IAAI,EAAE,EAAE,CAAC;iBAClE;qBAAM;oBACL,UAAU,GAAG,UAAU,GAAG,CAAC,KAAK,UAAU,GAAG,CAAC,QAAQ,EAAE,MAAM,IAAI,GAAG,CAAC,KAAK,IAAI,EAAE,EAAE,CAAC;iBACrF;aACF;iBAAM,IAAI,GAAG,CAAC,KAAK,EAAE;gBACpB,UAAU,GAAG,UAAU,GAAG,CAAC,KAAK,EAAE,CAAC;aACpC;iBAAM;gBACL,UAAU,GAAG,GAAG,CAAC,QAAQ,EAAE,MAAM,IAAI,GAAG,CAAC,KAAK,IAAI,EAAE,CAAC;aACtD;YAED,sEAAsE;YACtE,uCAAuC;YACvC,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,GAAG,OAAO,CAAC;YACvC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE;gBACzB,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG;oBACrB,OAAO,EAAE,EAAE;oBACX,IAAI,EAAE,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,IAAI,EAAE,CAAC;iBACxC,CAAC;aACH;YACD,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG,UAAU,CAAC;QACtD,CAAC,CACF,CAAC;QAEF,IAAI,WAAW,EAAE;YACf,WAAW,CAAC,IAAI,EAAE,CAAC;SACpB;QAED,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,KAAK,EAAE,CAAC;IAC3D,CAAC;CACF;AAED,SAAgB,QAAQ,CAAC,OAAwB;IAC/C,MAAM,EAAE,GAAG,IAAI,SAAS,CAAC,OAAO,CAAC,CAAC;IAClC,OAAO,EAAE,CAAC,QAAQ,EAAE,CAAC;AACvB,CAAC;AAHD,4BAGC"}
|
package/dist/main.js
CHANGED
|
@@ -38,6 +38,7 @@ const evaluator_js_1 = require("./evaluator.js");
|
|
|
38
38
|
const util_js_1 = require("./util.js");
|
|
39
39
|
const esm_js_1 = require("./esm.js");
|
|
40
40
|
const server_js_1 = require("./web/server.js");
|
|
41
|
+
const cache_js_1 = require("./cache.js");
|
|
41
42
|
function createDummyFiles(directory) {
|
|
42
43
|
if (directory) {
|
|
43
44
|
// Make the directory if it doesn't exist
|
|
@@ -134,13 +135,11 @@ async function main() {
|
|
|
134
135
|
.option('--prompt-prefix <path>', 'This prefix is prepended to every prompt', defaultConfig.promptPrefix)
|
|
135
136
|
.option('--prompt-suffix <path>', 'This suffix is append to every prompt', defaultConfig.promptSuffix)
|
|
136
137
|
.option('--no-write', 'Do not write results to promptfoo directory')
|
|
138
|
+
.option('--no-cache', 'Do not read or write results to disk cache')
|
|
137
139
|
.option('--grader', 'Model that will grade outputs', defaultConfig.grader)
|
|
138
140
|
.option('--verbose', 'Show debug logs', defaultConfig.verbose)
|
|
139
141
|
.option('--view [port]', 'View in browser ui')
|
|
140
142
|
.action(async (cmdObj) => {
|
|
141
|
-
if (cmdObj.verbose) {
|
|
142
|
-
(0, logger_js_1.setLogLevel)('debug');
|
|
143
|
-
}
|
|
144
143
|
const configPath = cmdObj.config;
|
|
145
144
|
let config = {};
|
|
146
145
|
if (configPath) {
|
|
@@ -157,6 +156,12 @@ async function main() {
|
|
|
157
156
|
throw new Error(`Unsupported configuration file format: ${ext}`);
|
|
158
157
|
}
|
|
159
158
|
}
|
|
159
|
+
if (cmdObj.verbose) {
|
|
160
|
+
(0, logger_js_1.setLogLevel)('debug');
|
|
161
|
+
}
|
|
162
|
+
if (!cmdObj.cache) {
|
|
163
|
+
(0, cache_js_1.disableCache)();
|
|
164
|
+
}
|
|
160
165
|
let vars = [];
|
|
161
166
|
if (cmdObj.vars) {
|
|
162
167
|
vars = (0, util_js_1.readVars)(cmdObj.vars);
|
|
@@ -228,7 +233,7 @@ async function main() {
|
|
|
228
233
|
}
|
|
229
234
|
logger_js_1.default.info('\n' + table.toString());
|
|
230
235
|
}
|
|
231
|
-
if (cmdObj.
|
|
236
|
+
if (cmdObj.view || !cmdObj.write) {
|
|
232
237
|
logger_js_1.default.info('Evaluation complete');
|
|
233
238
|
}
|
|
234
239
|
else {
|
|
@@ -237,7 +242,7 @@ async function main() {
|
|
|
237
242
|
}
|
|
238
243
|
logger_js_1.default.info(chalk_1.default.green.bold(`Successes: ${summary.stats.successes}`));
|
|
239
244
|
logger_js_1.default.info(chalk_1.default.red.bold(`Failures: ${summary.stats.failures}`));
|
|
240
|
-
logger_js_1.default.info(`Token usage: Total ${summary.stats.tokenUsage.total} Prompt ${summary.stats.tokenUsage.prompt} Completion ${summary.stats.tokenUsage.completion}`);
|
|
245
|
+
logger_js_1.default.info(`Token usage: Total ${summary.stats.tokenUsage.total}, Prompt ${summary.stats.tokenUsage.prompt}, Completion ${summary.stats.tokenUsage.completion}, Cached ${summary.stats.tokenUsage.cached}`);
|
|
241
246
|
logger_js_1.default.info('Done.');
|
|
242
247
|
if (cmdObj.view) {
|
|
243
248
|
(0, server_js_1.init)(parseInt(cmdObj.view, 10) || 15500);
|
package/dist/main.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"main.js","sourceRoot":"","sources":["../src/main.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;AACA,2BAAwE;AACxE,+BAA+C;AAE/C,4DAA+B;AAC/B,kDAA0B;AAC1B,yCAAoC;AAEpC,yDAAkD;AAClD,iDAAiD;AACjD,iDAA0C;AAC1C,uCAAmF;AACnF,qCAAwC;AACxC,+CAAuC;
|
|
1
|
+
{"version":3,"file":"main.js","sourceRoot":"","sources":["../src/main.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;AACA,2BAAwE;AACxE,+BAA+C;AAE/C,4DAA+B;AAC/B,kDAA0B;AAC1B,yCAAoC;AAEpC,yDAAkD;AAClD,iDAAiD;AACjD,iDAA0C;AAC1C,uCAAmF;AACnF,qCAAwC;AACxC,+CAAuC;AAGvC,yCAA0C;AAE1C,SAAS,gBAAgB,CAAC,SAAwB;IAChD,IAAI,SAAS,EAAE;QACb,yCAAyC;QACzC,IAAI,CAAC,IAAA,eAAU,EAAC,SAAS,CAAC,EAAE;YAC1B,IAAA,cAAS,EAAC,SAAS,CAAC,CAAC;SACtB;KACF;IACD,MAAM,YAAY,GAAG;;;;;;;;;YASX,CAAC;IACX,MAAM,SAAS,GACb,oFAAoF,CAAC;IACvF,MAAM,WAAW,GAAG;;;;;GAKnB,CAAC;IACF,MAAM,MAAM,GAAG;;;;;;CAMhB,CAAC;IAEA,IAAI,SAAS,EAAE;QACb,IAAI,CAAC,IAAA,eAAU,EAAC,SAAS,CAAC,EAAE;YAC1B,mBAAM,CAAC,IAAI,CAAC,sBAAsB,SAAS,MAAM,CAAC,CAAC;YACnD,IAAA,cAAS,EAAC,SAAS,CAAC,CAAC;SACtB;KACF;SAAM;QACL,SAAS,GAAG,GAAG,CAAC;KACjB;IAED,IAAA,kBAAa,EAAC,IAAA,WAAQ,EAAC,OAAO,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,aAAa,CAAC,EAAE,YAAY,CAAC,CAAC;IAC/E,IAAA,kBAAa,EAAC,IAAA,WAAQ,EAAC,OAAO,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,UAAU,CAAC,EAAE,SAAS,CAAC,CAAC;IACzE,IAAA,kBAAa,EAAC,IAAA,WAAQ,EAAC,OAAO,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,oBAAoB,CAAC,EAAE,WAAW,CAAC,CAAC;IACrF,IAAA,kBAAa,EAAC,IAAA,WAAQ,EAAC,OAAO,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,WAAW,CAAC,EAAE,MAAM,CAAC,CAAC;IAEvE,IAAI,SAAS,KAAK,GAAG,EAAE;QACrB,mBAAM,CAAC,IAAI,CACT,qFAAqF,CACtF,CAAC;KACH;SAAM;QACL,mBAAM,CAAC,IAAI,CAAC,0DAA0D,SAAS,EAAE,CAAC,CAAC;QACnF,mBAAM,CAAC,IAAI,CAAC,QAAQ,SAAS,uCAAuC,CAAC,CAAC;KACvE;AACH,CAAC;AAED,KAAK,UAAU,IAAI;IACjB,IAAI,aAAa,GAAgC,EAAE,CAAC;IACpD,IAAI,IAAA,eAAU,EAAC,oBAAoB,CAAC,EAAE;QACpC,aAAa;QACb,aAAa,GAAG,CAAC,yBAAa,IAAA,WAAQ,EAAC,OAAO,CAAC,GAAG,EAAE,EAAE,sBAAsB,CAAC,uCAAC,CAAC,CAAC,OAAO,CAAC;QACxF,mBAAM,CAAC,IAAI,CAAC,+CAA+C,CAAC,CAAC;KAC9D;IACD,IAAI,IAAA,eAAU,EAAC,sBAAsB,CAAC,EAAE;QACtC,aAAa,GAAG,IAAI,CAAC,KAAK,CAAC,IAAA,iBAAY,EAAC,sBAAsB,EAAE,OAAO,CAAC,CAAC,CAAC;QAC1E,mBAAM,CAAC,IAAI,CAAC,iDAAiD,CAAC,CAAC;KAChE;IAED,MAAM,OAAO,GAAG,IAAI,mBAAO,EAAE,CAAC;IAE9B,OAAO,CAAC,MAAM,CAAC,WAAW,EAAE,eAAe,EAAE,GAAG,EAAE;QAChD,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAC5B,IAAA,iBAAY,EAAC,IAAA,WAAQ,EAAC,IAAA,qBAAY,GAAE,EAAE,iBAAiB,CAAC,EAAE,MAAM,CAAC,CAClE,CAAC;QACF,OAAO,CAAC,GAAG,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;QACjC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,CAAC;IAEH,OAAO;SACJ,OAAO,CAAC,kBAAkB,CAAC;SAC3B,WAAW,CAAC,qCAAqC,CAAC;SAClD,MAAM,CAAC,CAAC,SAAwB,EAAE,EAAE;QACnC,gBAAgB,CAAC,SAAS,CAAC,CAAC;IAC9B,CAAC,CAAC,CAAC;IAEL,OAAO;SACJ,OAAO,CAAC,MAAM,CAAC;SACf,WAAW,CAAC,kBAAkB,CAAC;SAC/B,MAAM,CAAC,qBAAqB,EAAE,aAAa,EAAE,OAAO,CAAC;SACrD,MAAM,CAAC,CAAC,MAAkC,EAAE,EAAE;QAC7C,IAAA,gBAAI,EAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACpB,CAAC,CAAC,CAAC;IAEL,OAAO;SACJ,OAAO,CAAC,MAAM,CAAC;SACf,WAAW,CAAC,kBAAkB,CAAC;SAC/B,cAAc,CACb,0BAA0B,EAC1B,8BAA8B,EAC9B,aAAa,CAAC,OAAO,CACtB;SACA,cAAc,CACb,mCAAmC,EACnC,kGAAkG,EAClG,aAAa,CAAC,SAAS,CACxB;SACA,MAAM,CACL,qBAAqB,EACrB,6CAA6C,EAC7C,aAAa,CAAC,MAAM,CACrB;SACA,MAAM,CACL,mBAAmB,EACnB,sDAAsD,EACtD,aAAa,CAAC,IAAI,CACnB;SACA,MAAM,CACL,qBAAqB,EACrB,oEAAoE,EACpE,aAAa,CAAC,MAAM,CACrB;SACA,MAAM,CACL,gCAAgC,EAChC,wCAAwC,EACxC,MAAM,CAAC,aAAa,CAAC,cAAc,CAAC,CACrC;SACA,MAAM,CACL,kCAAkC,EAClC,6CAA6C,EAC7C,KAAK,CACN;SACA,MAAM,CACL,4BAA4B,EAC5B,2DAA2D,CAC5D;SACA,MAAM,CACL,wBAAwB,EACxB,0CAA0C,EAC1C,aAAa,CAAC,YAAY,CAC3B;SACA,MAAM,CACL,wBAAwB,EACxB,uCAAuC,EACvC,aAAa,CAAC,YAAY,CAC3B;SACA,MAAM,CAAC,YAAY,EAAE,6CAA6C,CAAC;SACnE,MAAM,CAAC,YAAY,EAAE,4CAA4C,CAAC;SAClE,MAAM,CAAC,UAAU,EAAE,+BAA+B,EAAE,aAAa,CAAC,MAAM,CAAC;SACzE,MAAM,CAAC,WAAW,EAAE,iBAAiB,EAAE,aAAa,CAAC,OAAO,CAAC;SAC7D,MAAM,CAAC,eAAe,EAAE,oBAAoB,CAAC;SAC7C,MAAM,CAAC,KAAK,EAAE,MAAoC,EAAE,EAAE;QACrD,MAAM,UAAU,GAAG,MAAM,CAAC,MAAM,CAAC;QACjC,IAAI,MAAM,GAAG,EAAE,CAAC;QAChB,IAAI,UAAU,EAAE;YACd,MAAM,GAAG,GAAG,IAAA,YAAK,EAAC,UAAU,CAAC,CAAC,GAAG,CAAC;YAClC,QAAQ,GAAG,EAAE;gBACX,KAAK,OAAO;oBACV,MAAM,OAAO,GAAG,IAAA,iBAAY,EAAC,UAAU,EAAE,OAAO,CAAC,CAAC;oBAClD,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;oBAC7B,MAAM;gBACR,KAAK,KAAK;oBACR,MAAM,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;oBAC7B,MAAM;gBACR;oBACE,MAAM,IAAI,KAAK,CAAC,0CAA0C,GAAG,EAAE,CAAC,CAAC;aACpE;SACF;QAED,IAAI,MAAM,CAAC,OAAO,EAAE;YAClB,IAAA,uBAAW,EAAC,OAAO,CAAC,CAAC;SACtB;QACD,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE;YACjB,IAAA,uBAAY,GAAE,CAAC;SAChB;QAED,IAAI,IAAI,GAAiB,EAAE,CAAC;QAC5B,IAAI,MAAM,CAAC,IAAI,EAAE;YACf,IAAI,GAAG,IAAA,kBAAQ,EAAC,MAAM,CAAC,IAAI,CAAC,CAAC;SAC9B;QAED,MAAM,SAAS,GAAG,MAAM,OAAO,CAAC,GAAG,CACjC,MAAM,CAAC,SAAS,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC,MAAM,IAAA,8BAAe,EAAC,CAAC,CAAC,CAAC,CAC5D,CAAC;QACF,MAAM,cAAc,GAAG,QAAQ,CAAC,MAAM,CAAC,cAAc,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC;QACjE,MAAM,OAAO,GAAoB;YAC/B,OAAO,EAAE,IAAA,qBAAW,EAAC,MAAM,CAAC,OAAO,CAAC;YACpC,IAAI;YACJ,SAAS;YACT,eAAe,EAAE,IAAI;YACrB,cAAc,EAAE,CAAC,KAAK,CAAC,cAAc,CAAC,IAAI,cAAc,GAAG,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,SAAS;YACzF,MAAM,EAAE;gBACN,MAAM,EAAE,MAAM,CAAC,YAAY;gBAC3B,MAAM,EAAE,MAAM,CAAC,YAAY;aAC5B;YACD,GAAG,MAAM;SACV,CAAC;QAEF,IAAI,MAAM,CAAC,MAAM,EAAE;YACjB,OAAO,CAAC,OAAO,GAAG;gBAChB,QAAQ,EAAE,MAAM,IAAA,8BAAe,EAAC,MAAM,CAAC,MAAM,CAAC;aAC/C,CAAC;SACH;QACD,IAAI,MAAM,CAAC,mBAAmB,EAAE;YAC9B,OAAO,CAAC,MAAO,CAAC,mBAAmB,GAAG,IAAI,CAAC;SAC5C;QAED,MAAM,OAAO,GAAG,MAAM,IAAA,uBAAQ,EAAC,OAAO,CAAC,CAAC;QAExC,IAAI,MAAM,CAAC,MAAM,EAAE;YACjB,mBAAM,CAAC,IAAI,CAAC,eAAK,CAAC,MAAM,CAAC,qBAAqB,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;YAChE,IAAA,qBAAW,EAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;SACrC;aAAM;YACL,0BAA0B;YAC1B,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,GAAG,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC;YAC5E,MAAM,IAAI,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC;YAChC,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC;YAC1D,MAAM,KAAK,GAAG,IAAI,oBAAK,CAAC;gBACtB,IAAI,EAAE,CAAC,GAAG,IAAI,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,IAAI,CAAC;gBACrC,SAAS,EAAE,KAAK,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,GAAG,UAAU,CAAC,CAAC;gBACpE,QAAQ,EAAE,IAAI;gBACd,kBAAkB,EAAE,KAAK;gBACzB,KAAK,EAAE;oBACL,IAAI,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC;iBACvB;aACF,CAAC,CAAC;YACH,4DAA4D;YAC5D,KAAK,MAAM,GAAG,IAAI,OAAO,CAAC,KAAK,CAAC,IAAI,EAAE;gBACpC,KAAK,CAAC,IAAI,CAAC;oBACT,GAAG,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;wBACzB,MAAM,kBAAkB,GAAG,QAAQ,CAAC,MAAM,CAAC,kBAAkB,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC;wBACzE,IAAI,CAAC,KAAK,CAAC,kBAAkB,CAAC,IAAI,GAAG,CAAC,MAAM,GAAG,kBAAkB,EAAE;4BACjE,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,kBAAkB,CAAC,GAAG,KAAK,CAAC;yBAChD;wBACD,IAAI,GAAG,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE;4BAC5B,uBAAuB;4BACvB,OAAO,eAAK,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;yBACzD;6BAAM,IAAI,GAAG,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE;4BACnC,sCAAsC;4BACtC,OAAO,GAAG;iCACP,KAAK,CAAC,KAAK,CAAC;iCACZ,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,eAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;iCACpD,IAAI,CAAC,KAAK,CAAC,CAAC;yBAChB;wBACD,OAAO,GAAG,CAAC;oBACb,CAAC,CAAC;oBACF,GAAG,GAAG,CAAC,IAAI;iBACZ,CAAC,CAAC;aACJ;YAED,mBAAM,CAAC,IAAI,CAAC,IAAI,GAAG,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC;SACtC;QACD,IAAI,MAAM,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE;YAChC,mBAAM,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;SACpC;aAAM;YACL,IAAA,4BAAkB,EAAC,OAAO,CAAC,CAAC;YAC5B,mBAAM,CAAC,IAAI,CAAC,+CAA+C,eAAK,CAAC,KAAK,CAAC,gBAAgB,CAAC,EAAE,CAAC,CAAC;SAC7F;QACD,mBAAM,CAAC,IAAI,CAAC,eAAK,CAAC,KAAK,CAAC,IAAI,CAAC,cAAc,OAAO,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC;QACvE,mBAAM,CAAC,IAAI,CAAC,eAAK,CAAC,GAAG,CAAC,IAAI,CAAC,aAAa,OAAO,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC;QACnE,mBAAM,CAAC,IAAI,CACT,sBAAsB,OAAO,CAAC,KAAK,CAAC,UAAU,CAAC,KAAK,YAAY,OAAO,CAAC,KAAK,CAAC,UAAU,CAAC,MAAM,gBAAgB,OAAO,CAAC,KAAK,CAAC,UAAU,CAAC,UAAU,YAAY,OAAO,CAAC,KAAK,CAAC,UAAU,CAAC,MAAM,EAAE,CAChM,CAAC;QACF,mBAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAErB,IAAI,MAAM,CAAC,IAAI,EAAE;YACf,IAAA,gBAAI,EAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,KAAK,CAAC,CAAC;SAC1C;IACH,CAAC,CAAC,CAAC;IAEL,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IAE5B,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE;QACjC,OAAO,CAAC,UAAU,EAAE,CAAC;KACtB;AACH,CAAC;AAED,IAAI,EAAE,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"localai.d.ts","sourceRoot":"","sources":["../../src/providers/localai.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,WAAW,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAEjE,cAAM,sBAAuB,YAAW,WAAW;IACjD,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;gBAEP,SAAS,EAAE,MAAM;IAK7B,EAAE,IAAI,MAAM;IAIZ,QAAQ,IAAI,MAAM;IAKZ,OAAO,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,CAAC;CAGzD;AAED,qBAAa,mBAAoB,SAAQ,sBAAsB;IACvD,OAAO,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,CAAC;
|
|
1
|
+
{"version":3,"file":"localai.d.ts","sourceRoot":"","sources":["../../src/providers/localai.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,WAAW,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAEjE,cAAM,sBAAuB,YAAW,WAAW;IACjD,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;gBAEP,SAAS,EAAE,MAAM;IAK7B,EAAE,IAAI,MAAM;IAIZ,QAAQ,IAAI,MAAM;IAKZ,OAAO,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,CAAC;CAGzD;AAED,qBAAa,mBAAoB,SAAQ,sBAAsB;IACvD,OAAO,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,CAAC;CAsCzD;AAED,qBAAa,yBAA0B,SAAQ,sBAAsB;IAC7D,OAAO,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,CAAC;CAsCzD"}
|
|
@@ -5,7 +5,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
5
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
6
|
exports.LocalAiCompletionProvider = exports.LocalAiChatProvider = void 0;
|
|
7
7
|
const logger_js_1 = __importDefault(require("../logger.js"));
|
|
8
|
-
const
|
|
8
|
+
const cache_js_1 = require("../cache.js");
|
|
9
9
|
const shared_js_1 = require("./shared.js");
|
|
10
10
|
class LocalAiGenericProvider {
|
|
11
11
|
constructor(modelName) {
|
|
@@ -31,16 +31,15 @@ class LocalAiChatProvider extends LocalAiGenericProvider {
|
|
|
31
31
|
temperature: process.env.LOCALAI_TEMPERATURE || 0.7,
|
|
32
32
|
};
|
|
33
33
|
logger_js_1.default.debug(`Calling LocalAI API: ${JSON.stringify(body)}`);
|
|
34
|
-
let
|
|
34
|
+
let data, cached = false;
|
|
35
35
|
try {
|
|
36
|
-
|
|
36
|
+
({ data, cached } = (await (0, cache_js_1.fetchJsonWithCache)(`${this.apiBaseUrl}/chat/completions`, {
|
|
37
37
|
method: 'POST',
|
|
38
38
|
headers: {
|
|
39
39
|
'Content-Type': 'application/json',
|
|
40
40
|
},
|
|
41
41
|
body: JSON.stringify(body),
|
|
42
|
-
}, shared_js_1.REQUEST_TIMEOUT_MS);
|
|
43
|
-
data = (await response.json());
|
|
42
|
+
}, shared_js_1.REQUEST_TIMEOUT_MS)));
|
|
44
43
|
}
|
|
45
44
|
catch (err) {
|
|
46
45
|
return {
|
|
@@ -69,16 +68,15 @@ class LocalAiCompletionProvider extends LocalAiGenericProvider {
|
|
|
69
68
|
temperature: process.env.LOCALAI_TEMPERATURE || 0.7,
|
|
70
69
|
};
|
|
71
70
|
logger_js_1.default.debug(`Calling LocalAI API: ${JSON.stringify(body)}`);
|
|
72
|
-
let
|
|
71
|
+
let data, cached = false;
|
|
73
72
|
try {
|
|
74
|
-
|
|
73
|
+
({ data, cached } = (await (0, cache_js_1.fetchJsonWithCache)(`${this.apiBaseUrl}/completions`, {
|
|
75
74
|
method: 'POST',
|
|
76
75
|
headers: {
|
|
77
76
|
'Content-Type': 'application/json',
|
|
78
77
|
},
|
|
79
78
|
body: JSON.stringify(body),
|
|
80
|
-
}, shared_js_1.REQUEST_TIMEOUT_MS);
|
|
81
|
-
data = (await response.json());
|
|
79
|
+
}, shared_js_1.REQUEST_TIMEOUT_MS)));
|
|
82
80
|
}
|
|
83
81
|
catch (err) {
|
|
84
82
|
return {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"localai.js","sourceRoot":"","sources":["../../src/providers/localai.ts"],"names":[],"mappings":";;;;;;AAAA,6DAAkC;AAClC,
|
|
1
|
+
{"version":3,"file":"localai.js","sourceRoot":"","sources":["../../src/providers/localai.ts"],"names":[],"mappings":";;;;;;AAAA,6DAAkC;AAClC,0CAAiD;AACjD,2CAAiD;AAIjD,MAAM,sBAAsB;IAI1B,YAAY,SAAiB;QAC3B,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,UAAU,GAAG,OAAO,CAAC,GAAG,CAAC,gBAAgB,IAAI,0BAA0B,CAAC;IAC/E,CAAC;IAED,EAAE;QACA,OAAO,WAAW,IAAI,CAAC,SAAS,EAAE,CAAC;IACrC,CAAC;IAED,QAAQ;QACN,OAAO,qBAAqB,IAAI,CAAC,SAAS,GAAG,CAAC;IAChD,CAAC;IAED,wDAAwD;IACxD,KAAK,CAAC,OAAO,CAAC,MAAc;QAC1B,MAAM,IAAI,KAAK,CAAC,iBAAiB,CAAC,CAAC;IACrC,CAAC;CACF;AAED,MAAa,mBAAoB,SAAQ,sBAAsB;IAC7D,KAAK,CAAC,OAAO,CAAC,MAAc;QAC1B,MAAM,IAAI,GAAG;YACX,KAAK,EAAE,IAAI,CAAC,SAAS;YACrB,MAAM;YACN,WAAW,EAAE,OAAO,CAAC,GAAG,CAAC,mBAAmB,IAAI,GAAG;SACpD,CAAC;QACF,mBAAM,CAAC,KAAK,CAAC,wBAAwB,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAE7D,IAAI,IAAI,EACN,MAAM,GAAG,KAAK,CAAC;QACjB,IAAI;YACF,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,IAAA,6BAAkB,EAC3C,GAAG,IAAI,CAAC,UAAU,mBAAmB,EACrC;gBACE,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE;oBACP,cAAc,EAAE,kBAAkB;iBACnC;gBACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;aAC3B,EACD,8BAAkB,CACnB,CAAmB,CAAC,CAAC;SACvB;QAAC,OAAO,GAAG,EAAE;YACZ,OAAO;gBACL,KAAK,EAAE,mBAAmB,MAAM,CAAC,GAAG,CAAC,EAAE;aACxC,CAAC;SACH;QACD,mBAAM,CAAC,KAAK,CAAC,0BAA0B,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC/D,IAAI;YACF,OAAO;gBACL,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI;aAC7B,CAAC;SACH;QAAC,OAAO,GAAG,EAAE;YACZ,OAAO;gBACL,KAAK,EAAE,uBAAuB,MAAM,CAAC,GAAG,CAAC,KAAK,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE;aACrE,CAAC;SACH;IACH,CAAC;CACF;AAvCD,kDAuCC;AAED,MAAa,yBAA0B,SAAQ,sBAAsB;IACnE,KAAK,CAAC,OAAO,CAAC,MAAc;QAC1B,MAAM,IAAI,GAAG;YACX,KAAK,EAAE,IAAI,CAAC,SAAS;YACrB,MAAM;YACN,WAAW,EAAE,OAAO,CAAC,GAAG,CAAC,mBAAmB,IAAI,GAAG;SACpD,CAAC;QACF,mBAAM,CAAC,KAAK,CAAC,wBAAwB,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAE7D,IAAI,IAAI,EACN,MAAM,GAAG,KAAK,CAAC;QACjB,IAAI;YACF,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,IAAA,6BAAkB,EAC3C,GAAG,IAAI,CAAC,UAAU,cAAc,EAChC;gBACE,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE;oBACP,cAAc,EAAE,kBAAkB;iBACnC;gBACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;aAC3B,EACD,8BAAkB,CACnB,CAAmB,CAAC,CAAC;SACvB;QAAC,OAAO,GAAG,EAAE;YACZ,OAAO;gBACL,KAAK,EAAE,mBAAmB,MAAM,CAAC,GAAG,CAAC,EAAE;aACxC,CAAC;SACH;QACD,mBAAM,CAAC,KAAK,CAAC,0BAA0B,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC/D,IAAI;YACF,OAAO;gBACL,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI;aAC7B,CAAC;SACH;QAAC,OAAO,GAAG,EAAE;YACZ,OAAO;gBACL,KAAK,EAAE,uBAAuB,MAAM,CAAC,GAAG,CAAC,KAAK,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE;aACrE,CAAC;SACH;IACH,CAAC;CACF;AAvCD,8DAuCC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"openai.d.ts","sourceRoot":"","sources":["../../src/providers/openai.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"openai.d.ts","sourceRoot":"","sources":["../../src/providers/openai.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,WAAW,EAAE,yBAAyB,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAI5F,UAAU,uBAAuB;IAC/B,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,cAAM,qBAAsB,YAAW,WAAW;IAChD,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;gBAEJ,SAAS,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM;IAQ9C,EAAE,IAAI,MAAM;IAIZ,QAAQ,IAAI,MAAM;IAKZ,OAAO,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,uBAAuB,GAAG,OAAO,CAAC,gBAAgB,CAAC;CAG5F;AAED,qBAAa,uBAAwB,SAAQ,qBAAqB;IAC1D,gBAAgB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,yBAAyB,CAAC;CA+DzE;AAED,qBAAa,wBAAyB,SAAQ,qBAAqB;IACjE,MAAM,CAAC,wBAAwB,WAM7B;gBAEU,SAAS,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM;IAOxC,OAAO,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,uBAAuB,GAAG,OAAO,CAAC,gBAAgB,CAAC;CAqD5F;AAED,qBAAa,4BAA6B,SAAQ,qBAAqB;IACrE,MAAM,CAAC,kBAAkB,WAOvB;gBAEU,SAAS,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM;IAQxC,OAAO,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,uBAAuB,GAAG,OAAO,CAAC,gBAAgB,CAAC;CA8D5F;AAED,eAAO,MAAM,wBAAwB,yBAAwD,CAAC;AAC9F,eAAO,MAAM,sBAAsB,8BAA4C,CAAC;AAChF,eAAO,MAAM,0BAA0B,8BAA4C,CAAC"}
|
package/dist/providers/openai.js
CHANGED
|
@@ -4,14 +4,10 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
4
4
|
};
|
|
5
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
6
|
exports.DefaultSuggestionsProvider = exports.DefaultGradingProvider = exports.DefaultEmbeddingProvider = exports.OpenAiChatCompletionProvider = exports.OpenAiCompletionProvider = exports.OpenAiEmbeddingProvider = void 0;
|
|
7
|
-
const lru_cache_1 = require("lru-cache");
|
|
8
7
|
const logger_js_1 = __importDefault(require("../logger.js"));
|
|
9
|
-
const
|
|
8
|
+
const cache_js_1 = require("../cache.js");
|
|
10
9
|
const shared_js_1 = require("./shared.js");
|
|
11
10
|
const DEFAULT_OPENAI_HOST = 'api.openai.com';
|
|
12
|
-
const embeddingsCache = new lru_cache_1.LRUCache({
|
|
13
|
-
max: 1000,
|
|
14
|
-
});
|
|
15
11
|
class OpenAiGenericProvider {
|
|
16
12
|
constructor(modelName, apiKey) {
|
|
17
13
|
this.modelName = modelName;
|
|
@@ -34,26 +30,20 @@ class OpenAiEmbeddingProvider extends OpenAiGenericProvider {
|
|
|
34
30
|
if (!this.apiKey) {
|
|
35
31
|
throw new Error('OpenAI API key must be set for similarity comparison');
|
|
36
32
|
}
|
|
37
|
-
// TODO(ian): Improve cache
|
|
38
|
-
const cached = embeddingsCache.get(text);
|
|
39
|
-
if (cached) {
|
|
40
|
-
return cached;
|
|
41
|
-
}
|
|
42
33
|
const body = {
|
|
43
34
|
input: text,
|
|
44
35
|
model: this.modelName,
|
|
45
36
|
};
|
|
46
|
-
let
|
|
37
|
+
let data, cached = false;
|
|
47
38
|
try {
|
|
48
|
-
|
|
39
|
+
({ data, cached } = (await (0, cache_js_1.fetchJsonWithCache)(`https://${this.apiHost}/v1/embeddings`, {
|
|
49
40
|
method: 'POST',
|
|
50
41
|
headers: {
|
|
51
42
|
'Content-Type': 'application/json',
|
|
52
43
|
Authorization: `Bearer ${this.apiKey}`,
|
|
53
44
|
},
|
|
54
45
|
body: JSON.stringify(body),
|
|
55
|
-
}, shared_js_1.REQUEST_TIMEOUT_MS);
|
|
56
|
-
data = (await response.json());
|
|
46
|
+
}, shared_js_1.REQUEST_TIMEOUT_MS)));
|
|
57
47
|
}
|
|
58
48
|
catch (err) {
|
|
59
49
|
return {
|
|
@@ -73,13 +63,14 @@ class OpenAiEmbeddingProvider extends OpenAiGenericProvider {
|
|
|
73
63
|
}
|
|
74
64
|
const ret = {
|
|
75
65
|
embedding,
|
|
76
|
-
tokenUsage:
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
66
|
+
tokenUsage: cached
|
|
67
|
+
? { cached: data.usage.total_tokens }
|
|
68
|
+
: {
|
|
69
|
+
total: data.usage.total_tokens,
|
|
70
|
+
prompt: data.usage.prompt_tokens,
|
|
71
|
+
completion: data.usage.completion_tokens,
|
|
72
|
+
},
|
|
81
73
|
};
|
|
82
|
-
embeddingsCache.set(text, ret);
|
|
83
74
|
return ret;
|
|
84
75
|
}
|
|
85
76
|
catch (err) {
|
|
@@ -114,17 +105,16 @@ class OpenAiCompletionProvider extends OpenAiGenericProvider {
|
|
|
114
105
|
stop: process.env.OPENAI_STOP ? JSON.parse(process.env.OPENAI_STOP) : undefined,
|
|
115
106
|
};
|
|
116
107
|
logger_js_1.default.debug(`Calling OpenAI API: ${JSON.stringify(body)}`);
|
|
117
|
-
let
|
|
108
|
+
let data, cached = false;
|
|
118
109
|
try {
|
|
119
|
-
|
|
110
|
+
({ data, cached } = (await (0, cache_js_1.fetchJsonWithCache)(`https://${this.apiHost}/v1/completions`, {
|
|
120
111
|
method: 'POST',
|
|
121
112
|
headers: {
|
|
122
113
|
'Content-Type': 'application/json',
|
|
123
114
|
Authorization: `Bearer ${this.apiKey}`,
|
|
124
115
|
},
|
|
125
116
|
body: JSON.stringify(body),
|
|
126
|
-
}, shared_js_1.REQUEST_TIMEOUT_MS);
|
|
127
|
-
data = (await response.json());
|
|
117
|
+
}, shared_js_1.REQUEST_TIMEOUT_MS)));
|
|
128
118
|
}
|
|
129
119
|
catch (err) {
|
|
130
120
|
return {
|
|
@@ -135,11 +125,13 @@ class OpenAiCompletionProvider extends OpenAiGenericProvider {
|
|
|
135
125
|
try {
|
|
136
126
|
return {
|
|
137
127
|
output: data.choices[0].text,
|
|
138
|
-
tokenUsage:
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
128
|
+
tokenUsage: cached
|
|
129
|
+
? { cached: data.usage.total_tokens }
|
|
130
|
+
: {
|
|
131
|
+
total: data.usage.total_tokens,
|
|
132
|
+
prompt: data.usage.prompt_tokens,
|
|
133
|
+
completion: data.usage.completion_tokens,
|
|
134
|
+
},
|
|
143
135
|
};
|
|
144
136
|
}
|
|
145
137
|
catch (err) {
|
|
@@ -185,17 +177,16 @@ class OpenAiChatCompletionProvider extends OpenAiGenericProvider {
|
|
|
185
177
|
temperature: options?.temperature ?? (process.env.OPENAI_MAX_TEMPERATURE || 0),
|
|
186
178
|
};
|
|
187
179
|
logger_js_1.default.debug(`Calling OpenAI API: ${JSON.stringify(body)}`);
|
|
188
|
-
let
|
|
180
|
+
let data, cached = false;
|
|
189
181
|
try {
|
|
190
|
-
|
|
182
|
+
({ data, cached } = (await (0, cache_js_1.fetchJsonWithCache)(`https://${this.apiHost}/v1/chat/completions`, {
|
|
191
183
|
method: 'POST',
|
|
192
184
|
headers: {
|
|
193
185
|
'Content-Type': 'application/json',
|
|
194
186
|
Authorization: `Bearer ${this.apiKey}`,
|
|
195
187
|
},
|
|
196
188
|
body: JSON.stringify(body),
|
|
197
|
-
}, shared_js_1.REQUEST_TIMEOUT_MS);
|
|
198
|
-
data = (await response.json());
|
|
189
|
+
}, shared_js_1.REQUEST_TIMEOUT_MS)));
|
|
199
190
|
}
|
|
200
191
|
catch (err) {
|
|
201
192
|
return {
|
|
@@ -206,11 +197,13 @@ class OpenAiChatCompletionProvider extends OpenAiGenericProvider {
|
|
|
206
197
|
try {
|
|
207
198
|
return {
|
|
208
199
|
output: data.choices[0].message.content,
|
|
209
|
-
tokenUsage:
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
200
|
+
tokenUsage: cached
|
|
201
|
+
? { cached: data.usage.total_tokens }
|
|
202
|
+
: {
|
|
203
|
+
total: data.usage.total_tokens,
|
|
204
|
+
prompt: data.usage.prompt_tokens,
|
|
205
|
+
completion: data.usage.completion_tokens,
|
|
206
|
+
},
|
|
214
207
|
};
|
|
215
208
|
}
|
|
216
209
|
catch (err) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"openai.js","sourceRoot":"","sources":["../../src/providers/openai.ts"],"names":[],"mappings":";;;;;;AAAA,
|
|
1
|
+
{"version":3,"file":"openai.js","sourceRoot":"","sources":["../../src/providers/openai.ts"],"names":[],"mappings":";;;;;;AAAA,6DAAkC;AAClC,0CAAiD;AACjD,2CAAiD;AAIjD,MAAM,mBAAmB,GAAG,gBAAgB,CAAC;AAM7C,MAAM,qBAAqB;IAKzB,YAAY,SAAiB,EAAE,MAAe;QAC5C,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAE3B,IAAI,CAAC,MAAM,GAAG,MAAM,IAAI,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC;QAEnD,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,eAAe,IAAI,mBAAmB,CAAC;IACpE,CAAC;IAED,EAAE;QACA,OAAO,UAAU,IAAI,CAAC,SAAS,EAAE,CAAC;IACpC,CAAC;IAED,QAAQ;QACN,OAAO,oBAAoB,IAAI,CAAC,SAAS,GAAG,CAAC;IAC/C,CAAC;IAED,wDAAwD;IACxD,KAAK,CAAC,OAAO,CAAC,MAAc,EAAE,OAAiC;QAC7D,MAAM,IAAI,KAAK,CAAC,iBAAiB,CAAC,CAAC;IACrC,CAAC;CACF;AAED,MAAa,uBAAwB,SAAQ,qBAAqB;IAChE,KAAK,CAAC,gBAAgB,CAAC,IAAY;QACjC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE;YAChB,MAAM,IAAI,KAAK,CAAC,sDAAsD,CAAC,CAAC;SACzE;QAED,MAAM,IAAI,GAAG;YACX,KAAK,EAAE,IAAI;YACX,KAAK,EAAE,IAAI,CAAC,SAAS;SACtB,CAAC;QACF,IAAI,IAAI,EACN,MAAM,GAAG,KAAK,CAAC;QACjB,IAAI;YACF,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,IAAA,6BAAkB,EAC3C,WAAW,IAAI,CAAC,OAAO,gBAAgB,EACvC;gBACE,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE;oBACP,cAAc,EAAE,kBAAkB;oBAClC,aAAa,EAAE,UAAU,IAAI,CAAC,MAAM,EAAE;iBACvC;gBACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;aAC3B,EACD,8BAAkB,CACnB,CAAmB,CAAC,CAAC;SACvB;QAAC,OAAO,GAAG,EAAE;YACZ,OAAO;gBACL,KAAK,EAAE,mBAAmB,MAAM,CAAC,GAAG,CAAC,EAAE;gBACvC,UAAU,EAAE;oBACV,KAAK,EAAE,CAAC;oBACR,MAAM,EAAE,CAAC;oBACT,UAAU,EAAE,CAAC;iBACd;aACF,CAAC;SACH;QACD,mBAAM,CAAC,KAAK,CAAC,uCAAuC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAE5E,IAAI;YACF,MAAM,SAAS,GAAG,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,SAAS,CAAC;YAC7C,IAAI,CAAC,SAAS,EAAE;gBACd,MAAM,IAAI,KAAK,CAAC,uBAAuB,CAAC,CAAC;aAC1C;YACD,MAAM,GAAG,GAAG;gBACV,SAAS;gBACT,UAAU,EAAE,MAAM;oBAChB,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,YAAY,EAAE;oBACrC,CAAC,CAAC;wBACE,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,YAAY;wBAC9B,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,aAAa;wBAChC,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,iBAAiB;qBACzC;aACN,CAAC;YACF,OAAO,GAAG,CAAC;SACZ;QAAC,OAAO,GAAG,EAAE;YACZ,OAAO;gBACL,KAAK,EAAE,uBAAuB,MAAM,CAAC,GAAG,CAAC,KAAK,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE;gBACpE,UAAU,EAAE;oBACV,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,YAAY;oBAChC,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,aAAa;oBAClC,UAAU,EAAE,IAAI,EAAE,KAAK,EAAE,iBAAiB;iBAC3C;aACF,CAAC;SACH;IACH,CAAC;CACF;AAhED,0DAgEC;AAED,MAAa,wBAAyB,SAAQ,qBAAqB;IASjE,YAAY,SAAiB,EAAE,MAAe;QAC5C,IAAI,CAAC,wBAAwB,CAAC,wBAAwB,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE;YAC1E,mBAAM,CAAC,IAAI,CAAC,0CAA0C,SAAS,EAAE,CAAC,CAAC;SACpE;QACD,KAAK,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;IAC3B,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,MAAc,EAAE,OAAiC;QAC7D,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE;YAChB,MAAM,IAAI,KAAK,CACb,kHAAkH,CACnH,CAAC;SACH;QAED,MAAM,IAAI,GAAG;YACX,KAAK,EAAE,IAAI,CAAC,SAAS;YACrB,MAAM;YACN,UAAU,EAAE,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,IAAI;YACjD,WAAW,EAAE,OAAO,EAAE,WAAW,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,sBAAsB,IAAI,CAAC,CAAC;YAC9E,IAAI,EAAE,OAAO,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,SAAS;SAChF,CAAC;QACF,mBAAM,CAAC,KAAK,CAAC,uBAAuB,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC5D,IAAI,IAAI,EACN,MAAM,GAAG,KAAK,CAAC;QACjB,IAAI;YACF,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,IAAA,6BAAkB,EAC3C,WAAW,IAAI,CAAC,OAAO,iBAAiB,EACxC;gBACE,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE;oBACP,cAAc,EAAE,kBAAkB;oBAClC,aAAa,EAAE,UAAU,IAAI,CAAC,MAAM,EAAE;iBACvC;gBACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;aAC3B,EACD,8BAAkB,CACnB,CAAmB,CAAC,CAAC;SACvB;QAAC,OAAO,GAAG,EAAE;YACZ,OAAO;gBACL,KAAK,EAAE,mBAAmB,MAAM,CAAC,GAAG,CAAC,EAAE;aACxC,CAAC;SACH;QACD,mBAAM,CAAC,KAAK,CAAC,0BAA0B,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC/D,IAAI;YACF,OAAO;gBACL,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI;gBAC5B,UAAU,EAAE,MAAM;oBAChB,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,YAAY,EAAE;oBACrC,CAAC,CAAC;wBACE,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,YAAY;wBAC9B,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,aAAa;wBAChC,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,iBAAiB;qBACzC;aACN,CAAC;SACH;QAAC,OAAO,GAAG,EAAE;YACZ,OAAO;gBACL,KAAK,EAAE,uBAAuB,MAAM,CAAC,GAAG,CAAC,KAAK,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE;aACrE,CAAC;SACH;IACH,CAAC;;AAnEM,iDAAwB,GAAG;IAChC,kBAAkB;IAClB,kBAAkB;IAClB,gBAAgB;IAChB,kBAAkB;IAClB,cAAc;CACf,CAAC;AAPS,4DAAwB;AAuErC,MAAa,4BAA6B,SAAQ,qBAAqB;IAUrE,YAAY,SAAiB,EAAE,MAAe;QAC5C,IAAI,CAAC,4BAA4B,CAAC,kBAAkB,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE;YACxE,mBAAM,CAAC,IAAI,CAAC,oCAAoC,SAAS,EAAE,CAAC,CAAC;SAC9D;QACD,KAAK,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;IAC3B,CAAC;IAED,oDAAoD;IACpD,KAAK,CAAC,OAAO,CAAC,MAAc,EAAE,OAAiC;QAC7D,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE;YAChB,MAAM,IAAI,KAAK,CACb,kHAAkH,CACnH,CAAC;SACH;QAED,IAAI,QAA6C,CAAC;QAClD,IAAI;YACF,qEAAqE;YACrE,yCAAyC;YACzC,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;SAC/B;QAAC,OAAO,GAAG,EAAE;YACZ,QAAQ,GAAG,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;SAChD;QACD,MAAM,IAAI,GAAG;YACX,KAAK,EAAE,IAAI,CAAC,SAAS;YACrB,QAAQ,EAAE,QAAQ;YAClB,UAAU,EAAE,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,IAAI;YACjD,WAAW,EAAE,OAAO,EAAE,WAAW,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,sBAAsB,IAAI,CAAC,CAAC;SAC/E,CAAC;QACF,mBAAM,CAAC,KAAK,CAAC,uBAAuB,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAE5D,IAAI,IAAI,EACN,MAAM,GAAG,KAAK,CAAC;QACjB,IAAI;YACF,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,IAAA,6BAAkB,EAC3C,WAAW,IAAI,CAAC,OAAO,sBAAsB,EAC7C;gBACE,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE;oBACP,cAAc,EAAE,kBAAkB;oBAClC,aAAa,EAAE,UAAU,IAAI,CAAC,MAAM,EAAE;iBACvC;gBACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;aAC3B,EACD,8BAAkB,CACnB,CAAmB,CAAC,CAAC;SACvB;QAAC,OAAO,GAAG,EAAE;YACZ,OAAO;gBACL,KAAK,EAAE,mBAAmB,MAAM,CAAC,GAAG,CAAC,EAAE;aACxC,CAAC;SACH;QAED,mBAAM,CAAC,KAAK,CAAC,0BAA0B,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC/D,IAAI;YACF,OAAO;gBACL,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO;gBACvC,UAAU,EAAE,MAAM;oBAChB,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,YAAY,EAAE;oBACrC,CAAC,CAAC;wBACE,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,YAAY;wBAC9B,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,aAAa;wBAChC,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,iBAAiB;qBACzC;aACN,CAAC;SACH;QAAC,OAAO,GAAG,EAAE;YACZ,OAAO;gBACL,KAAK,EAAE,uBAAuB,MAAM,CAAC,GAAG,CAAC,KAAK,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE;aACrE,CAAC;SACH;IACH,CAAC;;AA9EM,+CAAkB,GAAG;IAC1B,OAAO;IACP,YAAY;IACZ,WAAW;IACX,gBAAgB;IAChB,eAAe;IACf,oBAAoB;CACrB,CAAC;AARS,oEAA4B;AAkF5B,QAAA,wBAAwB,GAAG,IAAI,uBAAuB,CAAC,wBAAwB,CAAC,CAAC;AACjF,QAAA,sBAAsB,GAAG,IAAI,4BAA4B,CAAC,OAAO,CAAC,CAAC;AACnE,QAAA,0BAA0B,GAAG,IAAI,4BAA4B,CAAC,OAAO,CAAC,CAAC"}
|
package/dist/types.d.ts
CHANGED
|
@@ -8,8 +8,9 @@ export interface CommandLineOptions {
|
|
|
8
8
|
maxConcurrency?: string;
|
|
9
9
|
grader?: string;
|
|
10
10
|
view?: string;
|
|
11
|
-
noWrite?: boolean;
|
|
12
11
|
tableCellMaxLength?: string;
|
|
12
|
+
write?: boolean;
|
|
13
|
+
cache?: boolean;
|
|
13
14
|
generateSuggestions?: boolean;
|
|
14
15
|
promptPrefix?: string;
|
|
15
16
|
promptSuffix?: string;
|
|
@@ -22,16 +23,17 @@ export interface TokenUsage {
|
|
|
22
23
|
total: number;
|
|
23
24
|
prompt: number;
|
|
24
25
|
completion: number;
|
|
26
|
+
cached?: number;
|
|
25
27
|
}
|
|
26
28
|
export interface ProviderResponse {
|
|
27
29
|
error?: string;
|
|
28
30
|
output?: string;
|
|
29
|
-
tokenUsage?: TokenUsage
|
|
31
|
+
tokenUsage?: Partial<TokenUsage>;
|
|
30
32
|
}
|
|
31
33
|
export interface ProviderEmbeddingResponse {
|
|
32
34
|
error?: string;
|
|
33
35
|
embedding?: number[];
|
|
34
|
-
tokenUsage?: TokenUsage
|
|
36
|
+
tokenUsage?: Partial<TokenUsage>;
|
|
35
37
|
}
|
|
36
38
|
export interface CsvRow {
|
|
37
39
|
[key: string]: string;
|
|
@@ -79,7 +81,7 @@ export interface EvaluateTable {
|
|
|
79
81
|
export interface EvaluateStats {
|
|
80
82
|
successes: number;
|
|
81
83
|
failures: number;
|
|
82
|
-
tokenUsage: TokenUsage
|
|
84
|
+
tokenUsage: Required<TokenUsage>;
|
|
83
85
|
}
|
|
84
86
|
export interface EvaluateSummary {
|
|
85
87
|
version: number;
|
package/dist/types.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,kBAAkB;IACjC,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,kBAAkB;IACjC,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,KAAK,CAAC,EAAE,OAAO,CAAC;IAEhB,mBAAmB,CAAC,EAAE,OAAO,CAAC;IAC9B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED,MAAM,WAAW,WAAW;IAC1B,EAAE,EAAE,MAAM,MAAM,CAAC;IACjB,OAAO,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,OAAO,CAAC,gBAAgB,CAAC,CAAC;CACxD;AAED,MAAM,WAAW,UAAU;IACzB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,gBAAgB;IAC/B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,UAAU,CAAC,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC;CAClC;AAED,MAAM,WAAW,yBAAyB;IACxC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB,UAAU,CAAC,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC;CAClC;AAED,MAAM,WAAW,MAAM;IACrB,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAC;CACvB;AAED,MAAM,MAAM,UAAU,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;AAEhD,MAAM,WAAW,aAAa;IAC5B,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,MAAM,GAAG,WAAW,CAAC;CACjC;AAED,MAAM,WAAW,YAAY;IAC3B,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,mBAAmB,CAAC,EAAE,OAAO,CAAC;CAC/B;AAED,MAAM,WAAW,eAAe;IAC9B,SAAS,EAAE,WAAW,EAAE,CAAC;IACzB,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,IAAI,CAAC,EAAE,UAAU,EAAE,CAAC;IAEpB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,eAAe,CAAC,EAAE,OAAO,CAAC;IAE1B,OAAO,CAAC,EAAE,aAAa,CAAC;IAExB,MAAM,CAAC,EAAE,YAAY,CAAC;CACvB;AAED,MAAM,WAAW,MAAM;IACrB,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC7B,QAAQ,CAAC,EAAE,gBAAgB,CAAC;IAC5B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,OAAO,CAAC;CAClB;AAED,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE;QACJ,OAAO,EAAE,MAAM,EAAE,CAAC;QAClB,IAAI,EAAE,MAAM,EAAE,CAAC;KAChB,CAAC;IAEF,IAAI,EAAE;QACJ,OAAO,EAAE,MAAM,EAAE,CAAC;QAClB,IAAI,EAAE,MAAM,EAAE,CAAC;KAChB,EAAE,CAAC;CACL;AAED,MAAM,WAAW,aAAa;IAC5B,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,QAAQ,CAAC,UAAU,CAAC,CAAC;CAClC;AAED,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,cAAc,EAAE,CAAC;IAC1B,KAAK,EAAE,aAAa,CAAC;IACrB,KAAK,EAAE,aAAa,CAAC;CACtB"}
|
package/dist/util.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"util.d.ts","sourceRoot":"","sources":["../src/util.ts"],"names":[],"mappings":"AASA,OAAO,EAAE,MAAM,EAAE,MAAM,YAAY,CAAC;AAOpC,OAAO,KAAK,EAAE,WAAW,EAAE,WAAW,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAErE,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAYlD,wBAAgB,WAAW,CAAC,kBAAkB,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE,CAsBlE;AAED,wBAAgB,QAAQ,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,EAAE,CAanD;AAED,wBAAgB,WAAW,CAAC,UAAU,EAAE,MAAM,EAAE,OAAO,EAAE,eAAe,GAAG,IAAI,CA2B9E;AAED,wBAAgB,gBAAgB,CAC9B,GAAG,EAAE,WAAW,EAChB,OAAO,yBAAkB,EACzB,OAAO,EAAE,MAAM,GACd,OAAO,CAAC,QAAQ,CAAC,CAwBnB;AAED,wBAAgB,sBAAsB,IAAI,MAAM,CAE/C;AAED,wBAAgB,oBAAoB,IAAI,MAAM,CAE7C;AAED,wBAAgB,kBAAkB,CAAC,OAAO,EAAE,eAAe,
|
|
1
|
+
{"version":3,"file":"util.d.ts","sourceRoot":"","sources":["../src/util.ts"],"names":[],"mappings":"AASA,OAAO,EAAE,MAAM,EAAE,MAAM,YAAY,CAAC;AAOpC,OAAO,KAAK,EAAE,WAAW,EAAE,WAAW,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAErE,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAYlD,wBAAgB,WAAW,CAAC,kBAAkB,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE,CAsBlE;AAED,wBAAgB,QAAQ,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,EAAE,CAanD;AAED,wBAAgB,WAAW,CAAC,UAAU,EAAE,MAAM,EAAE,OAAO,EAAE,eAAe,GAAG,IAAI,CA2B9E;AAED,wBAAgB,gBAAgB,CAC9B,GAAG,EAAE,WAAW,EAChB,OAAO,yBAAkB,EACzB,OAAO,EAAE,MAAM,GACd,OAAO,CAAC,QAAQ,CAAC,CAwBnB;AAED,wBAAgB,sBAAsB,IAAI,MAAM,CAE/C;AAED,wBAAgB,oBAAoB,IAAI,MAAM,CAE7C;AAED,wBAAgB,kBAAkB,CAAC,OAAO,EAAE,eAAe,QAQ1D;AAED,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,UAQ9D"}
|
package/dist/util.js
CHANGED
|
@@ -155,7 +155,6 @@ function writeLatestResults(results) {
|
|
|
155
155
|
try {
|
|
156
156
|
fs.mkdirSync(path.dirname(latestResultsPath), { recursive: true });
|
|
157
157
|
fs.writeFileSync(latestResultsPath, JSON.stringify(results, null, 2));
|
|
158
|
-
logger_js_1.default.info(`Wrote latest results to ${latestResultsPath}.`);
|
|
159
158
|
}
|
|
160
159
|
catch (err) {
|
|
161
160
|
logger_js_1.default.error(`Failed to write latest results to ${latestResultsPath}:\n${err}`);
|
package/dist/util.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"util.js","sourceRoot":"","sources":["../src/util.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,uCAAyB;AACzB,gDAAkC;AAClC,4CAA8B;AAE9B,4DAA+B;AAC/B,sDAA2B;AAC3B,wDAAgC;AAChC,+BAAgC;AAChC,+BAA0C;AAE1C,yCAAmD;AACnD,6CAA+C;AAE/C,4DAAiC;AACjC,qCAAwC;AAMxC,MAAM,gBAAgB,GAAG,KAAK,CAAC;AAE/B,SAAS,SAAS,CAAC,IAAY;IAC7B,IAAI;QACF,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;KACzB;IAAC,OAAO,GAAG,EAAE;QACZ,OAAO,SAAS,CAAC;KAClB;AACH,CAAC;AAED,SAAgB,WAAW,CAAC,kBAA4B;IACtD,MAAM,WAAW,GAAG,kBAAkB,CAAC,OAAO,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,IAAA,eAAQ,EAAC,UAAU,CAAC,CAAC,CAAC;IACrF,IAAI,cAAc,GAAa,EAAE,CAAC;IAElC,KAAK,MAAM,UAAU,IAAI,WAAW,EAAE;QACpC,MAAM,IAAI,GAAG,EAAE,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;QACrC,IAAI,IAAI,CAAC,WAAW,EAAE,EAAE;YACtB,MAAM,gBAAgB,GAAG,EAAE,CAAC,WAAW,CAAC,UAAU,CAAC,CAAC;YACpD,MAAM,YAAY,GAAG,gBAAgB,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CACrD,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,QAAQ,CAAC,EAAE,OAAO,CAAC,CAC1D,CAAC;YACF,cAAc,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,CAAC;SACtC;aAAM;YACL,MAAM,WAAW,GAAG,EAAE,CAAC,YAAY,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;YACzD,cAAc,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;SAClC;KACF;IAED,IAAI,cAAc,CAAC,MAAM,KAAK,CAAC,EAAE;QAC/B,cAAc,GAAG,cAAc,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;KACjF;IACD,OAAO,cAAc,CAAC;AACxB,CAAC;AAtBD,kCAsBC;AAED,SAAgB,QAAQ,CAAC,QAAgB;IACvC,MAAM,aAAa,GAAG,IAAA,YAAS,EAAC,QAAQ,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IACvD,IAAI,IAAI,GAAa,EAAE,CAAC;IAExB,IAAI,aAAa,KAAK,KAAK,EAAE;QAC3B,IAAI,GAAG,IAAA,YAAQ,EAAC,EAAE,CAAC,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;KACxE;SAAM,IAAI,aAAa,KAAK,MAAM,EAAE;QACnC,IAAI,GAAG,SAAS,CAAC,EAAE,CAAC,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAC;KACtD;SAAM,IAAI,aAAa,KAAK,MAAM,IAAI,aAAa,KAAK,KAAK,EAAE;QAC9D,IAAI,GAAG,iBAAI,CAAC,IAAI,CAAC,EAAE,CAAC,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAmB,CAAC;KACxE;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAbD,4BAaC;AAED,SAAgB,WAAW,CAAC,UAAkB,EAAE,OAAwB;IACtE,MAAM,eAAe,GAAG,UAAU,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,WAAW,EAAE,CAAC;IAEnE,IAAI,eAAe,KAAK,KAAK,IAAI,eAAe,KAAK,KAAK,EAAE;QAC1D,MAAM,SAAS,GAAG,IAAA,gBAAS,EAAC;YAC1B,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC;YAC3D,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,GAAG,GAAG,CAAC,OAAO,EAAE,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC;SAClE,CAAC,CAAC;QACH,EAAE,CAAC,aAAa,CAAC,UAAU,EAAE,SAAS,CAAC,CAAC;KACzC;SAAM,IAAI,eAAe,KAAK,MAAM,EAAE;QACrC,EAAE,CAAC,aAAa,CAAC,UAAU,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;KAChE;SAAM,IAAI,eAAe,KAAK,MAAM,IAAI,eAAe,KAAK,KAAK,EAAE;QAClE,EAAE,CAAC,aAAa,CAAC,UAAU,EAAE,iBAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;KAClD;SAAM,IAAI,eAAe,KAAK,MAAM,EAAE;QACrC,MAAM,QAAQ,GAAG,EAAE,CAAC,YAAY,CAAC,GAAG,IAAA,qBAAY,GAAE,mBAAmB,EAAE,OAAO,CAAC,CAAC;QAChF,MAAM,KAAK,GAAG;YACZ,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC;YAC3D,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,GAAG,GAAG,CAAC,OAAO,EAAE,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC;SAClE,CAAC;QACF,MAAM,UAAU,GAAG,kBAAQ,CAAC,YAAY,CAAC,QAAQ,EAAE;YACjD,KAAK;YACL,OAAO,EAAE,OAAO,CAAC,OAAO;SACzB,CAAC,CAAC;QACH,EAAE,CAAC,aAAa,CAAC,UAAU,EAAE,UAAU,CAAC,CAAC;KAC1C;SAAM;QACL,MAAM,IAAI,KAAK,CAAC,yDAAyD,CAAC,CAAC;KAC5E;AACH,CAAC;AA3BD,kCA2BC;AAED,SAAgB,gBAAgB,CAC9B,GAAgB,EAChB,UAAuB,EAAE,EACzB,OAAe;IAEf,OAAO,IAAI,OAAO,CAAC,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE;QAC3C,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;QACzC,MAAM,EAAE,MAAM,EAAE,GAAG,UAAU,CAAC;QAC9B,OAAO,CAAC,MAAM,GAAG,MAAM,CAAC;QAExB,MAAM,SAAS,GAAG,UAAU,CAAC,GAAG,EAAE;YAChC,UAAU,CAAC,KAAK,EAAE,CAAC;YACnB,MAAM,CAAC,IAAI,KAAK,CAAC,2BAA2B,OAAO,KAAK,CAAC,CAAC,CAAC;QAC7D,CAAC,EAAE,OAAO,CAAC,CAAC;QAEZ,IAAI;YACF,MAAM,QAAQ,GAAG,MAAM,IAAA,oBAAK,EAAC,GAAG,EAAE,OAAO,CAAC,CAAC;YAC3C,YAAY,CAAC,SAAS,CAAC,CAAC;YACxB,OAAO,CAAC,QAAQ,CAAC,CAAC;SACnB;QAAC,OAAO,KAAK,EAAE;YACd,IAAI,KAAK,YAAY,KAAK,IAAI,KAAK,CAAC,IAAI,KAAK,YAAY,EAAE;gBACzD,qDAAqD;aACtD;iBAAM;gBACL,YAAY,CAAC,SAAS,CAAC,CAAC;gBACxB,MAAM,CAAC,KAAK,CAAC,CAAC;aACf;SACF;IACH,CAAC,CAAC,CAAC;AACL,CAAC;AA5BD,4CA4BC;AAED,SAAgB,sBAAsB;IACpC,OAAO,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,EAAE,YAAY,CAAC,CAAC;AAC/C,CAAC;AAFD,wDAEC;AAED,SAAgB,oBAAoB;IAClC,OAAO,IAAI,CAAC,IAAI,CAAC,sBAAsB,EAAE,EAAE,QAAQ,EAAE,aAAa,CAAC,CAAC;AACtE,CAAC;AAFD,oDAEC;AAED,SAAgB,kBAAkB,CAAC,OAAwB;IACzD,MAAM,iBAAiB,GAAG,oBAAoB,EAAE,CAAC;IACjD,IAAI;QACF,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,iBAAiB,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACnE,EAAE,CAAC,aAAa,CAAC,iBAAiB,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;
|
|
1
|
+
{"version":3,"file":"util.js","sourceRoot":"","sources":["../src/util.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,uCAAyB;AACzB,gDAAkC;AAClC,4CAA8B;AAE9B,4DAA+B;AAC/B,sDAA2B;AAC3B,wDAAgC;AAChC,+BAAgC;AAChC,+BAA0C;AAE1C,yCAAmD;AACnD,6CAA+C;AAE/C,4DAAiC;AACjC,qCAAwC;AAMxC,MAAM,gBAAgB,GAAG,KAAK,CAAC;AAE/B,SAAS,SAAS,CAAC,IAAY;IAC7B,IAAI;QACF,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;KACzB;IAAC,OAAO,GAAG,EAAE;QACZ,OAAO,SAAS,CAAC;KAClB;AACH,CAAC;AAED,SAAgB,WAAW,CAAC,kBAA4B;IACtD,MAAM,WAAW,GAAG,kBAAkB,CAAC,OAAO,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,IAAA,eAAQ,EAAC,UAAU,CAAC,CAAC,CAAC;IACrF,IAAI,cAAc,GAAa,EAAE,CAAC;IAElC,KAAK,MAAM,UAAU,IAAI,WAAW,EAAE;QACpC,MAAM,IAAI,GAAG,EAAE,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;QACrC,IAAI,IAAI,CAAC,WAAW,EAAE,EAAE;YACtB,MAAM,gBAAgB,GAAG,EAAE,CAAC,WAAW,CAAC,UAAU,CAAC,CAAC;YACpD,MAAM,YAAY,GAAG,gBAAgB,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CACrD,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,QAAQ,CAAC,EAAE,OAAO,CAAC,CAC1D,CAAC;YACF,cAAc,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,CAAC;SACtC;aAAM;YACL,MAAM,WAAW,GAAG,EAAE,CAAC,YAAY,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;YACzD,cAAc,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;SAClC;KACF;IAED,IAAI,cAAc,CAAC,MAAM,KAAK,CAAC,EAAE;QAC/B,cAAc,GAAG,cAAc,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;KACjF;IACD,OAAO,cAAc,CAAC;AACxB,CAAC;AAtBD,kCAsBC;AAED,SAAgB,QAAQ,CAAC,QAAgB;IACvC,MAAM,aAAa,GAAG,IAAA,YAAS,EAAC,QAAQ,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IACvD,IAAI,IAAI,GAAa,EAAE,CAAC;IAExB,IAAI,aAAa,KAAK,KAAK,EAAE;QAC3B,IAAI,GAAG,IAAA,YAAQ,EAAC,EAAE,CAAC,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;KACxE;SAAM,IAAI,aAAa,KAAK,MAAM,EAAE;QACnC,IAAI,GAAG,SAAS,CAAC,EAAE,CAAC,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAC;KACtD;SAAM,IAAI,aAAa,KAAK,MAAM,IAAI,aAAa,KAAK,KAAK,EAAE;QAC9D,IAAI,GAAG,iBAAI,CAAC,IAAI,CAAC,EAAE,CAAC,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAmB,CAAC;KACxE;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAbD,4BAaC;AAED,SAAgB,WAAW,CAAC,UAAkB,EAAE,OAAwB;IACtE,MAAM,eAAe,GAAG,UAAU,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,WAAW,EAAE,CAAC;IAEnE,IAAI,eAAe,KAAK,KAAK,IAAI,eAAe,KAAK,KAAK,EAAE;QAC1D,MAAM,SAAS,GAAG,IAAA,gBAAS,EAAC;YAC1B,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC;YAC3D,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,GAAG,GAAG,CAAC,OAAO,EAAE,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC;SAClE,CAAC,CAAC;QACH,EAAE,CAAC,aAAa,CAAC,UAAU,EAAE,SAAS,CAAC,CAAC;KACzC;SAAM,IAAI,eAAe,KAAK,MAAM,EAAE;QACrC,EAAE,CAAC,aAAa,CAAC,UAAU,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;KAChE;SAAM,IAAI,eAAe,KAAK,MAAM,IAAI,eAAe,KAAK,KAAK,EAAE;QAClE,EAAE,CAAC,aAAa,CAAC,UAAU,EAAE,iBAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;KAClD;SAAM,IAAI,eAAe,KAAK,MAAM,EAAE;QACrC,MAAM,QAAQ,GAAG,EAAE,CAAC,YAAY,CAAC,GAAG,IAAA,qBAAY,GAAE,mBAAmB,EAAE,OAAO,CAAC,CAAC;QAChF,MAAM,KAAK,GAAG;YACZ,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC;YAC3D,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,GAAG,GAAG,CAAC,OAAO,EAAE,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC;SAClE,CAAC;QACF,MAAM,UAAU,GAAG,kBAAQ,CAAC,YAAY,CAAC,QAAQ,EAAE;YACjD,KAAK;YACL,OAAO,EAAE,OAAO,CAAC,OAAO;SACzB,CAAC,CAAC;QACH,EAAE,CAAC,aAAa,CAAC,UAAU,EAAE,UAAU,CAAC,CAAC;KAC1C;SAAM;QACL,MAAM,IAAI,KAAK,CAAC,yDAAyD,CAAC,CAAC;KAC5E;AACH,CAAC;AA3BD,kCA2BC;AAED,SAAgB,gBAAgB,CAC9B,GAAgB,EAChB,UAAuB,EAAE,EACzB,OAAe;IAEf,OAAO,IAAI,OAAO,CAAC,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE;QAC3C,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;QACzC,MAAM,EAAE,MAAM,EAAE,GAAG,UAAU,CAAC;QAC9B,OAAO,CAAC,MAAM,GAAG,MAAM,CAAC;QAExB,MAAM,SAAS,GAAG,UAAU,CAAC,GAAG,EAAE;YAChC,UAAU,CAAC,KAAK,EAAE,CAAC;YACnB,MAAM,CAAC,IAAI,KAAK,CAAC,2BAA2B,OAAO,KAAK,CAAC,CAAC,CAAC;QAC7D,CAAC,EAAE,OAAO,CAAC,CAAC;QAEZ,IAAI;YACF,MAAM,QAAQ,GAAG,MAAM,IAAA,oBAAK,EAAC,GAAG,EAAE,OAAO,CAAC,CAAC;YAC3C,YAAY,CAAC,SAAS,CAAC,CAAC;YACxB,OAAO,CAAC,QAAQ,CAAC,CAAC;SACnB;QAAC,OAAO,KAAK,EAAE;YACd,IAAI,KAAK,YAAY,KAAK,IAAI,KAAK,CAAC,IAAI,KAAK,YAAY,EAAE;gBACzD,qDAAqD;aACtD;iBAAM;gBACL,YAAY,CAAC,SAAS,CAAC,CAAC;gBACxB,MAAM,CAAC,KAAK,CAAC,CAAC;aACf;SACF;IACH,CAAC,CAAC,CAAC;AACL,CAAC;AA5BD,4CA4BC;AAED,SAAgB,sBAAsB;IACpC,OAAO,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,EAAE,YAAY,CAAC,CAAC;AAC/C,CAAC;AAFD,wDAEC;AAED,SAAgB,oBAAoB;IAClC,OAAO,IAAI,CAAC,IAAI,CAAC,sBAAsB,EAAE,EAAE,QAAQ,EAAE,aAAa,CAAC,CAAC;AACtE,CAAC;AAFD,oDAEC;AAED,SAAgB,kBAAkB,CAAC,OAAwB;IACzD,MAAM,iBAAiB,GAAG,oBAAoB,EAAE,CAAC;IACjD,IAAI;QACF,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,iBAAiB,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACnE,EAAE,CAAC,aAAa,CAAC,iBAAiB,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;KACvE;IAAC,OAAO,GAAG,EAAE;QACZ,mBAAM,CAAC,KAAK,CAAC,qCAAqC,iBAAiB,MAAM,GAAG,EAAE,CAAC,CAAC;KACjF;AACH,CAAC;AARD,gDAQC;AAED,SAAgB,gBAAgB,CAAC,IAAc,EAAE,IAAc;IAC7D,IAAI,IAAI,CAAC,MAAM,KAAK,IAAI,CAAC,MAAM,EAAE;QAC/B,MAAM,IAAI,KAAK,CAAC,iCAAiC,CAAC,CAAC;KACpD;IACD,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC,GAAG,GAAG,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;IAC5E,MAAM,aAAa,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC,GAAG,GAAG,GAAG,GAAG,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC;IAC/E,MAAM,aAAa,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC,GAAG,GAAG,GAAG,GAAG,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC;IAC/E,OAAO,UAAU,GAAG,CAAC,aAAa,GAAG,aAAa,CAAC,CAAC;AACtD,CAAC;AARD,4CAQC"}
|
package/package.json
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "promptfoo",
|
|
3
|
+
"description": "Prompt engineering toolkit",
|
|
3
4
|
"author": "Ian Webster",
|
|
4
|
-
"version": "0.
|
|
5
|
+
"version": "0.7.0",
|
|
5
6
|
"license": "MIT",
|
|
6
7
|
"type": "commonjs",
|
|
7
8
|
"main": "dist/index.js",
|
|
@@ -37,6 +38,7 @@
|
|
|
37
38
|
},
|
|
38
39
|
"devDependencies": {
|
|
39
40
|
"@types/async": "^3.2.20",
|
|
41
|
+
"@types/cache-manager-fs-hash": "^0.0.1",
|
|
40
42
|
"@types/cli-progress": "^3.11.0",
|
|
41
43
|
"@types/cors": "^2.8.13",
|
|
42
44
|
"@types/debounce": "^1.2.1",
|
|
@@ -56,6 +58,8 @@
|
|
|
56
58
|
},
|
|
57
59
|
"dependencies": {
|
|
58
60
|
"async": "^3.2.4",
|
|
61
|
+
"cache-manager": "^4.1.0",
|
|
62
|
+
"cache-manager-fs-hash": "^1.0.0",
|
|
59
63
|
"chalk": "^4.1.2",
|
|
60
64
|
"cli-progress": "^3.12.0",
|
|
61
65
|
"cli-table3": "^0.6.3",
|
|
@@ -67,7 +71,6 @@
|
|
|
67
71
|
"express": "^4.18.2",
|
|
68
72
|
"glob": "^10.2.6",
|
|
69
73
|
"js-yaml": "^4.1.0",
|
|
70
|
-
"lru-cache": "^9.1.1",
|
|
71
74
|
"node-fetch": "^2.6.7",
|
|
72
75
|
"nunjucks": "^3.2.4",
|
|
73
76
|
"opener": "^1.5.2",
|
package/src/cache.ts
ADDED
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
import path from 'node:path';
|
|
2
|
+
|
|
3
|
+
import cacheManager from 'cache-manager';
|
|
4
|
+
import fsStore from 'cache-manager-fs-hash';
|
|
5
|
+
import fetch, { Response } from 'node-fetch';
|
|
6
|
+
|
|
7
|
+
import logger from './logger.js';
|
|
8
|
+
import { getConfigDirectoryPath, fetchWithTimeout } from './util.js';
|
|
9
|
+
|
|
10
|
+
import type { Cache } from 'cache-manager';
|
|
11
|
+
import type { RequestInfo, RequestInit } from 'node-fetch';
|
|
12
|
+
|
|
13
|
+
let cacheInstance: Cache | undefined;
|
|
14
|
+
|
|
15
|
+
let enabled =
|
|
16
|
+
typeof process.env.PROMPTFOO_CACHE_ENABLED === 'undefined'
|
|
17
|
+
? true
|
|
18
|
+
: Boolean(process.env.PROMPTFOO_CACHE_ENABLED);
|
|
19
|
+
|
|
20
|
+
const cacheType =
|
|
21
|
+
process.env.PROMPTFOO_CACHE_TYPE || (process.env.NODE_ENV === 'test' ? 'memory' : 'disk');
|
|
22
|
+
|
|
23
|
+
function getCache() {
|
|
24
|
+
if (!cacheInstance) {
|
|
25
|
+
cacheInstance = cacheManager.caching({
|
|
26
|
+
store: cacheType === 'disk' ? fsStore : 'memory',
|
|
27
|
+
options: {
|
|
28
|
+
max: process.env.PROMPTFOO_CACHE_MAX_FILE_COUNT || 10_000, // number of files
|
|
29
|
+
path: process.env.PROMPTFOO_CACHE_PATH || path.join(getConfigDirectoryPath(), 'cache'),
|
|
30
|
+
ttl: process.env.PROMPTFOO_CACHE_TTL || 60 * 60 * 24 * 14, // in seconds, 14 days
|
|
31
|
+
maxsize: process.env.PROMPTFOO_CACHE_MAX_SIZE || 1e7, // in bytes, 10mb
|
|
32
|
+
//zip: true, // whether to use gzip compression
|
|
33
|
+
},
|
|
34
|
+
});
|
|
35
|
+
}
|
|
36
|
+
return cacheInstance;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
export async function fetchJsonWithCache(
|
|
40
|
+
url: RequestInfo,
|
|
41
|
+
options: RequestInit = {},
|
|
42
|
+
timeout: number,
|
|
43
|
+
): Promise<{ data: any; cached: boolean }> {
|
|
44
|
+
if (!enabled) {
|
|
45
|
+
const resp = await fetchWithTimeout(url, options, timeout);
|
|
46
|
+
return {
|
|
47
|
+
cached: false,
|
|
48
|
+
data: await resp.json(),
|
|
49
|
+
};
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
const cache = await getCache();
|
|
53
|
+
|
|
54
|
+
const copy = Object.assign({}, options);
|
|
55
|
+
delete copy.headers;
|
|
56
|
+
const cacheKey = `fetch:${url}:${JSON.stringify(copy)}`;
|
|
57
|
+
|
|
58
|
+
// Try to get the cached response
|
|
59
|
+
const cachedResponse = await cache.get(cacheKey);
|
|
60
|
+
|
|
61
|
+
if (cachedResponse) {
|
|
62
|
+
logger.debug(`Returning cached response for ${url}: ${cachedResponse}`);
|
|
63
|
+
return {
|
|
64
|
+
cached: true,
|
|
65
|
+
data: JSON.parse(cachedResponse as string),
|
|
66
|
+
};
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
// Fetch the actual data and store it in the cache
|
|
70
|
+
const response = await fetchWithTimeout(url, options, timeout);
|
|
71
|
+
try {
|
|
72
|
+
const data = await response.json();
|
|
73
|
+
logger.debug(`Storing ${url} response in cache: ${data}`);
|
|
74
|
+
await cache.set(cacheKey, JSON.stringify(data));
|
|
75
|
+
return {
|
|
76
|
+
cached: false,
|
|
77
|
+
data,
|
|
78
|
+
};
|
|
79
|
+
} catch (err) {
|
|
80
|
+
throw new Error(`Error parsing response from ${url}: ${err}`);
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
export function enableCache() {
|
|
85
|
+
enabled = true;
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
export function disableCache() {
|
|
89
|
+
logger.info('Cache is disabled.');
|
|
90
|
+
enabled = false;
|
|
91
|
+
}
|
package/src/evaluator.ts
CHANGED
|
@@ -44,6 +44,7 @@ class Evaluator {
|
|
|
44
44
|
total: 0,
|
|
45
45
|
prompt: 0,
|
|
46
46
|
completion: 0,
|
|
47
|
+
cached: 0,
|
|
47
48
|
},
|
|
48
49
|
};
|
|
49
50
|
}
|
|
@@ -94,6 +95,7 @@ class Evaluator {
|
|
|
94
95
|
this.stats.tokenUsage.total += response.tokenUsage?.total || 0;
|
|
95
96
|
this.stats.tokenUsage.prompt += response.tokenUsage?.prompt || 0;
|
|
96
97
|
this.stats.tokenUsage.completion += response.tokenUsage?.completion || 0;
|
|
98
|
+
this.stats.tokenUsage.cached += response.tokenUsage?.cached || 0;
|
|
97
99
|
|
|
98
100
|
if (ret.success) {
|
|
99
101
|
this.stats.successes++;
|
package/src/main.ts
CHANGED
|
@@ -14,6 +14,7 @@ import { getDirectory } from './esm.js';
|
|
|
14
14
|
import { init } from './web/server.js';
|
|
15
15
|
|
|
16
16
|
import type { CommandLineOptions, EvaluateOptions, VarMapping } from './types.js';
|
|
17
|
+
import { disableCache } from './cache.js';
|
|
17
18
|
|
|
18
19
|
function createDummyFiles(directory: string | null) {
|
|
19
20
|
if (directory) {
|
|
@@ -162,14 +163,11 @@ async function main() {
|
|
|
162
163
|
defaultConfig.promptSuffix,
|
|
163
164
|
)
|
|
164
165
|
.option('--no-write', 'Do not write results to promptfoo directory')
|
|
166
|
+
.option('--no-cache', 'Do not read or write results to disk cache')
|
|
165
167
|
.option('--grader', 'Model that will grade outputs', defaultConfig.grader)
|
|
166
168
|
.option('--verbose', 'Show debug logs', defaultConfig.verbose)
|
|
167
169
|
.option('--view [port]', 'View in browser ui')
|
|
168
170
|
.action(async (cmdObj: CommandLineOptions & Command) => {
|
|
169
|
-
if (cmdObj.verbose) {
|
|
170
|
-
setLogLevel('debug');
|
|
171
|
-
}
|
|
172
|
-
|
|
173
171
|
const configPath = cmdObj.config;
|
|
174
172
|
let config = {};
|
|
175
173
|
if (configPath) {
|
|
@@ -187,6 +185,13 @@ async function main() {
|
|
|
187
185
|
}
|
|
188
186
|
}
|
|
189
187
|
|
|
188
|
+
if (cmdObj.verbose) {
|
|
189
|
+
setLogLevel('debug');
|
|
190
|
+
}
|
|
191
|
+
if (!cmdObj.cache) {
|
|
192
|
+
disableCache();
|
|
193
|
+
}
|
|
194
|
+
|
|
190
195
|
let vars: VarMapping[] = [];
|
|
191
196
|
if (cmdObj.vars) {
|
|
192
197
|
vars = readVars(cmdObj.vars);
|
|
@@ -263,7 +268,7 @@ async function main() {
|
|
|
263
268
|
|
|
264
269
|
logger.info('\n' + table.toString());
|
|
265
270
|
}
|
|
266
|
-
if (cmdObj.
|
|
271
|
+
if (cmdObj.view || !cmdObj.write) {
|
|
267
272
|
logger.info('Evaluation complete');
|
|
268
273
|
} else {
|
|
269
274
|
writeLatestResults(summary);
|
|
@@ -272,7 +277,7 @@ async function main() {
|
|
|
272
277
|
logger.info(chalk.green.bold(`Successes: ${summary.stats.successes}`));
|
|
273
278
|
logger.info(chalk.red.bold(`Failures: ${summary.stats.failures}`));
|
|
274
279
|
logger.info(
|
|
275
|
-
`Token usage: Total ${summary.stats.tokenUsage.total} Prompt ${summary.stats.tokenUsage.prompt} Completion ${summary.stats.tokenUsage.completion}`,
|
|
280
|
+
`Token usage: Total ${summary.stats.tokenUsage.total}, Prompt ${summary.stats.tokenUsage.prompt}, Completion ${summary.stats.tokenUsage.completion}, Cached ${summary.stats.tokenUsage.cached}`,
|
|
276
281
|
);
|
|
277
282
|
logger.info('Done.');
|
|
278
283
|
|
package/src/providers/localai.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import logger from '../logger.js';
|
|
2
|
-
import {
|
|
2
|
+
import { fetchJsonWithCache } from '../cache.js';
|
|
3
3
|
import { REQUEST_TIMEOUT_MS } from './shared.js';
|
|
4
4
|
|
|
5
5
|
import type { ApiProvider, ProviderResponse } from '../types.js';
|
|
@@ -36,9 +36,10 @@ export class LocalAiChatProvider extends LocalAiGenericProvider {
|
|
|
36
36
|
};
|
|
37
37
|
logger.debug(`Calling LocalAI API: ${JSON.stringify(body)}`);
|
|
38
38
|
|
|
39
|
-
let
|
|
39
|
+
let data,
|
|
40
|
+
cached = false;
|
|
40
41
|
try {
|
|
41
|
-
|
|
42
|
+
({ data, cached } = (await fetchJsonWithCache(
|
|
42
43
|
`${this.apiBaseUrl}/chat/completions`,
|
|
43
44
|
{
|
|
44
45
|
method: 'POST',
|
|
@@ -48,9 +49,7 @@ export class LocalAiChatProvider extends LocalAiGenericProvider {
|
|
|
48
49
|
body: JSON.stringify(body),
|
|
49
50
|
},
|
|
50
51
|
REQUEST_TIMEOUT_MS,
|
|
51
|
-
);
|
|
52
|
-
|
|
53
|
-
data = (await response.json()) as unknown as any;
|
|
52
|
+
)) as unknown as any);
|
|
54
53
|
} catch (err) {
|
|
55
54
|
return {
|
|
56
55
|
error: `API call error: ${String(err)}`,
|
|
@@ -78,9 +77,10 @@ export class LocalAiCompletionProvider extends LocalAiGenericProvider {
|
|
|
78
77
|
};
|
|
79
78
|
logger.debug(`Calling LocalAI API: ${JSON.stringify(body)}`);
|
|
80
79
|
|
|
81
|
-
let
|
|
80
|
+
let data,
|
|
81
|
+
cached = false;
|
|
82
82
|
try {
|
|
83
|
-
|
|
83
|
+
({ data, cached } = (await fetchJsonWithCache(
|
|
84
84
|
`${this.apiBaseUrl}/completions`,
|
|
85
85
|
{
|
|
86
86
|
method: 'POST',
|
|
@@ -90,9 +90,7 @@ export class LocalAiCompletionProvider extends LocalAiGenericProvider {
|
|
|
90
90
|
body: JSON.stringify(body),
|
|
91
91
|
},
|
|
92
92
|
REQUEST_TIMEOUT_MS,
|
|
93
|
-
);
|
|
94
|
-
|
|
95
|
-
data = (await response.json()) as unknown as any;
|
|
93
|
+
)) as unknown as any);
|
|
96
94
|
} catch (err) {
|
|
97
95
|
return {
|
|
98
96
|
error: `API call error: ${String(err)}`,
|
package/src/providers/openai.ts
CHANGED
|
@@ -1,17 +1,11 @@
|
|
|
1
|
-
import { LRUCache } from 'lru-cache';
|
|
2
|
-
|
|
3
1
|
import logger from '../logger.js';
|
|
4
|
-
import {
|
|
2
|
+
import { fetchJsonWithCache } from '../cache.js';
|
|
5
3
|
import { REQUEST_TIMEOUT_MS } from './shared.js';
|
|
6
4
|
|
|
7
5
|
import type { ApiProvider, ProviderEmbeddingResponse, ProviderResponse } from '../types.js';
|
|
8
6
|
|
|
9
7
|
const DEFAULT_OPENAI_HOST = 'api.openai.com';
|
|
10
8
|
|
|
11
|
-
const embeddingsCache = new LRUCache<string, ProviderEmbeddingResponse>({
|
|
12
|
-
max: 1000,
|
|
13
|
-
});
|
|
14
|
-
|
|
15
9
|
interface OpenAiCompletionOptions {
|
|
16
10
|
temperature: number;
|
|
17
11
|
}
|
|
@@ -49,19 +43,14 @@ export class OpenAiEmbeddingProvider extends OpenAiGenericProvider {
|
|
|
49
43
|
throw new Error('OpenAI API key must be set for similarity comparison');
|
|
50
44
|
}
|
|
51
45
|
|
|
52
|
-
// TODO(ian): Improve cache
|
|
53
|
-
const cached = embeddingsCache.get(text);
|
|
54
|
-
if (cached) {
|
|
55
|
-
return cached;
|
|
56
|
-
}
|
|
57
|
-
|
|
58
46
|
const body = {
|
|
59
47
|
input: text,
|
|
60
48
|
model: this.modelName,
|
|
61
49
|
};
|
|
62
|
-
let
|
|
50
|
+
let data,
|
|
51
|
+
cached = false;
|
|
63
52
|
try {
|
|
64
|
-
|
|
53
|
+
({ data, cached } = (await fetchJsonWithCache(
|
|
65
54
|
`https://${this.apiHost}/v1/embeddings`,
|
|
66
55
|
{
|
|
67
56
|
method: 'POST',
|
|
@@ -72,8 +61,7 @@ export class OpenAiEmbeddingProvider extends OpenAiGenericProvider {
|
|
|
72
61
|
body: JSON.stringify(body),
|
|
73
62
|
},
|
|
74
63
|
REQUEST_TIMEOUT_MS,
|
|
75
|
-
);
|
|
76
|
-
data = (await response.json()) as unknown as any;
|
|
64
|
+
)) as unknown as any);
|
|
77
65
|
} catch (err) {
|
|
78
66
|
return {
|
|
79
67
|
error: `API call error: ${String(err)}`,
|
|
@@ -93,13 +81,14 @@ export class OpenAiEmbeddingProvider extends OpenAiGenericProvider {
|
|
|
93
81
|
}
|
|
94
82
|
const ret = {
|
|
95
83
|
embedding,
|
|
96
|
-
tokenUsage:
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
84
|
+
tokenUsage: cached
|
|
85
|
+
? { cached: data.usage.total_tokens }
|
|
86
|
+
: {
|
|
87
|
+
total: data.usage.total_tokens,
|
|
88
|
+
prompt: data.usage.prompt_tokens,
|
|
89
|
+
completion: data.usage.completion_tokens,
|
|
90
|
+
},
|
|
101
91
|
};
|
|
102
|
-
embeddingsCache.set(text, ret);
|
|
103
92
|
return ret;
|
|
104
93
|
} catch (err) {
|
|
105
94
|
return {
|
|
@@ -145,9 +134,10 @@ export class OpenAiCompletionProvider extends OpenAiGenericProvider {
|
|
|
145
134
|
stop: process.env.OPENAI_STOP ? JSON.parse(process.env.OPENAI_STOP) : undefined,
|
|
146
135
|
};
|
|
147
136
|
logger.debug(`Calling OpenAI API: ${JSON.stringify(body)}`);
|
|
148
|
-
let
|
|
137
|
+
let data,
|
|
138
|
+
cached = false;
|
|
149
139
|
try {
|
|
150
|
-
|
|
140
|
+
({ data, cached } = (await fetchJsonWithCache(
|
|
151
141
|
`https://${this.apiHost}/v1/completions`,
|
|
152
142
|
{
|
|
153
143
|
method: 'POST',
|
|
@@ -158,9 +148,7 @@ export class OpenAiCompletionProvider extends OpenAiGenericProvider {
|
|
|
158
148
|
body: JSON.stringify(body),
|
|
159
149
|
},
|
|
160
150
|
REQUEST_TIMEOUT_MS,
|
|
161
|
-
);
|
|
162
|
-
|
|
163
|
-
data = (await response.json()) as unknown as any;
|
|
151
|
+
)) as unknown as any);
|
|
164
152
|
} catch (err) {
|
|
165
153
|
return {
|
|
166
154
|
error: `API call error: ${String(err)}`,
|
|
@@ -170,11 +158,13 @@ export class OpenAiCompletionProvider extends OpenAiGenericProvider {
|
|
|
170
158
|
try {
|
|
171
159
|
return {
|
|
172
160
|
output: data.choices[0].text,
|
|
173
|
-
tokenUsage:
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
161
|
+
tokenUsage: cached
|
|
162
|
+
? { cached: data.usage.total_tokens }
|
|
163
|
+
: {
|
|
164
|
+
total: data.usage.total_tokens,
|
|
165
|
+
prompt: data.usage.prompt_tokens,
|
|
166
|
+
completion: data.usage.completion_tokens,
|
|
167
|
+
},
|
|
178
168
|
};
|
|
179
169
|
} catch (err) {
|
|
180
170
|
return {
|
|
@@ -225,9 +215,10 @@ export class OpenAiChatCompletionProvider extends OpenAiGenericProvider {
|
|
|
225
215
|
};
|
|
226
216
|
logger.debug(`Calling OpenAI API: ${JSON.stringify(body)}`);
|
|
227
217
|
|
|
228
|
-
let
|
|
218
|
+
let data,
|
|
219
|
+
cached = false;
|
|
229
220
|
try {
|
|
230
|
-
|
|
221
|
+
({ data, cached } = (await fetchJsonWithCache(
|
|
231
222
|
`https://${this.apiHost}/v1/chat/completions`,
|
|
232
223
|
{
|
|
233
224
|
method: 'POST',
|
|
@@ -238,8 +229,7 @@ export class OpenAiChatCompletionProvider extends OpenAiGenericProvider {
|
|
|
238
229
|
body: JSON.stringify(body),
|
|
239
230
|
},
|
|
240
231
|
REQUEST_TIMEOUT_MS,
|
|
241
|
-
);
|
|
242
|
-
data = (await response.json()) as unknown as any;
|
|
232
|
+
)) as unknown as any);
|
|
243
233
|
} catch (err) {
|
|
244
234
|
return {
|
|
245
235
|
error: `API call error: ${String(err)}`,
|
|
@@ -250,11 +240,13 @@ export class OpenAiChatCompletionProvider extends OpenAiGenericProvider {
|
|
|
250
240
|
try {
|
|
251
241
|
return {
|
|
252
242
|
output: data.choices[0].message.content,
|
|
253
|
-
tokenUsage:
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
243
|
+
tokenUsage: cached
|
|
244
|
+
? { cached: data.usage.total_tokens }
|
|
245
|
+
: {
|
|
246
|
+
total: data.usage.total_tokens,
|
|
247
|
+
prompt: data.usage.prompt_tokens,
|
|
248
|
+
completion: data.usage.completion_tokens,
|
|
249
|
+
},
|
|
258
250
|
};
|
|
259
251
|
} catch (err) {
|
|
260
252
|
return {
|
package/src/types.ts
CHANGED
|
@@ -8,8 +8,9 @@ export interface CommandLineOptions {
|
|
|
8
8
|
maxConcurrency?: string;
|
|
9
9
|
grader?: string;
|
|
10
10
|
view?: string;
|
|
11
|
-
noWrite?: boolean;
|
|
12
11
|
tableCellMaxLength?: string;
|
|
12
|
+
write?: boolean;
|
|
13
|
+
cache?: boolean;
|
|
13
14
|
|
|
14
15
|
generateSuggestions?: boolean;
|
|
15
16
|
promptPrefix?: string;
|
|
@@ -25,18 +26,19 @@ export interface TokenUsage {
|
|
|
25
26
|
total: number;
|
|
26
27
|
prompt: number;
|
|
27
28
|
completion: number;
|
|
29
|
+
cached?: number;
|
|
28
30
|
}
|
|
29
31
|
|
|
30
32
|
export interface ProviderResponse {
|
|
31
33
|
error?: string;
|
|
32
34
|
output?: string;
|
|
33
|
-
tokenUsage?: TokenUsage
|
|
35
|
+
tokenUsage?: Partial<TokenUsage>;
|
|
34
36
|
}
|
|
35
37
|
|
|
36
38
|
export interface ProviderEmbeddingResponse {
|
|
37
39
|
error?: string;
|
|
38
40
|
embedding?: number[];
|
|
39
|
-
tokenUsage?: TokenUsage
|
|
41
|
+
tokenUsage?: Partial<TokenUsage>;
|
|
40
42
|
}
|
|
41
43
|
|
|
42
44
|
export interface CsvRow {
|
|
@@ -97,7 +99,7 @@ export interface EvaluateTable {
|
|
|
97
99
|
export interface EvaluateStats {
|
|
98
100
|
successes: number;
|
|
99
101
|
failures: number;
|
|
100
|
-
tokenUsage: TokenUsage
|
|
102
|
+
tokenUsage: Required<TokenUsage>;
|
|
101
103
|
}
|
|
102
104
|
|
|
103
105
|
export interface EvaluateSummary {
|
package/src/util.ts
CHANGED
|
@@ -139,7 +139,6 @@ export function writeLatestResults(results: EvaluateSummary) {
|
|
|
139
139
|
try {
|
|
140
140
|
fs.mkdirSync(path.dirname(latestResultsPath), { recursive: true });
|
|
141
141
|
fs.writeFileSync(latestResultsPath, JSON.stringify(results, null, 2));
|
|
142
|
-
logger.info(`Wrote latest results to ${latestResultsPath}.`);
|
|
143
142
|
} catch (err) {
|
|
144
143
|
logger.error(`Failed to write latest results to ${latestResultsPath}:\n${err}`);
|
|
145
144
|
}
|