promptfoo 0.4.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +40 -20
- package/dist/evaluator.d.ts.map +1 -1
- package/dist/evaluator.js +63 -11
- package/dist/evaluator.js.map +1 -1
- package/dist/main.js +9 -3
- package/dist/main.js.map +1 -1
- package/dist/providers/localai.d.ts +17 -0
- package/dist/providers/localai.d.ts.map +1 -0
- package/dist/providers/localai.js +94 -0
- package/dist/providers/localai.js.map +1 -0
- package/dist/providers/openai.d.ts +26 -0
- package/dist/providers/openai.d.ts.map +1 -0
- package/dist/providers/openai.js +223 -0
- package/dist/providers/openai.js.map +1 -0
- package/dist/providers/shared.d.ts +2 -0
- package/dist/providers/shared.d.ts.map +1 -0
- package/dist/providers/shared.js +4 -0
- package/dist/providers/shared.js.map +1 -0
- package/dist/providers.d.ts +1 -20
- package/dist/providers.d.ts.map +1 -1
- package/dist/providers.js +16 -154
- package/dist/providers.js.map +1 -1
- package/dist/types.d.ts +7 -1
- package/dist/types.d.ts.map +1 -1
- package/dist/util.d.ts +1 -0
- package/dist/util.d.ts.map +1 -1
- package/dist/util.js +9 -0
- package/dist/util.js.map +1 -1
- package/package.json +2 -1
- package/src/evaluator.ts +73 -11
- package/src/main.ts +13 -4
- package/src/providers/localai.ts +112 -0
- package/src/providers/openai.ts +256 -0
- package/src/providers/shared.ts +3 -0
- package/src/providers.ts +17 -178
- package/src/types.ts +8 -1
- package/src/util.ts +10 -1
|
@@ -0,0 +1,223 @@
|
|
|
1
|
+
import { LRUCache } from 'lru-cache';
|
|
2
|
+
import logger from '../logger.js';
|
|
3
|
+
import { fetchWithTimeout } from '../util.js';
|
|
4
|
+
import { REQUEST_TIMEOUT_MS } from './shared.js';
|
|
5
|
+
const DEFAULT_OPENAI_HOST = 'api.openai.com';
|
|
6
|
+
const embeddingsCache = new LRUCache({
|
|
7
|
+
max: 1000,
|
|
8
|
+
});
|
|
9
|
+
class OpenAiGenericProvider {
|
|
10
|
+
constructor(modelName, apiKey) {
|
|
11
|
+
this.modelName = modelName;
|
|
12
|
+
const key = apiKey || process.env.OPENAI_API_KEY;
|
|
13
|
+
if (!key) {
|
|
14
|
+
throw new Error('OpenAI API key is not set. Set OPENAI_API_KEY environment variable or pass it as an argument to the constructor.');
|
|
15
|
+
}
|
|
16
|
+
this.apiKey = key;
|
|
17
|
+
this.apiHost = process.env.OPENAI_API_HOST || DEFAULT_OPENAI_HOST;
|
|
18
|
+
}
|
|
19
|
+
id() {
|
|
20
|
+
return `openai:${this.modelName}`;
|
|
21
|
+
}
|
|
22
|
+
toString() {
|
|
23
|
+
return `[OpenAI Provider ${this.modelName}]`;
|
|
24
|
+
}
|
|
25
|
+
// @ts-ignore: Prompt is not used in this implementation
|
|
26
|
+
async callApi(prompt) {
|
|
27
|
+
throw new Error('Not implemented');
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
export class OpenAiEmbeddingProvider extends OpenAiGenericProvider {
|
|
31
|
+
async callEmbeddingApi(text) {
|
|
32
|
+
if (!this.apiKey) {
|
|
33
|
+
throw new Error('OpenAI API key must be set for similarity comparison');
|
|
34
|
+
}
|
|
35
|
+
// TODO(ian): Improve cache
|
|
36
|
+
const cached = embeddingsCache.get(text);
|
|
37
|
+
if (cached) {
|
|
38
|
+
return cached;
|
|
39
|
+
}
|
|
40
|
+
const body = {
|
|
41
|
+
input: text,
|
|
42
|
+
model: this.modelName,
|
|
43
|
+
};
|
|
44
|
+
let response, data;
|
|
45
|
+
try {
|
|
46
|
+
response = await fetchWithTimeout(`https://${this.apiHost}/v1/embeddings`, {
|
|
47
|
+
method: 'POST',
|
|
48
|
+
headers: {
|
|
49
|
+
'Content-Type': 'application/json',
|
|
50
|
+
Authorization: `Bearer ${this.apiKey}`,
|
|
51
|
+
},
|
|
52
|
+
body: JSON.stringify(body),
|
|
53
|
+
}, REQUEST_TIMEOUT_MS);
|
|
54
|
+
data = (await response.json());
|
|
55
|
+
}
|
|
56
|
+
catch (err) {
|
|
57
|
+
return {
|
|
58
|
+
error: `API call error: ${String(err)}`,
|
|
59
|
+
tokenUsage: {
|
|
60
|
+
total: 0,
|
|
61
|
+
prompt: 0,
|
|
62
|
+
completion: 0,
|
|
63
|
+
},
|
|
64
|
+
};
|
|
65
|
+
}
|
|
66
|
+
logger.debug(`\tOpenAI API response (embeddings): ${JSON.stringify(data)}`);
|
|
67
|
+
try {
|
|
68
|
+
const embedding = data?.data?.[0]?.embedding;
|
|
69
|
+
if (!embedding) {
|
|
70
|
+
throw new Error('No embedding returned');
|
|
71
|
+
}
|
|
72
|
+
const ret = {
|
|
73
|
+
embedding,
|
|
74
|
+
tokenUsage: {
|
|
75
|
+
total: data.usage.total_tokens,
|
|
76
|
+
prompt: data.usage.prompt_tokens,
|
|
77
|
+
completion: data.usage.completion_tokens,
|
|
78
|
+
},
|
|
79
|
+
};
|
|
80
|
+
embeddingsCache.set(text, ret);
|
|
81
|
+
return ret;
|
|
82
|
+
}
|
|
83
|
+
catch (err) {
|
|
84
|
+
return {
|
|
85
|
+
error: `API response error: ${String(err)}: ${JSON.stringify(data)}`,
|
|
86
|
+
tokenUsage: {
|
|
87
|
+
total: data?.usage?.total_tokens,
|
|
88
|
+
prompt: data?.usage?.prompt_tokens,
|
|
89
|
+
completion: data?.usage?.completion_tokens,
|
|
90
|
+
},
|
|
91
|
+
};
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
export const DefaultEmbeddingProvider = new OpenAiEmbeddingProvider('text-embedding-ada-002');
|
|
96
|
+
class OpenAiCompletionProvider extends OpenAiGenericProvider {
|
|
97
|
+
constructor(modelName, apiKey) {
|
|
98
|
+
if (!OpenAiCompletionProvider.OPENAI_COMPLETION_MODELS.includes(modelName)) {
|
|
99
|
+
logger.warn(`Using unknown OpenAI completion model: ${modelName}`);
|
|
100
|
+
}
|
|
101
|
+
super(modelName, apiKey);
|
|
102
|
+
}
|
|
103
|
+
async callApi(prompt) {
|
|
104
|
+
const body = {
|
|
105
|
+
model: this.modelName,
|
|
106
|
+
prompt,
|
|
107
|
+
max_tokens: process.env.OPENAI_MAX_TOKENS || 1024,
|
|
108
|
+
temperature: process.env.OPENAI_TEMPERATURE || 0,
|
|
109
|
+
stop: process.env.OPENAI_STOP ? JSON.parse(process.env.OPENAI_STOP) : undefined,
|
|
110
|
+
};
|
|
111
|
+
logger.debug(`Calling OpenAI API: ${JSON.stringify(body)}`);
|
|
112
|
+
let response, data;
|
|
113
|
+
try {
|
|
114
|
+
response = await fetchWithTimeout(`https://${this.apiHost}/v1/completions`, {
|
|
115
|
+
method: 'POST',
|
|
116
|
+
headers: {
|
|
117
|
+
'Content-Type': 'application/json',
|
|
118
|
+
Authorization: `Bearer ${this.apiKey}`,
|
|
119
|
+
},
|
|
120
|
+
body: JSON.stringify(body),
|
|
121
|
+
}, REQUEST_TIMEOUT_MS);
|
|
122
|
+
data = (await response.json());
|
|
123
|
+
}
|
|
124
|
+
catch (err) {
|
|
125
|
+
return {
|
|
126
|
+
error: `API call error: ${String(err)}`,
|
|
127
|
+
};
|
|
128
|
+
}
|
|
129
|
+
logger.debug(`\tOpenAI API response: ${JSON.stringify(data)}`);
|
|
130
|
+
try {
|
|
131
|
+
return {
|
|
132
|
+
output: data.choices[0].text,
|
|
133
|
+
tokenUsage: {
|
|
134
|
+
total: data.usage.total_tokens,
|
|
135
|
+
prompt: data.usage.prompt_tokens,
|
|
136
|
+
completion: data.usage.completion_tokens,
|
|
137
|
+
},
|
|
138
|
+
};
|
|
139
|
+
}
|
|
140
|
+
catch (err) {
|
|
141
|
+
return {
|
|
142
|
+
error: `API response error: ${String(err)}: ${JSON.stringify(data)}`,
|
|
143
|
+
};
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
OpenAiCompletionProvider.OPENAI_COMPLETION_MODELS = [
|
|
148
|
+
'text-davinci-003',
|
|
149
|
+
'text-davinci-002',
|
|
150
|
+
'text-curie-001',
|
|
151
|
+
'text-babbage-001',
|
|
152
|
+
'text-ada-001',
|
|
153
|
+
];
|
|
154
|
+
export { OpenAiCompletionProvider };
|
|
155
|
+
class OpenAiChatCompletionProvider extends OpenAiGenericProvider {
|
|
156
|
+
constructor(modelName, apiKey) {
|
|
157
|
+
if (!OpenAiChatCompletionProvider.OPENAI_CHAT_MODELS.includes(modelName)) {
|
|
158
|
+
logger.warn(`Using unknown OpenAI chat model: ${modelName}`);
|
|
159
|
+
}
|
|
160
|
+
super(modelName, apiKey);
|
|
161
|
+
}
|
|
162
|
+
async callApi(prompt) {
|
|
163
|
+
let messages;
|
|
164
|
+
try {
|
|
165
|
+
// User can specify `messages` payload as JSON, or we'll just put the
|
|
166
|
+
// string prompt into a `messages` array.
|
|
167
|
+
messages = JSON.parse(prompt);
|
|
168
|
+
}
|
|
169
|
+
catch (err) {
|
|
170
|
+
messages = [{ role: 'user', content: prompt }];
|
|
171
|
+
}
|
|
172
|
+
const body = {
|
|
173
|
+
model: this.modelName,
|
|
174
|
+
messages: messages,
|
|
175
|
+
max_tokens: process.env.OPENAI_MAX_TOKENS || 1024,
|
|
176
|
+
temperature: process.env.OPENAI_MAX_TEMPERATURE || 0,
|
|
177
|
+
};
|
|
178
|
+
logger.debug(`Calling OpenAI API: ${JSON.stringify(body)}`);
|
|
179
|
+
let response, data;
|
|
180
|
+
try {
|
|
181
|
+
response = await fetchWithTimeout(`https://${this.apiHost}/v1/chat/completions`, {
|
|
182
|
+
method: 'POST',
|
|
183
|
+
headers: {
|
|
184
|
+
'Content-Type': 'application/json',
|
|
185
|
+
Authorization: `Bearer ${this.apiKey}`,
|
|
186
|
+
},
|
|
187
|
+
body: JSON.stringify(body),
|
|
188
|
+
}, REQUEST_TIMEOUT_MS);
|
|
189
|
+
data = (await response.json());
|
|
190
|
+
}
|
|
191
|
+
catch (err) {
|
|
192
|
+
return {
|
|
193
|
+
error: `API call error: ${String(err)}`,
|
|
194
|
+
};
|
|
195
|
+
}
|
|
196
|
+
logger.debug(`\tOpenAI API response: ${JSON.stringify(data)}`);
|
|
197
|
+
try {
|
|
198
|
+
return {
|
|
199
|
+
output: data.choices[0].message.content,
|
|
200
|
+
tokenUsage: {
|
|
201
|
+
total: data.usage.total_tokens,
|
|
202
|
+
prompt: data.usage.prompt_tokens,
|
|
203
|
+
completion: data.usage.completion_tokens,
|
|
204
|
+
},
|
|
205
|
+
};
|
|
206
|
+
}
|
|
207
|
+
catch (err) {
|
|
208
|
+
return {
|
|
209
|
+
error: `API response error: ${String(err)}: ${JSON.stringify(data)}`,
|
|
210
|
+
};
|
|
211
|
+
}
|
|
212
|
+
}
|
|
213
|
+
}
|
|
214
|
+
OpenAiChatCompletionProvider.OPENAI_CHAT_MODELS = [
|
|
215
|
+
'gpt-4',
|
|
216
|
+
'gpt-4-0314',
|
|
217
|
+
'gpt-4-32k',
|
|
218
|
+
'gpt-4-32k-0314',
|
|
219
|
+
'gpt-3.5-turbo',
|
|
220
|
+
'gpt-3.5-turbo-0301',
|
|
221
|
+
];
|
|
222
|
+
export { OpenAiChatCompletionProvider };
|
|
223
|
+
//# sourceMappingURL=openai.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"openai.js","sourceRoot":"","sources":["../../src/providers/openai.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAErC,OAAO,MAAM,MAAM,cAAc,CAAC;AAClC,OAAO,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AAC9C,OAAO,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AAIjD,MAAM,mBAAmB,GAAG,gBAAgB,CAAC;AAE7C,MAAM,eAAe,GAAG,IAAI,QAAQ,CAAoC;IACtE,GAAG,EAAE,IAAI;CACV,CAAC,CAAC;AAEH,MAAM,qBAAqB;IAKzB,YAAY,SAAiB,EAAE,MAAe;QAC5C,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAE3B,MAAM,GAAG,GAAG,MAAM,IAAI,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC;QACjD,IAAI,CAAC,GAAG,EAAE;YACR,MAAM,IAAI,KAAK,CACb,kHAAkH,CACnH,CAAC;SACH;QACD,IAAI,CAAC,MAAM,GAAG,GAAG,CAAC;QAElB,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,eAAe,IAAI,mBAAmB,CAAC;IACpE,CAAC;IAED,EAAE;QACA,OAAO,UAAU,IAAI,CAAC,SAAS,EAAE,CAAC;IACpC,CAAC;IAED,QAAQ;QACN,OAAO,oBAAoB,IAAI,CAAC,SAAS,GAAG,CAAC;IAC/C,CAAC;IAED,wDAAwD;IACxD,KAAK,CAAC,OAAO,CAAC,MAAc;QAC1B,MAAM,IAAI,KAAK,CAAC,iBAAiB,CAAC,CAAC;IACrC,CAAC;CACF;AAED,MAAM,OAAO,uBAAwB,SAAQ,qBAAqB;IAChE,KAAK,CAAC,gBAAgB,CAAC,IAAY;QACjC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE;YAChB,MAAM,IAAI,KAAK,CAAC,sDAAsD,CAAC,CAAC;SACzE;QAED,2BAA2B;QAC3B,MAAM,MAAM,GAAG,eAAe,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACzC,IAAI,MAAM,EAAE;YACV,OAAO,MAAM,CAAC;SACf;QAED,MAAM,IAAI,GAAG;YACX,KAAK,EAAE,IAAI;YACX,KAAK,EAAE,IAAI,CAAC,SAAS;SACtB,CAAC;QACF,IAAI,QAAQ,EAAE,IAAI,CAAC;QACnB,IAAI;YACF,QAAQ,GAAG,MAAM,gBAAgB,CAC/B,WAAW,IAAI,CAAC,OAAO,gBAAgB,EACvC;gBACE,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE;oBACP,cAAc,EAAE,kBAAkB;oBAClC,aAAa,EAAE,UAAU,IAAI,CAAC,MAAM,EAAE;iBACvC;gBACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;aAC3B,EACD,kBAAkB,CACnB,CAAC;YACF,IAAI,GAAG,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAmB,CAAC;SAClD;QAAC,OAAO,GAAG,EAAE;YACZ,OAAO;gBACL,KAAK,EAAE,mBAAmB,MAAM,CAAC,GAAG,CAAC,EAAE;gBACvC,UAAU,EAAE;oBACV,KAAK,EAAE,CAAC;oBACR,MAAM,EAAE,CAAC;oBACT,UAAU,EAAE,CAAC;iBACd;aACF,CAAC;SACH;QACD,MAAM,CAAC,KAAK,CAAC,uCAAuC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAE5E,IAAI;YACF,MAAM,SAAS,GAAG,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,SAAS,CAAC;YAC7C,IAAI,CAAC,SAAS,EAAE;gBACd,MAAM,IAAI,KAAK,CAAC,uBAAuB,CAAC,CAAC;aAC1C;YACD,MAAM,GAAG,GAAG;gBACV,SAAS;gBACT,UAAU,EAAE;oBACV,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,YAAY;oBAC9B,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,aAAa;oBAChC,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,iBAAiB;iBACzC;aACF,CAAC;YACF,eAAe,CAAC,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;YAC/B,OAAO,GAAG,CAAC;SACZ;QAAC,OAAO,GAAG,EAAE;YACZ,OAAO;gBACL,KAAK,EAAE,uBAAuB,MAAM,CAAC,GAAG,CAAC,KAAK,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE;gBACpE,UAAU,EAAE;oBACV,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,YAAY;oBAChC,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,aAAa;oBAClC,UAAU,EAAE,IAAI,EAAE,KAAK,EAAE,iBAAiB;iBAC3C;aACF,CAAC;SACH;IACH,CAAC;CACF;AAED,MAAM,CAAC,MAAM,wBAAwB,GAAG,IAAI,uBAAuB,CAAC,wBAAwB,CAAC,CAAC;AAE9F,MAAa,wBAAyB,SAAQ,qBAAqB;IASjE,YAAY,SAAiB,EAAE,MAAe;QAC5C,IAAI,CAAC,wBAAwB,CAAC,wBAAwB,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE;YAC1E,MAAM,CAAC,IAAI,CAAC,0CAA0C,SAAS,EAAE,CAAC,CAAC;SACpE;QACD,KAAK,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;IAC3B,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,MAAc;QAC1B,MAAM,IAAI,GAAG;YACX,KAAK,EAAE,IAAI,CAAC,SAAS;YACrB,MAAM;YACN,UAAU,EAAE,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,IAAI;YACjD,WAAW,EAAE,OAAO,CAAC,GAAG,CAAC,kBAAkB,IAAI,CAAC;YAChD,IAAI,EAAE,OAAO,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,SAAS;SAChF,CAAC;QACF,MAAM,CAAC,KAAK,CAAC,uBAAuB,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC5D,IAAI,QAAQ,EAAE,IAAI,CAAC;QACnB,IAAI;YACF,QAAQ,GAAG,MAAM,gBAAgB,CAC/B,WAAW,IAAI,CAAC,OAAO,iBAAiB,EACxC;gBACE,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE;oBACP,cAAc,EAAE,kBAAkB;oBAClC,aAAa,EAAE,UAAU,IAAI,CAAC,MAAM,EAAE;iBACvC;gBACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;aAC3B,EACD,kBAAkB,CACnB,CAAC;YAEF,IAAI,GAAG,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAmB,CAAC;SAClD;QAAC,OAAO,GAAG,EAAE;YACZ,OAAO;gBACL,KAAK,EAAE,mBAAmB,MAAM,CAAC,GAAG,CAAC,EAAE;aACxC,CAAC;SACH;QACD,MAAM,CAAC,KAAK,CAAC,0BAA0B,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC/D,IAAI;YACF,OAAO;gBACL,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI;gBAC5B,UAAU,EAAE;oBACV,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,YAAY;oBAC9B,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,aAAa;oBAChC,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,iBAAiB;iBACzC;aACF,CAAC;SACH;QAAC,OAAO,GAAG,EAAE;YACZ,OAAO;gBACL,KAAK,EAAE,uBAAuB,MAAM,CAAC,GAAG,CAAC,KAAK,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE;aACrE,CAAC;SACH;IACH,CAAC;;AA5DM,iDAAwB,GAAG;IAChC,kBAAkB;IAClB,kBAAkB;IAClB,gBAAgB;IAChB,kBAAkB;IAClB,cAAc;CACf,CAAC;SAPS,wBAAwB;AAgErC,MAAa,4BAA6B,SAAQ,qBAAqB;IAUrE,YAAY,SAAiB,EAAE,MAAe;QAC5C,IAAI,CAAC,4BAA4B,CAAC,kBAAkB,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE;YACxE,MAAM,CAAC,IAAI,CAAC,oCAAoC,SAAS,EAAE,CAAC,CAAC;SAC9D;QACD,KAAK,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;IAC3B,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,MAAc;QAC1B,IAAI,QAA6C,CAAC;QAClD,IAAI;YACF,qEAAqE;YACrE,yCAAyC;YACzC,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;SAC/B;QAAC,OAAO,GAAG,EAAE;YACZ,QAAQ,GAAG,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;SAChD;QACD,MAAM,IAAI,GAAG;YACX,KAAK,EAAE,IAAI,CAAC,SAAS;YACrB,QAAQ,EAAE,QAAQ;YAClB,UAAU,EAAE,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,IAAI;YACjD,WAAW,EAAE,OAAO,CAAC,GAAG,CAAC,sBAAsB,IAAI,CAAC;SACrD,CAAC;QACF,MAAM,CAAC,KAAK,CAAC,uBAAuB,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAE5D,IAAI,QAAQ,EAAE,IAAI,CAAC;QACnB,IAAI;YACF,QAAQ,GAAG,MAAM,gBAAgB,CAC/B,WAAW,IAAI,CAAC,OAAO,sBAAsB,EAC7C;gBACE,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE;oBACP,cAAc,EAAE,kBAAkB;oBAClC,aAAa,EAAE,UAAU,IAAI,CAAC,MAAM,EAAE;iBACvC;gBACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;aAC3B,EACD,kBAAkB,CACnB,CAAC;YACF,IAAI,GAAG,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAmB,CAAC;SAClD;QAAC,OAAO,GAAG,EAAE;YACZ,OAAO;gBACL,KAAK,EAAE,mBAAmB,MAAM,CAAC,GAAG,CAAC,EAAE;aACxC,CAAC;SACH;QAED,MAAM,CAAC,KAAK,CAAC,0BAA0B,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC/D,IAAI;YACF,OAAO;gBACL,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO;gBACvC,UAAU,EAAE;oBACV,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,YAAY;oBAC9B,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,aAAa;oBAChC,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,iBAAiB;iBACzC;aACF,CAAC;SACH;QAAC,OAAO,GAAG,EAAE;YACZ,OAAO;gBACL,KAAK,EAAE,uBAAuB,MAAM,CAAC,GAAG,CAAC,KAAK,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE;aACrE,CAAC;SACH;IACH,CAAC;;AArEM,+CAAkB,GAAG;IAC1B,OAAO;IACP,YAAY;IACZ,WAAW;IACX,gBAAgB;IAChB,eAAe;IACf,oBAAoB;CACrB,CAAC;SARS,4BAA4B"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"shared.d.ts","sourceRoot":"","sources":["../../src/providers/shared.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,kBAAkB,QAErB,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"shared.js","sourceRoot":"","sources":["../../src/providers/shared.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,kBAAkB,GAAG,OAAO,CAAC,GAAG,CAAC,kBAAkB;IAC9D,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,kBAAkB,EAAE,EAAE,CAAC;IAC9C,CAAC,CAAC,KAAM,CAAC"}
|
package/dist/providers.d.ts
CHANGED
|
@@ -1,22 +1,3 @@
|
|
|
1
|
-
import { ApiProvider
|
|
2
|
-
export declare class OpenAiGenericProvider implements ApiProvider {
|
|
3
|
-
modelName: string;
|
|
4
|
-
apiKey: string;
|
|
5
|
-
apiHost: string;
|
|
6
|
-
constructor(modelName: string, apiKey?: string);
|
|
7
|
-
id(): string;
|
|
8
|
-
toString(): string;
|
|
9
|
-
callApi(prompt: string): Promise<ProviderResponse>;
|
|
10
|
-
}
|
|
11
|
-
export declare class OpenAiCompletionProvider extends OpenAiGenericProvider {
|
|
12
|
-
static OPENAI_COMPLETION_MODELS: string[];
|
|
13
|
-
constructor(modelName: string, apiKey?: string);
|
|
14
|
-
callApi(prompt: string): Promise<ProviderResponse>;
|
|
15
|
-
}
|
|
16
|
-
export declare class OpenAiChatCompletionProvider extends OpenAiGenericProvider {
|
|
17
|
-
static OPENAI_CHAT_MODELS: string[];
|
|
18
|
-
constructor(modelName: string, apiKey?: string);
|
|
19
|
-
callApi(prompt: string): Promise<ProviderResponse>;
|
|
20
|
-
}
|
|
1
|
+
import { ApiProvider } from './types.js';
|
|
21
2
|
export declare function loadApiProvider(providerPath: string): Promise<ApiProvider>;
|
|
22
3
|
//# sourceMappingURL=providers.d.ts.map
|
package/dist/providers.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"providers.d.ts","sourceRoot":"","sources":["../src/providers.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,WAAW,EAAE,
|
|
1
|
+
{"version":3,"file":"providers.d.ts","sourceRoot":"","sources":["../src/providers.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAKzC,wBAAsB,eAAe,CAAC,YAAY,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC,CAuChF"}
|
package/dist/providers.js
CHANGED
|
@@ -1,158 +1,6 @@
|
|
|
1
1
|
import path from 'node:path';
|
|
2
|
-
import {
|
|
3
|
-
import
|
|
4
|
-
const DEFAULT_OPENAI_HOST = 'api.openai.com';
|
|
5
|
-
const REQUEST_TIMEOUT_MS = process.env.REQUEST_TIMEOUT_MS
|
|
6
|
-
? parseInt(process.env.REQUEST_TIMEOUT_MS, 10)
|
|
7
|
-
: 60000;
|
|
8
|
-
export class OpenAiGenericProvider {
|
|
9
|
-
constructor(modelName, apiKey) {
|
|
10
|
-
this.modelName = modelName;
|
|
11
|
-
const key = apiKey || process.env.OPENAI_API_KEY;
|
|
12
|
-
if (!key) {
|
|
13
|
-
throw new Error('OpenAI API key is not set. Set OPENAI_API_KEY environment variable or pass it as an argument to the constructor.');
|
|
14
|
-
}
|
|
15
|
-
this.apiKey = key;
|
|
16
|
-
this.apiHost = process.env.OPENAI_API_HOST || DEFAULT_OPENAI_HOST;
|
|
17
|
-
}
|
|
18
|
-
id() {
|
|
19
|
-
return `openai:${this.modelName}`;
|
|
20
|
-
}
|
|
21
|
-
toString() {
|
|
22
|
-
return `[OpenAI Provider ${this.modelName}]`;
|
|
23
|
-
}
|
|
24
|
-
// @ts-ignore: Prompt is not used in this implementation
|
|
25
|
-
async callApi(prompt) {
|
|
26
|
-
throw new Error('Not implemented');
|
|
27
|
-
}
|
|
28
|
-
}
|
|
29
|
-
class OpenAiCompletionProvider extends OpenAiGenericProvider {
|
|
30
|
-
constructor(modelName, apiKey) {
|
|
31
|
-
if (!OpenAiCompletionProvider.OPENAI_COMPLETION_MODELS.includes(modelName)) {
|
|
32
|
-
logger.warn(`Using unknown OpenAI completion model: ${modelName}`);
|
|
33
|
-
}
|
|
34
|
-
super(modelName, apiKey);
|
|
35
|
-
}
|
|
36
|
-
async callApi(prompt) {
|
|
37
|
-
const body = {
|
|
38
|
-
model: this.modelName,
|
|
39
|
-
prompt,
|
|
40
|
-
max_tokens: process.env.OPENAI_MAX_TOKENS || 1024,
|
|
41
|
-
temperature: process.env.OPENAI_TEMPERATURE || 0,
|
|
42
|
-
stop: process.env.OPENAI_STOP ? JSON.parse(process.env.OPENAI_STOP) : undefined,
|
|
43
|
-
};
|
|
44
|
-
logger.debug(`Calling OpenAI API: ${JSON.stringify(body)}`);
|
|
45
|
-
let response, data;
|
|
46
|
-
try {
|
|
47
|
-
response = await fetchWithTimeout(`https://${this.apiHost}/v1/completions`, {
|
|
48
|
-
method: 'POST',
|
|
49
|
-
headers: {
|
|
50
|
-
'Content-Type': 'application/json',
|
|
51
|
-
Authorization: `Bearer ${this.apiKey}`,
|
|
52
|
-
},
|
|
53
|
-
body: JSON.stringify(body),
|
|
54
|
-
}, REQUEST_TIMEOUT_MS);
|
|
55
|
-
data = (await response.json());
|
|
56
|
-
}
|
|
57
|
-
catch (err) {
|
|
58
|
-
return {
|
|
59
|
-
error: `API call error: ${String(err)}`,
|
|
60
|
-
};
|
|
61
|
-
}
|
|
62
|
-
logger.debug(`\tOpenAI API response: ${JSON.stringify(data)}`);
|
|
63
|
-
try {
|
|
64
|
-
return {
|
|
65
|
-
output: data.choices[0].text,
|
|
66
|
-
tokenUsage: {
|
|
67
|
-
total: data.usage.total_tokens,
|
|
68
|
-
prompt: data.usage.prompt_tokens,
|
|
69
|
-
completion: data.usage.completion_tokens,
|
|
70
|
-
},
|
|
71
|
-
};
|
|
72
|
-
}
|
|
73
|
-
catch (err) {
|
|
74
|
-
return {
|
|
75
|
-
error: `API response error: ${String(err)}: ${JSON.stringify(data)}`,
|
|
76
|
-
};
|
|
77
|
-
}
|
|
78
|
-
}
|
|
79
|
-
}
|
|
80
|
-
OpenAiCompletionProvider.OPENAI_COMPLETION_MODELS = [
|
|
81
|
-
'text-davinci-003',
|
|
82
|
-
'text-davinci-002',
|
|
83
|
-
'text-curie-001',
|
|
84
|
-
'text-babbage-001',
|
|
85
|
-
'text-ada-001',
|
|
86
|
-
];
|
|
87
|
-
export { OpenAiCompletionProvider };
|
|
88
|
-
class OpenAiChatCompletionProvider extends OpenAiGenericProvider {
|
|
89
|
-
constructor(modelName, apiKey) {
|
|
90
|
-
if (!OpenAiChatCompletionProvider.OPENAI_CHAT_MODELS.includes(modelName)) {
|
|
91
|
-
logger.warn(`Using unknown OpenAI chat model: ${modelName}`);
|
|
92
|
-
}
|
|
93
|
-
super(modelName, apiKey);
|
|
94
|
-
}
|
|
95
|
-
async callApi(prompt) {
|
|
96
|
-
let messages;
|
|
97
|
-
try {
|
|
98
|
-
// User can specify `messages` payload as JSON, or we'll just put the
|
|
99
|
-
// string prompt into a `messages` array.
|
|
100
|
-
messages = JSON.parse(prompt);
|
|
101
|
-
}
|
|
102
|
-
catch (err) {
|
|
103
|
-
messages = [{ role: 'user', content: prompt }];
|
|
104
|
-
}
|
|
105
|
-
const body = {
|
|
106
|
-
model: this.modelName,
|
|
107
|
-
messages: messages,
|
|
108
|
-
max_tokens: process.env.OPENAI_MAX_TOKENS || 1024,
|
|
109
|
-
temperature: process.env.OPENAI_MAX_TEMPERATURE || 0,
|
|
110
|
-
};
|
|
111
|
-
logger.debug(`Calling OpenAI API: ${JSON.stringify(body)}`);
|
|
112
|
-
let response, data;
|
|
113
|
-
try {
|
|
114
|
-
response = await fetchWithTimeout(`https://${this.apiHost}/v1/chat/completions`, {
|
|
115
|
-
method: 'POST',
|
|
116
|
-
headers: {
|
|
117
|
-
'Content-Type': 'application/json',
|
|
118
|
-
Authorization: `Bearer ${this.apiKey}`,
|
|
119
|
-
},
|
|
120
|
-
body: JSON.stringify(body),
|
|
121
|
-
}, REQUEST_TIMEOUT_MS);
|
|
122
|
-
data = (await response.json());
|
|
123
|
-
}
|
|
124
|
-
catch (err) {
|
|
125
|
-
return {
|
|
126
|
-
error: `API call error: ${String(err)}`,
|
|
127
|
-
};
|
|
128
|
-
}
|
|
129
|
-
logger.debug(`\tOpenAI API response: ${JSON.stringify(data)}`);
|
|
130
|
-
try {
|
|
131
|
-
return {
|
|
132
|
-
output: data.choices[0].message.content,
|
|
133
|
-
tokenUsage: {
|
|
134
|
-
total: data.usage.total_tokens,
|
|
135
|
-
prompt: data.usage.prompt_tokens,
|
|
136
|
-
completion: data.usage.completion_tokens,
|
|
137
|
-
},
|
|
138
|
-
};
|
|
139
|
-
}
|
|
140
|
-
catch (err) {
|
|
141
|
-
return {
|
|
142
|
-
error: `API response error: ${String(err)}: ${JSON.stringify(data)}`,
|
|
143
|
-
};
|
|
144
|
-
}
|
|
145
|
-
}
|
|
146
|
-
}
|
|
147
|
-
OpenAiChatCompletionProvider.OPENAI_CHAT_MODELS = [
|
|
148
|
-
'gpt-4',
|
|
149
|
-
'gpt-4-0314',
|
|
150
|
-
'gpt-4-32k',
|
|
151
|
-
'gpt-4-32k-0314',
|
|
152
|
-
'gpt-3.5-turbo',
|
|
153
|
-
'gpt-3.5-turbo-0301',
|
|
154
|
-
];
|
|
155
|
-
export { OpenAiChatCompletionProvider };
|
|
2
|
+
import { OpenAiCompletionProvider, OpenAiChatCompletionProvider } from './providers/openai.js';
|
|
3
|
+
import { LocalAiCompletionProvider, LocalAiChatProvider } from './providers/localai.js';
|
|
156
4
|
export async function loadApiProvider(providerPath) {
|
|
157
5
|
if (providerPath?.startsWith('openai:')) {
|
|
158
6
|
// Load OpenAI module
|
|
@@ -175,6 +23,20 @@ export async function loadApiProvider(providerPath) {
|
|
|
175
23
|
throw new Error(`Unknown OpenAI model type: ${modelType}. Use one of the following providers: openai:chat:<model name>, openai:completion:<model name>`);
|
|
176
24
|
}
|
|
177
25
|
}
|
|
26
|
+
if (providerPath?.startsWith('localai:')) {
|
|
27
|
+
const options = providerPath.split(':');
|
|
28
|
+
const modelType = options[1];
|
|
29
|
+
const modelName = options[2];
|
|
30
|
+
if (modelType === 'chat') {
|
|
31
|
+
return new LocalAiChatProvider(modelName);
|
|
32
|
+
}
|
|
33
|
+
else if (modelType === 'completion') {
|
|
34
|
+
return new LocalAiCompletionProvider(modelName);
|
|
35
|
+
}
|
|
36
|
+
else {
|
|
37
|
+
return new LocalAiChatProvider(modelType);
|
|
38
|
+
}
|
|
39
|
+
}
|
|
178
40
|
// Load custom module
|
|
179
41
|
const CustomApiProvider = (await import(path.join(process.cwd(), providerPath))).default;
|
|
180
42
|
return new CustomApiProvider();
|
package/dist/providers.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"providers.js","sourceRoot":"","sources":["../src/providers.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,WAAW,CAAC;
|
|
1
|
+
{"version":3,"file":"providers.js","sourceRoot":"","sources":["../src/providers.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,WAAW,CAAC;AAI7B,OAAO,EAAE,wBAAwB,EAAE,4BAA4B,EAAE,MAAM,uBAAuB,CAAC;AAC/F,OAAO,EAAE,yBAAyB,EAAE,mBAAmB,EAAE,MAAM,wBAAwB,CAAC;AAExF,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,YAAoB;IACxD,IAAI,YAAY,EAAE,UAAU,CAAC,SAAS,CAAC,EAAE;QACvC,qBAAqB;QACrB,MAAM,OAAO,GAAG,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACxC,MAAM,SAAS,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;QAC7B,MAAM,SAAS,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;QAE7B,IAAI,SAAS,KAAK,MAAM,EAAE;YACxB,OAAO,IAAI,4BAA4B,CAAC,SAAS,IAAI,eAAe,CAAC,CAAC;SACvE;aAAM,IAAI,SAAS,KAAK,YAAY,EAAE;YACrC,OAAO,IAAI,wBAAwB,CAAC,SAAS,IAAI,kBAAkB,CAAC,CAAC;SACtE;aAAM,IAAI,4BAA4B,CAAC,kBAAkB,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE;YAC9E,OAAO,IAAI,4BAA4B,CAAC,SAAS,CAAC,CAAC;SACpD;aAAM,IAAI,wBAAwB,CAAC,wBAAwB,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE;YAChF,OAAO,IAAI,wBAAwB,CAAC,SAAS,CAAC,CAAC;SAChD;aAAM;YACL,MAAM,IAAI,KAAK,CACb,8BAA8B,SAAS,gGAAgG,CACxI,CAAC;SACH;KACF;IAED,IAAI,YAAY,EAAE,UAAU,CAAC,UAAU,CAAC,EAAE;QACxC,MAAM,OAAO,GAAG,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACxC,MAAM,SAAS,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;QAC7B,MAAM,SAAS,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;QAE7B,IAAI,SAAS,KAAK,MAAM,EAAE;YACxB,OAAO,IAAI,mBAAmB,CAAC,SAAS,CAAC,CAAC;SAC3C;aAAM,IAAI,SAAS,KAAK,YAAY,EAAE;YACrC,OAAO,IAAI,yBAAyB,CAAC,SAAS,CAAC,CAAC;SACjD;aAAM;YACL,OAAO,IAAI,mBAAmB,CAAC,SAAS,CAAC,CAAC;SAC3C;KACF;IAED,qBAAqB;IACrB,MAAM,iBAAiB,GAAG,CAAC,MAAM,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,YAAY,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC;IACzF,OAAO,IAAI,iBAAiB,EAAE,CAAC;AACjC,CAAC"}
|
package/dist/types.d.ts
CHANGED
|
@@ -5,10 +5,11 @@ export interface CommandLineOptions {
|
|
|
5
5
|
vars?: string;
|
|
6
6
|
config?: string;
|
|
7
7
|
verbose?: boolean;
|
|
8
|
-
maxConcurrency?:
|
|
8
|
+
maxConcurrency?: string;
|
|
9
9
|
grader?: string;
|
|
10
10
|
view?: boolean;
|
|
11
11
|
noWrite?: boolean;
|
|
12
|
+
tableCellMaxLength?: string;
|
|
12
13
|
}
|
|
13
14
|
export interface ApiProvider {
|
|
14
15
|
id: () => string;
|
|
@@ -24,6 +25,11 @@ export interface ProviderResponse {
|
|
|
24
25
|
output?: string;
|
|
25
26
|
tokenUsage?: TokenUsage;
|
|
26
27
|
}
|
|
28
|
+
export interface ProviderEmbeddingResponse {
|
|
29
|
+
error?: string;
|
|
30
|
+
embedding?: number[];
|
|
31
|
+
tokenUsage?: TokenUsage;
|
|
32
|
+
}
|
|
27
33
|
export interface CsvRow {
|
|
28
34
|
[key: string]: string;
|
|
29
35
|
}
|
package/dist/types.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,kBAAkB;IACjC,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,OAAO,CAAC,EAAE,OAAO,CAAC;
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,kBAAkB;IACjC,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,kBAAkB,CAAC,EAAE,MAAM,CAAC;CAC7B;AAED,MAAM,WAAW,WAAW;IAC1B,EAAE,EAAE,MAAM,MAAM,CAAC;IACjB,OAAO,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,OAAO,CAAC,gBAAgB,CAAC,CAAC;CACxD;AAED,MAAM,WAAW,UAAU;IACzB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,gBAAgB;IAC/B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,UAAU,CAAC,EAAE,UAAU,CAAC;CACzB;AAED,MAAM,WAAW,yBAAyB;IACxC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB,UAAU,CAAC,EAAE,UAAU,CAAC;CACzB;AAED,MAAM,WAAW,MAAM;IACrB,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAC;CACvB;AAED,MAAM,MAAM,UAAU,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;AAEhD,MAAM,WAAW,aAAa;IAC5B,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,WAAW,CAAC;CACvB;AAED,MAAM,WAAW,eAAe;IAC9B,SAAS,EAAE,WAAW,EAAE,CAAC;IACzB,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,IAAI,CAAC,EAAE,UAAU,EAAE,CAAC;IAEpB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,eAAe,CAAC,EAAE,OAAO,CAAC;IAE1B,OAAO,CAAC,EAAE,aAAa,CAAC;CACzB;AAED,MAAM,WAAW,MAAM;IACrB,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC7B,QAAQ,CAAC,EAAE,gBAAgB,CAAC;IAC5B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,OAAO,CAAC;CAClB;AAED,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE;QACJ,OAAO,EAAE,MAAM,EAAE,CAAC;QAClB,IAAI,EAAE,MAAM,EAAE,CAAC;KAChB,CAAC;IAEF,IAAI,EAAE;QACJ,OAAO,EAAE,MAAM,EAAE,CAAC;QAClB,IAAI,EAAE,MAAM,EAAE,CAAC;KAChB,EAAE,CAAC;CACL;AAED,MAAM,WAAW,aAAa;IAC5B,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,UAAU,CAAC;CACxB;AAED,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,cAAc,EAAE,CAAC;IAC1B,KAAK,EAAE,aAAa,CAAC;IACrB,KAAK,EAAE,aAAa,CAAC;CACtB"}
|
package/dist/util.d.ts
CHANGED
|
@@ -8,4 +8,5 @@ export declare function fetchWithTimeout(url: RequestInfo, options: RequestInit
|
|
|
8
8
|
export declare function getConfigDirectoryPath(): string;
|
|
9
9
|
export declare function getLatestResultsPath(): string;
|
|
10
10
|
export declare function writeLatestResults(results: EvaluateSummary): void;
|
|
11
|
+
export declare function cosineSimilarity(vecA: number[], vecB: number[]): number;
|
|
11
12
|
//# sourceMappingURL=util.d.ts.map
|
package/dist/util.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"util.d.ts","sourceRoot":"","sources":["../src/util.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"util.d.ts","sourceRoot":"","sources":["../src/util.ts"],"names":[],"mappings":"AAQA,OAAO,EAAE,MAAM,EAAE,MAAM,YAAY,CAAC;AAOpC,OAAO,KAAK,EAAE,WAAW,EAAE,WAAW,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAErE,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAYlD,wBAAgB,WAAW,CAAC,WAAW,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE,CAM3D;AAED,wBAAgB,QAAQ,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,EAAE,CAanD;AAED,wBAAgB,WAAW,CAAC,UAAU,EAAE,MAAM,EAAE,OAAO,EAAE,eAAe,GAAG,IAAI,CAuB9E;AAED,wBAAgB,gBAAgB,CAC9B,GAAG,EAAE,WAAW,EAChB,OAAO,yBAAkB,EACzB,OAAO,EAAE,MAAM,GACd,OAAO,CAAC,QAAQ,CAAC,CAwBnB;AAED,wBAAgB,sBAAsB,IAAI,MAAM,CAE/C;AAED,wBAAgB,oBAAoB,IAAI,MAAM,CAE7C;AAED,wBAAgB,kBAAkB,CAAC,OAAO,EAAE,eAAe,QAS1D;AAED,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,UAQ9D"}
|
package/dist/util.js
CHANGED
|
@@ -108,4 +108,13 @@ export function writeLatestResults(results) {
|
|
|
108
108
|
logger.error(`Failed to write latest results to ${latestResultsPath}:\n${err}`);
|
|
109
109
|
}
|
|
110
110
|
}
|
|
111
|
+
export function cosineSimilarity(vecA, vecB) {
|
|
112
|
+
if (vecA.length !== vecB.length) {
|
|
113
|
+
throw new Error('Vectors must be of equal length');
|
|
114
|
+
}
|
|
115
|
+
const dotProduct = vecA.reduce((acc, val, idx) => acc + val * vecB[idx], 0);
|
|
116
|
+
const vecAMagnitude = Math.sqrt(vecA.reduce((acc, val) => acc + val * val, 0));
|
|
117
|
+
const vecBMagnitude = Math.sqrt(vecB.reduce((acc, val) => acc + val * val, 0));
|
|
118
|
+
return dotProduct / (vecAMagnitude * vecBMagnitude);
|
|
119
|
+
}
|
|
111
120
|
//# sourceMappingURL=util.js.map
|
package/dist/util.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"util.js","sourceRoot":"","sources":["../src/util.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AACzB,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAClC,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;
|
|
1
|
+
{"version":3,"file":"util.js","sourceRoot":"","sources":["../src/util.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AACzB,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAClC,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAE9B,OAAO,KAAK,MAAM,YAAY,CAAC;AAC/B,OAAO,IAAI,MAAM,SAAS,CAAC;AAC3B,OAAO,QAAQ,MAAM,UAAU,CAAC;AAChC,OAAO,EAAE,KAAK,IAAI,SAAS,EAAE,MAAM,MAAM,CAAC;AAE1C,OAAO,EAAE,KAAK,IAAI,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AACnD,OAAO,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AAE/C,OAAO,MAAM,MAAM,aAAa,CAAC;AACjC,OAAO,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC;AAMxC,MAAM,gBAAgB,GAAG,KAAK,CAAC;AAE/B,SAAS,SAAS,CAAC,IAAY;IAC7B,IAAI;QACF,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;KACzB;IAAC,OAAO,GAAG,EAAE;QACZ,OAAO,SAAS,CAAC;KAClB;AACH,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,WAAqB;IAC/C,IAAI,cAAc,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,EAAE,CAAC,YAAY,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC;IAC/E,IAAI,cAAc,CAAC,MAAM,KAAK,CAAC,EAAE;QAC/B,cAAc,GAAG,cAAc,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;KACjF;IACD,OAAO,cAAc,CAAC;AACxB,CAAC;AAED,MAAM,UAAU,QAAQ,CAAC,QAAgB;IACvC,MAAM,aAAa,GAAG,SAAS,CAAC,QAAQ,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IACvD,IAAI,IAAI,GAAa,EAAE,CAAC;IAExB,IAAI,aAAa,KAAK,KAAK,EAAE;QAC3B,IAAI,GAAG,QAAQ,CAAC,EAAE,CAAC,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;KACxE;SAAM,IAAI,aAAa,KAAK,MAAM,EAAE;QACnC,IAAI,GAAG,SAAS,CAAC,EAAE,CAAC,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAC;KACtD;SAAM,IAAI,aAAa,KAAK,MAAM,IAAI,aAAa,KAAK,KAAK,EAAE;QAC9D,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAmB,CAAC;KACxE;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,UAAkB,EAAE,OAAwB;IACtE,MAAM,eAAe,GAAG,UAAU,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,WAAW,EAAE,CAAC;IAEnE,IAAI,eAAe,KAAK,KAAK,IAAI,eAAe,KAAK,KAAK,EAAE;QAC1D,MAAM,SAAS,GAAG,SAAS,CAAC;YAC1B,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC;YAC3D,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,GAAG,GAAG,CAAC,OAAO,EAAE,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC;SAClE,CAAC,CAAC;QACH,EAAE,CAAC,aAAa,CAAC,UAAU,EAAE,SAAS,CAAC,CAAC;KACzC;SAAM,IAAI,eAAe,KAAK,MAAM,EAAE;QACrC,EAAE,CAAC,aAAa,CAAC,UAAU,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;KAChE;SAAM,IAAI,eAAe,KAAK,MAAM,IAAI,eAAe,KAAK,KAAK,EAAE;QAClE,EAAE,CAAC,aAAa,CAAC,UAAU,EAAE,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;KAClD;SAAM,IAAI,eAAe,KAAK,MAAM,EAAE;QACrC,MAAM,QAAQ,GAAG,EAAE,CAAC,YAAY,CAAC,GAAG,YAAY,EAAE,mBAAmB,EAAE,OAAO,CAAC,CAAC;QAChF,MAAM,UAAU,GAAG,QAAQ,CAAC,YAAY,CAAC,QAAQ,EAAE;YACjD,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,OAAO,EAAE,OAAO,CAAC,OAAO;SACzB,CAAC,CAAC;QACH,EAAE,CAAC,aAAa,CAAC,UAAU,EAAE,UAAU,CAAC,CAAC;KAC1C;SAAM;QACL,MAAM,IAAI,KAAK,CAAC,yDAAyD,CAAC,CAAC;KAC5E;AACH,CAAC;AAED,MAAM,UAAU,gBAAgB,CAC9B,GAAgB,EAChB,UAAuB,EAAE,EACzB,OAAe;IAEf,OAAO,IAAI,OAAO,CAAC,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE;QAC3C,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;QACzC,MAAM,EAAE,MAAM,EAAE,GAAG,UAAU,CAAC;QAC9B,OAAO,CAAC,MAAM,GAAG,MAAM,CAAC;QAExB,MAAM,SAAS,GAAG,UAAU,CAAC,GAAG,EAAE;YAChC,UAAU,CAAC,KAAK,EAAE,CAAC;YACnB,MAAM,CAAC,IAAI,KAAK,CAAC,2BAA2B,OAAO,KAAK,CAAC,CAAC,CAAC;QAC7D,CAAC,EAAE,OAAO,CAAC,CAAC;QAEZ,IAAI;YACF,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;YAC3C,YAAY,CAAC,SAAS,CAAC,CAAC;YACxB,OAAO,CAAC,QAAQ,CAAC,CAAC;SACnB;QAAC,OAAO,KAAK,EAAE;YACd,IAAI,KAAK,YAAY,KAAK,IAAI,KAAK,CAAC,IAAI,KAAK,YAAY,EAAE;gBACzD,qDAAqD;aACtD;iBAAM;gBACL,YAAY,CAAC,SAAS,CAAC,CAAC;gBACxB,MAAM,CAAC,KAAK,CAAC,CAAC;aACf;SACF;IACH,CAAC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,UAAU,sBAAsB;IACpC,OAAO,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,EAAE,YAAY,CAAC,CAAC;AAC/C,CAAC;AAED,MAAM,UAAU,oBAAoB;IAClC,OAAO,IAAI,CAAC,IAAI,CAAC,sBAAsB,EAAE,EAAE,QAAQ,EAAE,aAAa,CAAC,CAAC;AACtE,CAAC;AAED,MAAM,UAAU,kBAAkB,CAAC,OAAwB;IACzD,MAAM,iBAAiB,GAAG,oBAAoB,EAAE,CAAC;IACjD,IAAI;QACF,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,iBAAiB,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACnE,EAAE,CAAC,aAAa,CAAC,iBAAiB,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QACtE,MAAM,CAAC,IAAI,CAAC,2BAA2B,iBAAiB,GAAG,CAAC,CAAC;KAC9D;IAAC,OAAO,GAAG,EAAE;QACZ,MAAM,CAAC,KAAK,CAAC,qCAAqC,iBAAiB,MAAM,GAAG,EAAE,CAAC,CAAC;KACjF;AACH,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,IAAc,EAAE,IAAc;IAC7D,IAAI,IAAI,CAAC,MAAM,KAAK,IAAI,CAAC,MAAM,EAAE;QAC/B,MAAM,IAAI,KAAK,CAAC,iCAAiC,CAAC,CAAC;KACpD;IACD,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC,GAAG,GAAG,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;IAC5E,MAAM,aAAa,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC,GAAG,GAAG,GAAG,GAAG,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC;IAC/E,MAAM,aAAa,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC,GAAG,GAAG,GAAG,GAAG,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC;IAC/E,OAAO,UAAU,GAAG,CAAC,aAAa,GAAG,aAAa,CAAC,CAAC;AACtD,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "promptfoo",
|
|
3
3
|
"author": "Ian Webster",
|
|
4
|
-
"version": "0.
|
|
4
|
+
"version": "0.5.0",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"type": "module",
|
|
7
7
|
"main": "dist/index.js",
|
|
@@ -57,6 +57,7 @@
|
|
|
57
57
|
"debounce": "^1.2.1",
|
|
58
58
|
"express": "^4.18.2",
|
|
59
59
|
"js-yaml": "^4.1.0",
|
|
60
|
+
"lru-cache": "^9.1.1",
|
|
60
61
|
"node-fetch": "^3.3.1",
|
|
61
62
|
"nunjucks": "^3.2.4",
|
|
62
63
|
"open": "^9.1.0",
|