promptfoo 0.9.0 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +50 -40
- package/dist/assertions.d.ts +2 -2
- package/dist/assertions.d.ts.map +1 -1
- package/dist/assertions.js +186 -44
- package/dist/assertions.js.map +1 -1
- package/dist/cache.js +9 -9
- package/dist/cache.js.map +1 -1
- package/dist/evaluator.d.ts +1 -1
- package/dist/evaluator.d.ts.map +1 -1
- package/dist/evaluator.js +30 -23
- package/dist/evaluator.js.map +1 -1
- package/dist/index.d.ts +10 -10
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +18 -14
- package/dist/index.js.map +1 -1
- package/dist/main.js +49 -44
- package/dist/main.js.map +1 -1
- package/dist/providers/localai.js +11 -11
- package/dist/providers/localai.js.map +1 -1
- package/dist/providers/openai.d.ts.map +1 -1
- package/dist/providers/openai.js +30 -21
- package/dist/providers/openai.js.map +1 -1
- package/dist/providers.d.ts +3 -3
- package/dist/providers.d.ts.map +1 -1
- package/dist/providers.js +15 -15
- package/dist/providers.js.map +1 -1
- package/dist/types.d.ts +7 -3
- package/dist/types.d.ts.map +1 -1
- package/dist/util.d.ts +4 -4
- package/dist/util.d.ts.map +1 -1
- package/dist/util.js +49 -18
- package/dist/util.js.map +1 -1
- package/dist/web/client/assets/index-15dfcd18.js +172 -0
- package/dist/web/client/assets/index-87905193.css +1 -0
- package/dist/web/client/index.html +2 -2
- package/dist/web/server.js +9 -9
- package/dist/web/server.js.map +1 -1
- package/package.json +3 -1
- package/src/assertions.ts +249 -38
- package/src/cache.ts +2 -2
- package/src/evaluator.ts +25 -18
- package/src/index.ts +13 -8
- package/src/main.ts +28 -15
- package/src/providers/localai.ts +3 -3
- package/src/providers/openai.ts +16 -8
- package/src/providers.ts +3 -3
- package/src/types.ts +24 -3
- package/src/util.ts +48 -17
- package/src/web/client/package-lock.json +5729 -0
- package/src/web/client/src/ResultsTable.css +35 -4
- package/src/web/client/src/ResultsTable.tsx +150 -70
- package/src/web/client/src/ResultsView.tsx +83 -18
- package/src/web/client/src/index.css +6 -0
- package/src/web/client/src/types.ts +2 -0
- package/src/web/server.ts +3 -3
- package/dist/web/client/assets/index-207192fc.css +0 -1
- package/dist/web/client/assets/index-8751749f.js +0 -172
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"openai.d.ts","sourceRoot":"","sources":["../../src/providers/openai.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,WAAW,EAAE,yBAAyB,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAI5F,UAAU,uBAAuB;IAC/B,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,cAAM,qBAAsB,YAAW,WAAW;IAChD,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;gBAEJ,SAAS,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM;IAQ9C,EAAE,IAAI,MAAM;IAIZ,QAAQ,IAAI,MAAM;IAKZ,OAAO,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,uBAAuB,GAAG,OAAO,CAAC,gBAAgB,CAAC;CAG5F;AAED,qBAAa,uBAAwB,SAAQ,qBAAqB;IAC1D,gBAAgB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,yBAAyB,CAAC;CA+DzE;AAED,qBAAa,wBAAyB,SAAQ,qBAAqB;IACjE,MAAM,CAAC,wBAAwB,WAM7B;gBAEU,SAAS,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM;IAOxC,OAAO,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,uBAAuB,GAAG,OAAO,CAAC,gBAAgB,CAAC;
|
|
1
|
+
{"version":3,"file":"openai.d.ts","sourceRoot":"","sources":["../../src/providers/openai.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,WAAW,EAAE,yBAAyB,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAI5F,UAAU,uBAAuB;IAC/B,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,cAAM,qBAAsB,YAAW,WAAW;IAChD,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;gBAEJ,SAAS,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM;IAQ9C,EAAE,IAAI,MAAM;IAIZ,QAAQ,IAAI,MAAM;IAKZ,OAAO,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,uBAAuB,GAAG,OAAO,CAAC,gBAAgB,CAAC;CAG5F;AAED,qBAAa,uBAAwB,SAAQ,qBAAqB;IAC1D,gBAAgB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,yBAAyB,CAAC;CA+DzE;AAED,qBAAa,wBAAyB,SAAQ,qBAAqB;IACjE,MAAM,CAAC,wBAAwB,WAM7B;gBAEU,SAAS,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM;IAOxC,OAAO,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,uBAAuB,GAAG,OAAO,CAAC,gBAAgB,CAAC;CA6D5F;AAED,qBAAa,4BAA6B,SAAQ,qBAAqB;IACrE,MAAM,CAAC,kBAAkB,WAOvB;gBAEU,SAAS,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM;IAQxC,OAAO,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,uBAAuB,GAAG,OAAO,CAAC,gBAAgB,CAAC;CA8D5F;AAED,eAAO,MAAM,wBAAwB,yBAAwD,CAAC;AAC9F,eAAO,MAAM,sBAAsB,8BAA4C,CAAC;AAChF,eAAO,MAAM,0BAA0B,8BAA4C,CAAC"}
|
package/dist/providers/openai.js
CHANGED
|
@@ -4,9 +4,9 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
4
4
|
};
|
|
5
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
6
|
exports.DefaultSuggestionsProvider = exports.DefaultGradingProvider = exports.DefaultEmbeddingProvider = exports.OpenAiChatCompletionProvider = exports.OpenAiCompletionProvider = exports.OpenAiEmbeddingProvider = void 0;
|
|
7
|
-
const
|
|
8
|
-
const
|
|
9
|
-
const
|
|
7
|
+
const logger_1 = __importDefault(require("../logger"));
|
|
8
|
+
const cache_1 = require("../cache");
|
|
9
|
+
const shared_1 = require("./shared");
|
|
10
10
|
const DEFAULT_OPENAI_HOST = 'api.openai.com';
|
|
11
11
|
class OpenAiGenericProvider {
|
|
12
12
|
constructor(modelName, apiKey) {
|
|
@@ -36,14 +36,14 @@ class OpenAiEmbeddingProvider extends OpenAiGenericProvider {
|
|
|
36
36
|
};
|
|
37
37
|
let data, cached = false;
|
|
38
38
|
try {
|
|
39
|
-
({ data, cached } = (await (0,
|
|
39
|
+
({ data, cached } = (await (0, cache_1.fetchJsonWithCache)(`https://${this.apiHost}/v1/embeddings`, {
|
|
40
40
|
method: 'POST',
|
|
41
41
|
headers: {
|
|
42
42
|
'Content-Type': 'application/json',
|
|
43
43
|
Authorization: `Bearer ${this.apiKey}`,
|
|
44
44
|
},
|
|
45
45
|
body: JSON.stringify(body),
|
|
46
|
-
},
|
|
46
|
+
}, shared_1.REQUEST_TIMEOUT_MS)));
|
|
47
47
|
}
|
|
48
48
|
catch (err) {
|
|
49
49
|
return {
|
|
@@ -55,7 +55,7 @@ class OpenAiEmbeddingProvider extends OpenAiGenericProvider {
|
|
|
55
55
|
},
|
|
56
56
|
};
|
|
57
57
|
}
|
|
58
|
-
|
|
58
|
+
logger_1.default.debug(`\tOpenAI API response (embeddings): ${JSON.stringify(data)}`);
|
|
59
59
|
try {
|
|
60
60
|
const embedding = data?.data?.[0]?.embedding;
|
|
61
61
|
if (!embedding) {
|
|
@@ -89,7 +89,7 @@ exports.OpenAiEmbeddingProvider = OpenAiEmbeddingProvider;
|
|
|
89
89
|
class OpenAiCompletionProvider extends OpenAiGenericProvider {
|
|
90
90
|
constructor(modelName, apiKey) {
|
|
91
91
|
if (!OpenAiCompletionProvider.OPENAI_COMPLETION_MODELS.includes(modelName)) {
|
|
92
|
-
|
|
92
|
+
logger_1.default.warn(`Using unknown OpenAI completion model: ${modelName}`);
|
|
93
93
|
}
|
|
94
94
|
super(modelName, apiKey);
|
|
95
95
|
}
|
|
@@ -97,31 +97,40 @@ class OpenAiCompletionProvider extends OpenAiGenericProvider {
|
|
|
97
97
|
if (!this.apiKey) {
|
|
98
98
|
throw new Error('OpenAI API key is not set. Set OPENAI_API_KEY environment variable or pass it as an argument to the constructor.');
|
|
99
99
|
}
|
|
100
|
+
let stop;
|
|
101
|
+
try {
|
|
102
|
+
stop = process.env.OPENAI_STOP
|
|
103
|
+
? JSON.parse(process.env.OPENAI_STOP)
|
|
104
|
+
: ['<|im_end|>', '<|endoftext|>'];
|
|
105
|
+
}
|
|
106
|
+
catch (err) {
|
|
107
|
+
throw new Error(`OPENAI_STOP is not a valid JSON string: ${err}`);
|
|
108
|
+
}
|
|
100
109
|
const body = {
|
|
101
110
|
model: this.modelName,
|
|
102
111
|
prompt,
|
|
103
|
-
max_tokens: process.env.OPENAI_MAX_TOKENS || 1024,
|
|
104
|
-
temperature: options?.temperature ?? (process.env.
|
|
105
|
-
stop
|
|
112
|
+
max_tokens: parseInt(process.env.OPENAI_MAX_TOKENS || '1024'),
|
|
113
|
+
temperature: options?.temperature ?? parseFloat(process.env.OPENAI_TEMPERATURE || '0'),
|
|
114
|
+
stop,
|
|
106
115
|
};
|
|
107
|
-
|
|
116
|
+
logger_1.default.debug(`Calling OpenAI API: ${JSON.stringify(body)}`);
|
|
108
117
|
let data, cached = false;
|
|
109
118
|
try {
|
|
110
|
-
({ data, cached } = (await (0,
|
|
119
|
+
({ data, cached } = (await (0, cache_1.fetchJsonWithCache)(`https://${this.apiHost}/v1/completions`, {
|
|
111
120
|
method: 'POST',
|
|
112
121
|
headers: {
|
|
113
122
|
'Content-Type': 'application/json',
|
|
114
123
|
Authorization: `Bearer ${this.apiKey}`,
|
|
115
124
|
},
|
|
116
125
|
body: JSON.stringify(body),
|
|
117
|
-
},
|
|
126
|
+
}, shared_1.REQUEST_TIMEOUT_MS)));
|
|
118
127
|
}
|
|
119
128
|
catch (err) {
|
|
120
129
|
return {
|
|
121
130
|
error: `API call error: ${String(err)}`,
|
|
122
131
|
};
|
|
123
132
|
}
|
|
124
|
-
|
|
133
|
+
logger_1.default.debug(`\tOpenAI API response: ${JSON.stringify(data)}`);
|
|
125
134
|
try {
|
|
126
135
|
return {
|
|
127
136
|
output: data.choices[0].text,
|
|
@@ -152,7 +161,7 @@ exports.OpenAiCompletionProvider = OpenAiCompletionProvider;
|
|
|
152
161
|
class OpenAiChatCompletionProvider extends OpenAiGenericProvider {
|
|
153
162
|
constructor(modelName, apiKey) {
|
|
154
163
|
if (!OpenAiChatCompletionProvider.OPENAI_CHAT_MODELS.includes(modelName)) {
|
|
155
|
-
|
|
164
|
+
logger_1.default.warn(`Using unknown OpenAI chat model: ${modelName}`);
|
|
156
165
|
}
|
|
157
166
|
super(modelName, apiKey);
|
|
158
167
|
}
|
|
@@ -173,27 +182,27 @@ class OpenAiChatCompletionProvider extends OpenAiGenericProvider {
|
|
|
173
182
|
const body = {
|
|
174
183
|
model: this.modelName,
|
|
175
184
|
messages: messages,
|
|
176
|
-
max_tokens: process.env.OPENAI_MAX_TOKENS || 1024,
|
|
177
|
-
temperature: options?.temperature ?? (process.env.
|
|
185
|
+
max_tokens: parseInt(process.env.OPENAI_MAX_TOKENS || '1024'),
|
|
186
|
+
temperature: options?.temperature ?? parseFloat(process.env.OPENAI_TEMPERATURE || '0'),
|
|
178
187
|
};
|
|
179
|
-
|
|
188
|
+
logger_1.default.debug(`Calling OpenAI API: ${JSON.stringify(body)}`);
|
|
180
189
|
let data, cached = false;
|
|
181
190
|
try {
|
|
182
|
-
({ data, cached } = (await (0,
|
|
191
|
+
({ data, cached } = (await (0, cache_1.fetchJsonWithCache)(`https://${this.apiHost}/v1/chat/completions`, {
|
|
183
192
|
method: 'POST',
|
|
184
193
|
headers: {
|
|
185
194
|
'Content-Type': 'application/json',
|
|
186
195
|
Authorization: `Bearer ${this.apiKey}`,
|
|
187
196
|
},
|
|
188
197
|
body: JSON.stringify(body),
|
|
189
|
-
},
|
|
198
|
+
}, shared_1.REQUEST_TIMEOUT_MS)));
|
|
190
199
|
}
|
|
191
200
|
catch (err) {
|
|
192
201
|
return {
|
|
193
202
|
error: `API call error: ${String(err)}`,
|
|
194
203
|
};
|
|
195
204
|
}
|
|
196
|
-
|
|
205
|
+
logger_1.default.debug(`\tOpenAI API response: ${JSON.stringify(data)}`);
|
|
197
206
|
try {
|
|
198
207
|
return {
|
|
199
208
|
output: data.choices[0].message.content,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"openai.js","sourceRoot":"","sources":["../../src/providers/openai.ts"],"names":[],"mappings":";;;;;;AAAA,
|
|
1
|
+
{"version":3,"file":"openai.js","sourceRoot":"","sources":["../../src/providers/openai.ts"],"names":[],"mappings":";;;;;;AAAA,uDAA+B;AAC/B,oCAA8C;AAC9C,qCAA8C;AAI9C,MAAM,mBAAmB,GAAG,gBAAgB,CAAC;AAM7C,MAAM,qBAAqB;IAKzB,YAAY,SAAiB,EAAE,MAAe;QAC5C,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAE3B,IAAI,CAAC,MAAM,GAAG,MAAM,IAAI,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC;QAEnD,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,eAAe,IAAI,mBAAmB,CAAC;IACpE,CAAC;IAED,EAAE;QACA,OAAO,UAAU,IAAI,CAAC,SAAS,EAAE,CAAC;IACpC,CAAC;IAED,QAAQ;QACN,OAAO,oBAAoB,IAAI,CAAC,SAAS,GAAG,CAAC;IAC/C,CAAC;IAED,wDAAwD;IACxD,KAAK,CAAC,OAAO,CAAC,MAAc,EAAE,OAAiC;QAC7D,MAAM,IAAI,KAAK,CAAC,iBAAiB,CAAC,CAAC;IACrC,CAAC;CACF;AAED,MAAa,uBAAwB,SAAQ,qBAAqB;IAChE,KAAK,CAAC,gBAAgB,CAAC,IAAY;QACjC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE;YAChB,MAAM,IAAI,KAAK,CAAC,sDAAsD,CAAC,CAAC;SACzE;QAED,MAAM,IAAI,GAAG;YACX,KAAK,EAAE,IAAI;YACX,KAAK,EAAE,IAAI,CAAC,SAAS;SACtB,CAAC;QACF,IAAI,IAAI,EACN,MAAM,GAAG,KAAK,CAAC;QACjB,IAAI;YACF,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,IAAA,0BAAkB,EAC3C,WAAW,IAAI,CAAC,OAAO,gBAAgB,EACvC;gBACE,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE;oBACP,cAAc,EAAE,kBAAkB;oBAClC,aAAa,EAAE,UAAU,IAAI,CAAC,MAAM,EAAE;iBACvC;gBACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;aAC3B,EACD,2BAAkB,CACnB,CAAmB,CAAC,CAAC;SACvB;QAAC,OAAO,GAAG,EAAE;YACZ,OAAO;gBACL,KAAK,EAAE,mBAAmB,MAAM,CAAC,GAAG,CAAC,EAAE;gBACvC,UAAU,EAAE;oBACV,KAAK,EAAE,CAAC;oBACR,MAAM,EAAE,CAAC;oBACT,UAAU,EAAE,CAAC;iBACd;aACF,CAAC;SACH;QACD,gBAAM,CAAC,KAAK,CAAC,uCAAuC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAE5E,IAAI;YACF,MAAM,SAAS,GAAG,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,SAAS,CAAC;YAC7C,IAAI,CAAC,SAAS,EAAE;gBACd,MAAM,IAAI,KAAK,CAAC,uBAAuB,CAAC,CAAC;aAC1C;YACD,MAAM,GAAG,GAAG;gBACV,SAAS;gBACT,UAAU,EAAE,MAAM;oBAChB,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,YAAY,EAAE;oBACrC,CAAC,CAAC;wBACE,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,YAAY;wBAC9B,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,aAAa;wBAChC,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,iBAAiB;qBACzC;aACN,CAAC;YACF,OAAO,GAAG,CAAC;SACZ;QAAC,OAAO,GAAG,EAAE;YACZ,OAAO;gBACL,KAAK,EAAE,uBAAuB,MAAM,CAAC,GAAG,CAAC,KAAK,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE;gBACpE,UAAU,EAAE;oBACV,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,YAAY;oBAChC,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,aAAa;oBAClC,UAAU,EAAE,IAAI,EAAE,KAAK,EAAE,iBAAiB;iBAC3C;aACF,CAAC;SACH;IACH,CAAC;CACF;AAhED,0DAgEC;AAED,MAAa,wBAAyB,SAAQ,qBAAqB;IASjE,YAAY,SAAiB,EAAE,MAAe;QAC5C,IAAI,CAAC,wBAAwB,CAAC,wBAAwB,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE;YAC1E,gBAAM,CAAC,IAAI,CAAC,0CAA0C,SAAS,EAAE,CAAC,CAAC;SACpE;QACD,KAAK,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;IAC3B,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,MAAc,EAAE,OAAiC;QAC7D,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE;YAChB,MAAM,IAAI,KAAK,CACb,kHAAkH,CACnH,CAAC;SACH;QAED,IAAI,IAAY,CAAC;QACjB,IAAI;YACF,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,WAAW;gBAC5B,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,WAAW,CAAC;gBACrC,CAAC,CAAC,CAAC,YAAY,EAAE,eAAe,CAAC,CAAC;SACrC;QAAC,OAAO,GAAG,EAAE;YACZ,MAAM,IAAI,KAAK,CAAC,2CAA2C,GAAG,EAAE,CAAC,CAAC;SACnE;QACD,MAAM,IAAI,GAAG;YACX,KAAK,EAAE,IAAI,CAAC,SAAS;YACrB,MAAM;YACN,UAAU,EAAE,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,MAAM,CAAC;YAC7D,WAAW,EAAE,OAAO,EAAE,WAAW,IAAI,UAAU,CAAC,OAAO,CAAC,GAAG,CAAC,kBAAkB,IAAI,GAAG,CAAC;YACtF,IAAI;SACL,CAAC;QACF,gBAAM,CAAC,KAAK,CAAC,uBAAuB,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC5D,IAAI,IAAI,EACN,MAAM,GAAG,KAAK,CAAC;QACjB,IAAI;YACF,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,IAAA,0BAAkB,EAC3C,WAAW,IAAI,CAAC,OAAO,iBAAiB,EACxC;gBACE,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE;oBACP,cAAc,EAAE,kBAAkB;oBAClC,aAAa,EAAE,UAAU,IAAI,CAAC,MAAM,EAAE;iBACvC;gBACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;aAC3B,EACD,2BAAkB,CACnB,CAAmB,CAAC,CAAC;SACvB;QAAC,OAAO,GAAG,EAAE;YACZ,OAAO;gBACL,KAAK,EAAE,mBAAmB,MAAM,CAAC,GAAG,CAAC,EAAE;aACxC,CAAC;SACH;QACD,gBAAM,CAAC,KAAK,CAAC,0BAA0B,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC/D,IAAI;YACF,OAAO;gBACL,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI;gBAC5B,UAAU,EAAE,MAAM;oBAChB,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,YAAY,EAAE;oBACrC,CAAC,CAAC;wBACE,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,YAAY;wBAC9B,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,aAAa;wBAChC,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,iBAAiB;qBACzC;aACN,CAAC;SACH;QAAC,OAAO,GAAG,EAAE;YACZ,OAAO;gBACL,KAAK,EAAE,uBAAuB,MAAM,CAAC,GAAG,CAAC,KAAK,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE;aACrE,CAAC;SACH;IACH,CAAC;;AA3EM,iDAAwB,GAAG;IAChC,kBAAkB;IAClB,kBAAkB;IAClB,gBAAgB;IAChB,kBAAkB;IAClB,cAAc;CACf,CAAC;AAPS,4DAAwB;AA+ErC,MAAa,4BAA6B,SAAQ,qBAAqB;IAUrE,YAAY,SAAiB,EAAE,MAAe;QAC5C,IAAI,CAAC,4BAA4B,CAAC,kBAAkB,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE;YACxE,gBAAM,CAAC,IAAI,CAAC,oCAAoC,SAAS,EAAE,CAAC,CAAC;SAC9D;QACD,KAAK,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;IAC3B,CAAC;IAED,oDAAoD;IACpD,KAAK,CAAC,OAAO,CAAC,MAAc,EAAE,OAAiC;QAC7D,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE;YAChB,MAAM,IAAI,KAAK,CACb,kHAAkH,CACnH,CAAC;SACH;QAED,IAAI,QAA6C,CAAC;QAClD,IAAI;YACF,qEAAqE;YACrE,yCAAyC;YACzC,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;SAC/B;QAAC,OAAO,GAAG,EAAE;YACZ,QAAQ,GAAG,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;SAChD;QACD,MAAM,IAAI,GAAG;YACX,KAAK,EAAE,IAAI,CAAC,SAAS;YACrB,QAAQ,EAAE,QAAQ;YAClB,UAAU,EAAE,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,MAAM,CAAC;YAC7D,WAAW,EAAE,OAAO,EAAE,WAAW,IAAI,UAAU,CAAC,OAAO,CAAC,GAAG,CAAC,kBAAkB,IAAI,GAAG,CAAC;SACvF,CAAC;QACF,gBAAM,CAAC,KAAK,CAAC,uBAAuB,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAE5D,IAAI,IAAI,EACN,MAAM,GAAG,KAAK,CAAC;QACjB,IAAI;YACF,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,IAAA,0BAAkB,EAC3C,WAAW,IAAI,CAAC,OAAO,sBAAsB,EAC7C;gBACE,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE;oBACP,cAAc,EAAE,kBAAkB;oBAClC,aAAa,EAAE,UAAU,IAAI,CAAC,MAAM,EAAE;iBACvC;gBACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;aAC3B,EACD,2BAAkB,CACnB,CAAmB,CAAC,CAAC;SACvB;QAAC,OAAO,GAAG,EAAE;YACZ,OAAO;gBACL,KAAK,EAAE,mBAAmB,MAAM,CAAC,GAAG,CAAC,EAAE;aACxC,CAAC;SACH;QAED,gBAAM,CAAC,KAAK,CAAC,0BAA0B,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC/D,IAAI;YACF,OAAO;gBACL,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO;gBACvC,UAAU,EAAE,MAAM;oBAChB,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,YAAY,EAAE;oBACrC,CAAC,CAAC;wBACE,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,YAAY;wBAC9B,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,aAAa;wBAChC,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,iBAAiB;qBACzC;aACN,CAAC;SACH;QAAC,OAAO,GAAG,EAAE;YACZ,OAAO;gBACL,KAAK,EAAE,uBAAuB,MAAM,CAAC,GAAG,CAAC,KAAK,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE;aACrE,CAAC;SACH;IACH,CAAC;;AA9EM,+CAAkB,GAAG;IAC1B,OAAO;IACP,YAAY;IACZ,WAAW;IACX,gBAAgB;IAChB,eAAe;IACf,oBAAoB;CACrB,CAAC;AARS,oEAA4B;AAkF5B,QAAA,wBAAwB,GAAG,IAAI,uBAAuB,CAAC,wBAAwB,CAAC,CAAC;AACjF,QAAA,sBAAsB,GAAG,IAAI,4BAA4B,CAAC,OAAO,CAAC,CAAC;AACnE,QAAA,0BAA0B,GAAG,IAAI,4BAA4B,CAAC,OAAO,CAAC,CAAC"}
|
package/dist/providers.d.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { ApiProvider } from './types
|
|
2
|
-
import { OpenAiCompletionProvider, OpenAiChatCompletionProvider } from './providers/openai
|
|
3
|
-
import { LocalAiCompletionProvider, LocalAiChatProvider } from './providers/localai
|
|
1
|
+
import { ApiProvider } from './types';
|
|
2
|
+
import { OpenAiCompletionProvider, OpenAiChatCompletionProvider } from './providers/openai';
|
|
3
|
+
import { LocalAiCompletionProvider, LocalAiChatProvider } from './providers/localai';
|
|
4
4
|
export declare function loadApiProviders(providerPaths: string | string[]): Promise<ApiProvider[]>;
|
|
5
5
|
export declare function loadApiProvider(providerPath: string): Promise<ApiProvider>;
|
|
6
6
|
declare const _default: {
|
package/dist/providers.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"providers.d.ts","sourceRoot":"","sources":["../src/providers.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,WAAW,EAAE,MAAM,
|
|
1
|
+
{"version":3,"file":"providers.d.ts","sourceRoot":"","sources":["../src/providers.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AAEtC,OAAO,EAAE,wBAAwB,EAAE,4BAA4B,EAAE,MAAM,oBAAoB,CAAC;AAC5F,OAAO,EAAE,yBAAyB,EAAE,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;AAErF,wBAAsB,gBAAgB,CAAC,aAAa,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC,CAO/F;AAED,wBAAsB,eAAe,CAAC,YAAY,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC,CAuChF;;;;;;;;AAED,wBAME"}
|
package/dist/providers.js
CHANGED
|
@@ -28,8 +28,8 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
28
28
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
29
29
|
exports.loadApiProvider = exports.loadApiProviders = void 0;
|
|
30
30
|
const node_path_1 = __importDefault(require("node:path"));
|
|
31
|
-
const
|
|
32
|
-
const
|
|
31
|
+
const openai_1 = require("./providers/openai");
|
|
32
|
+
const localai_1 = require("./providers/localai");
|
|
33
33
|
async function loadApiProviders(providerPaths) {
|
|
34
34
|
if (typeof providerPaths === 'string') {
|
|
35
35
|
return [await loadApiProvider(providerPaths)];
|
|
@@ -47,16 +47,16 @@ async function loadApiProvider(providerPath) {
|
|
|
47
47
|
const modelType = options[1];
|
|
48
48
|
const modelName = options[2];
|
|
49
49
|
if (modelType === 'chat') {
|
|
50
|
-
return new
|
|
50
|
+
return new openai_1.OpenAiChatCompletionProvider(modelName || 'gpt-3.5-turbo');
|
|
51
51
|
}
|
|
52
52
|
else if (modelType === 'completion') {
|
|
53
|
-
return new
|
|
53
|
+
return new openai_1.OpenAiCompletionProvider(modelName || 'text-davinci-003');
|
|
54
54
|
}
|
|
55
|
-
else if (
|
|
56
|
-
return new
|
|
55
|
+
else if (openai_1.OpenAiChatCompletionProvider.OPENAI_CHAT_MODELS.includes(modelType)) {
|
|
56
|
+
return new openai_1.OpenAiChatCompletionProvider(modelType);
|
|
57
57
|
}
|
|
58
|
-
else if (
|
|
59
|
-
return new
|
|
58
|
+
else if (openai_1.OpenAiCompletionProvider.OPENAI_COMPLETION_MODELS.includes(modelType)) {
|
|
59
|
+
return new openai_1.OpenAiCompletionProvider(modelType);
|
|
60
60
|
}
|
|
61
61
|
else {
|
|
62
62
|
throw new Error(`Unknown OpenAI model type: ${modelType}. Use one of the following providers: openai:chat:<model name>, openai:completion:<model name>`);
|
|
@@ -67,13 +67,13 @@ async function loadApiProvider(providerPath) {
|
|
|
67
67
|
const modelType = options[1];
|
|
68
68
|
const modelName = options[2];
|
|
69
69
|
if (modelType === 'chat') {
|
|
70
|
-
return new
|
|
70
|
+
return new localai_1.LocalAiChatProvider(modelName);
|
|
71
71
|
}
|
|
72
72
|
else if (modelType === 'completion') {
|
|
73
|
-
return new
|
|
73
|
+
return new localai_1.LocalAiCompletionProvider(modelName);
|
|
74
74
|
}
|
|
75
75
|
else {
|
|
76
|
-
return new
|
|
76
|
+
return new localai_1.LocalAiChatProvider(modelType);
|
|
77
77
|
}
|
|
78
78
|
}
|
|
79
79
|
// Load custom module
|
|
@@ -82,10 +82,10 @@ async function loadApiProvider(providerPath) {
|
|
|
82
82
|
}
|
|
83
83
|
exports.loadApiProvider = loadApiProvider;
|
|
84
84
|
exports.default = {
|
|
85
|
-
OpenAiCompletionProvider:
|
|
86
|
-
OpenAiChatCompletionProvider:
|
|
87
|
-
LocalAiCompletionProvider:
|
|
88
|
-
LocalAiChatProvider:
|
|
85
|
+
OpenAiCompletionProvider: openai_1.OpenAiCompletionProvider,
|
|
86
|
+
OpenAiChatCompletionProvider: openai_1.OpenAiChatCompletionProvider,
|
|
87
|
+
LocalAiCompletionProvider: localai_1.LocalAiCompletionProvider,
|
|
88
|
+
LocalAiChatProvider: localai_1.LocalAiChatProvider,
|
|
89
89
|
loadApiProvider,
|
|
90
90
|
};
|
|
91
91
|
//# sourceMappingURL=providers.js.map
|
package/dist/providers.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"providers.js","sourceRoot":"","sources":["../src/providers.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,0DAA6B;AAI7B
|
|
1
|
+
{"version":3,"file":"providers.js","sourceRoot":"","sources":["../src/providers.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,0DAA6B;AAI7B,+CAA4F;AAC5F,iDAAqF;AAE9E,KAAK,UAAU,gBAAgB,CAAC,aAAgC;IACrE,IAAI,OAAO,aAAa,KAAK,QAAQ,EAAE;QACrC,OAAO,CAAC,MAAM,eAAe,CAAC,aAAa,CAAC,CAAC,CAAC;KAC/C;SAAM,IAAI,KAAK,CAAC,OAAO,CAAC,aAAa,CAAC,EAAE;QACvC,OAAO,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,eAAe,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;KAChF;IACD,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC;AAC5C,CAAC;AAPD,4CAOC;AAEM,KAAK,UAAU,eAAe,CAAC,YAAoB;IACxD,IAAI,YAAY,EAAE,UAAU,CAAC,SAAS,CAAC,EAAE;QACvC,qBAAqB;QACrB,MAAM,OAAO,GAAG,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACxC,MAAM,SAAS,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;QAC7B,MAAM,SAAS,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;QAE7B,IAAI,SAAS,KAAK,MAAM,EAAE;YACxB,OAAO,IAAI,qCAA4B,CAAC,SAAS,IAAI,eAAe,CAAC,CAAC;SACvE;aAAM,IAAI,SAAS,KAAK,YAAY,EAAE;YACrC,OAAO,IAAI,iCAAwB,CAAC,SAAS,IAAI,kBAAkB,CAAC,CAAC;SACtE;aAAM,IAAI,qCAA4B,CAAC,kBAAkB,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE;YAC9E,OAAO,IAAI,qCAA4B,CAAC,SAAS,CAAC,CAAC;SACpD;aAAM,IAAI,iCAAwB,CAAC,wBAAwB,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE;YAChF,OAAO,IAAI,iCAAwB,CAAC,SAAS,CAAC,CAAC;SAChD;aAAM;YACL,MAAM,IAAI,KAAK,CACb,8BAA8B,SAAS,gGAAgG,CACxI,CAAC;SACH;KACF;IAED,IAAI,YAAY,EAAE,UAAU,CAAC,UAAU,CAAC,EAAE;QACxC,MAAM,OAAO,GAAG,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACxC,MAAM,SAAS,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;QAC7B,MAAM,SAAS,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;QAE7B,IAAI,SAAS,KAAK,MAAM,EAAE;YACxB,OAAO,IAAI,6BAAmB,CAAC,SAAS,CAAC,CAAC;SAC3C;aAAM,IAAI,SAAS,KAAK,YAAY,EAAE;YACrC,OAAO,IAAI,mCAAyB,CAAC,SAAS,CAAC,CAAC;SACjD;aAAM;YACL,OAAO,IAAI,6BAAmB,CAAC,SAAS,CAAC,CAAC;SAC3C;KACF;IAED,qBAAqB;IACrB,MAAM,iBAAiB,GAAG,CAAC,yBAAa,mBAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,YAAY,CAAC,uCAAC,CAAC,CAAC,OAAO,CAAC;IACzF,OAAO,IAAI,iBAAiB,EAAE,CAAC;AACjC,CAAC;AAvCD,0CAuCC;AAED,kBAAe;IACb,wBAAwB,EAAxB,iCAAwB;IACxB,4BAA4B,EAA5B,qCAA4B;IAC5B,yBAAyB,EAAzB,mCAAyB;IACzB,mBAAmB,EAAnB,6BAAmB;IACnB,eAAe;CAChB,CAAC"}
|
package/dist/types.d.ts
CHANGED
|
@@ -90,9 +90,12 @@ export interface GradingResult {
|
|
|
90
90
|
reason: string;
|
|
91
91
|
tokensUsed?: TokenUsage;
|
|
92
92
|
}
|
|
93
|
+
type BaseAssertionTypes = 'equals' | 'contains' | 'icontains' | 'contains-all' | 'contains-any' | 'regex' | 'is-json' | 'contains-json' | 'javascript' | 'similar' | 'llm-rubric' | 'webhook' | 'rouge-n' | 'rouge-s' | 'rouge-l';
|
|
94
|
+
type NotPrefixed<T extends string> = `not-${T}`;
|
|
95
|
+
export type AssertionType = BaseAssertionTypes | NotPrefixed<BaseAssertionTypes>;
|
|
93
96
|
export interface Assertion {
|
|
94
|
-
type:
|
|
95
|
-
value?: string;
|
|
97
|
+
type: AssertionType;
|
|
98
|
+
value?: string | string[];
|
|
96
99
|
threshold?: number;
|
|
97
100
|
provider?: ApiProvider;
|
|
98
101
|
}
|
|
@@ -108,7 +111,7 @@ export interface AtomicTestCase extends TestCase {
|
|
|
108
111
|
export interface TestSuite {
|
|
109
112
|
description?: string;
|
|
110
113
|
providers: ApiProvider[];
|
|
111
|
-
prompts:
|
|
114
|
+
prompts: Prompt[];
|
|
112
115
|
tests?: TestCase[];
|
|
113
116
|
defaultTest?: Partial<TestCase>;
|
|
114
117
|
}
|
|
@@ -124,4 +127,5 @@ export type UnifiedConfig = TestSuiteConfig & {
|
|
|
124
127
|
evaluateOptions: EvaluateOptions;
|
|
125
128
|
commandLineOptions: Partial<CommandLineOptions>;
|
|
126
129
|
};
|
|
130
|
+
export {};
|
|
127
131
|
//# sourceMappingURL=types.d.ts.map
|
package/dist/types.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,kBAAkB;IAEjC,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,MAAM,EAAE,MAAM,CAAC;IAGf,cAAc,EAAE,MAAM,CAAC;IAGvB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,KAAK,CAAC,EAAE,OAAO,CAAC;IAEhB,mBAAmB,CAAC,EAAE,OAAO,CAAC;IAC9B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED,MAAM,WAAW,WAAW;IAC1B,EAAE,EAAE,MAAM,MAAM,CAAC;IACjB,OAAO,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,OAAO,CAAC,gBAAgB,CAAC,CAAC;CACxD;AAED,MAAM,WAAW,UAAU;IACzB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,gBAAgB;IAC/B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,UAAU,CAAC,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC;CAClC;AAED,MAAM,WAAW,yBAAyB;IACxC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB,UAAU,CAAC,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC;CAClC;AAED,MAAM,WAAW,MAAM;IACrB,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAC;CACvB;AAED,MAAM,MAAM,UAAU,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;AAEhD,MAAM,WAAW,aAAa;IAC5B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,MAAM,GAAG,WAAW,CAAC;CACjC;AAED,MAAM,WAAW,YAAY;IAC3B,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,eAAe;IAC9B,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,mBAAmB,CAAC,EAAE,OAAO,CAAC;CAC/B;AAED,MAAM,WAAW,MAAM;IACrB,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC7B,QAAQ,CAAC,EAAE,gBAAgB,CAAC;IAC5B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,OAAO,CAAC;CAClB;AAED,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE;QACJ,OAAO,EAAE,MAAM,EAAE,CAAC;QAClB,IAAI,EAAE,MAAM,EAAE,CAAC;KAChB,CAAC;IAEF,IAAI,EAAE;QACJ,OAAO,EAAE,MAAM,EAAE,CAAC;QAClB,IAAI,EAAE,MAAM,EAAE,CAAC;KAChB,EAAE,CAAC;CACL;AAED,MAAM,WAAW,aAAa;IAC5B,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,QAAQ,CAAC,UAAU,CAAC,CAAC;CAClC;AAED,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,cAAc,EAAE,CAAC;IAC1B,KAAK,EAAE,aAAa,CAAC;IACrB,KAAK,EAAE,aAAa,CAAC;CACtB;AAED,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,OAAO,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,CAAC,EAAE,UAAU,CAAC;CACzB;
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,kBAAkB;IAEjC,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,MAAM,EAAE,MAAM,CAAC;IAGf,cAAc,EAAE,MAAM,CAAC;IAGvB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,KAAK,CAAC,EAAE,OAAO,CAAC;IAEhB,mBAAmB,CAAC,EAAE,OAAO,CAAC;IAC9B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED,MAAM,WAAW,WAAW;IAC1B,EAAE,EAAE,MAAM,MAAM,CAAC;IACjB,OAAO,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,OAAO,CAAC,gBAAgB,CAAC,CAAC;CACxD;AAED,MAAM,WAAW,UAAU;IACzB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,gBAAgB;IAC/B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,UAAU,CAAC,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC;CAClC;AAED,MAAM,WAAW,yBAAyB;IACxC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB,UAAU,CAAC,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC;CAClC;AAED,MAAM,WAAW,MAAM;IACrB,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAC;CACvB;AAED,MAAM,MAAM,UAAU,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;AAEhD,MAAM,WAAW,aAAa;IAC5B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,MAAM,GAAG,WAAW,CAAC;CACjC;AAED,MAAM,WAAW,YAAY;IAC3B,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,eAAe;IAC9B,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,mBAAmB,CAAC,EAAE,OAAO,CAAC;CAC/B;AAED,MAAM,WAAW,MAAM;IACrB,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC7B,QAAQ,CAAC,EAAE,gBAAgB,CAAC;IAC5B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,OAAO,CAAC;CAClB;AAED,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE;QACJ,OAAO,EAAE,MAAM,EAAE,CAAC;QAClB,IAAI,EAAE,MAAM,EAAE,CAAC;KAChB,CAAC;IAEF,IAAI,EAAE;QACJ,OAAO,EAAE,MAAM,EAAE,CAAC;QAClB,IAAI,EAAE,MAAM,EAAE,CAAC;KAChB,EAAE,CAAC;CACL;AAED,MAAM,WAAW,aAAa;IAC5B,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,QAAQ,CAAC,UAAU,CAAC,CAAC;CAClC;AAED,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,cAAc,EAAE,CAAC;IAC1B,KAAK,EAAE,aAAa,CAAC;IACrB,KAAK,EAAE,aAAa,CAAC;CACtB;AAED,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,OAAO,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,CAAC,EAAE,UAAU,CAAC;CACzB;AAED,KAAK,kBAAkB,GACnB,QAAQ,GACR,UAAU,GACV,WAAW,GACX,cAAc,GACd,cAAc,GACd,OAAO,GACP,SAAS,GACT,eAAe,GACf,YAAY,GACZ,SAAS,GACT,YAAY,GACZ,SAAS,GACT,SAAS,GACT,SAAS,GACT,SAAS,CAAC;AAEd,KAAK,WAAW,CAAC,CAAC,SAAS,MAAM,IAAI,OAAO,CAAC,EAAE,CAAC;AAEhD,MAAM,MAAM,aAAa,GAAG,kBAAkB,GAAG,WAAW,CAAC,kBAAkB,CAAC,CAAC;AAGjF,MAAM,WAAW,SAAS;IAExB,IAAI,EAAE,aAAa,CAAC;IAGpB,KAAK,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IAG1B,SAAS,CAAC,EAAE,MAAM,CAAC;IAGnB,QAAQ,CAAC,EAAE,WAAW,CAAC;CACxB;AAGD,MAAM,WAAW,QAAQ;IAEvB,WAAW,CAAC,EAAE,MAAM,CAAC;IAGrB,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC,CAAC;IAGzC,MAAM,CAAC,EAAE,SAAS,EAAE,CAAC;IAGrB,OAAO,CAAC,EAAE,YAAY,GAAG,aAAa,CAAC;CACxC;AAGD,MAAM,WAAW,cAAe,SAAQ,QAAQ;IAC9C,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAC/B;AAGD,MAAM,WAAW,SAAS;IAExB,WAAW,CAAC,EAAE,MAAM,CAAC;IAGrB,SAAS,EAAE,WAAW,EAAE,CAAC;IAGzB,OAAO,EAAE,MAAM,EAAE,CAAC;IAGlB,KAAK,CAAC,EAAE,QAAQ,EAAE,CAAC;IAGnB,WAAW,CAAC,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;CACjC;AAGD,MAAM,WAAW,eAAe;IAE9B,WAAW,CAAC,EAAE,MAAM,CAAC;IAGrB,SAAS,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IAG7B,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IAG3B,KAAK,EAAE,MAAM,GAAG,QAAQ,EAAE,CAAC;IAG3B,WAAW,CAAC,EAAE,IAAI,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAC;IAG5C,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,MAAM,aAAa,GAAG,eAAe,GAAG;IAC5C,eAAe,EAAE,eAAe,CAAC;IACjC,kBAAkB,EAAE,OAAO,CAAC,kBAAkB,CAAC,CAAC;CACjD,CAAC"}
|
package/dist/util.d.ts
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import type { RequestInfo, RequestInit, Response } from 'node-fetch';
|
|
2
|
-
import type { CsvRow, EvaluateSummary, UnifiedConfig, TestCase } from './types
|
|
3
|
-
export declare function maybeReadConfig(configPath: string): UnifiedConfig | undefined
|
|
4
|
-
export declare function readConfig(configPath: string): UnifiedConfig
|
|
5
|
-
export declare function readPrompts(
|
|
2
|
+
import type { CsvRow, EvaluateSummary, UnifiedConfig, TestCase, Prompt } from './types';
|
|
3
|
+
export declare function maybeReadConfig(configPath: string): Promise<UnifiedConfig | undefined>;
|
|
4
|
+
export declare function readConfig(configPath: string): Promise<UnifiedConfig>;
|
|
5
|
+
export declare function readPrompts(promptPathOrGlobs: string | string[] | Record<string, string>): Prompt[];
|
|
6
6
|
export declare function fetchCsvFromGoogleSheet(url: string): Promise<string>;
|
|
7
7
|
export declare function readVars(varsPath: string): Promise<CsvRow[]>;
|
|
8
8
|
export declare function readTests(tests: string | TestCase[] | undefined): Promise<TestCase[]>;
|
package/dist/util.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"util.d.ts","sourceRoot":"","sources":["../src/util.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"util.d.ts","sourceRoot":"","sources":["../src/util.ts"],"names":[],"mappings":"AAgBA,OAAO,KAAK,EAAE,WAAW,EAAE,WAAW,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAErE,OAAO,KAAK,EAAa,MAAM,EAAE,eAAe,EAAE,aAAa,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AAanG,wBAAsB,eAAe,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,GAAG,SAAS,CAAC,CAK5F;AAED,wBAAsB,UAAU,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,CAAC,CAgB3E;AAQD,wBAAgB,WAAW,CACzB,iBAAiB,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAC5D,MAAM,EAAE,CA8CV;AAED,wBAAsB,uBAAuB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAQ1E;AAED,wBAAsB,QAAQ,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAkBlE;AAED,wBAAsB,SAAS,CAAC,KAAK,EAAE,MAAM,GAAG,QAAQ,EAAE,GAAG,SAAS,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC,CA6B3F;AAED,wBAAgB,WAAW,CAAC,UAAU,EAAE,MAAM,EAAE,OAAO,EAAE,eAAe,GAAG,IAAI,CA2B9E;AAED,wBAAgB,gBAAgB,CAC9B,GAAG,EAAE,WAAW,EAChB,OAAO,yBAAkB,EACzB,OAAO,EAAE,MAAM,GACd,OAAO,CAAC,QAAQ,CAAC,CAwBnB;AAED,wBAAgB,sBAAsB,IAAI,MAAM,CAE/C;AAED,wBAAgB,oBAAoB,IAAI,MAAM,CAE7C;AAED,wBAAgB,kBAAkB,CAAC,OAAO,EAAE,eAAe,QAQ1D;AAED,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,UAQ9D;AAED,wBAAgB,kBAAkB,CAAC,GAAG,EAAE,MAAM,GAAG,QAAQ,CAexD"}
|
package/dist/util.js
CHANGED
|
@@ -30,6 +30,7 @@ exports.testCaseFromCsvRow = exports.cosineSimilarity = exports.writeLatestResul
|
|
|
30
30
|
const fs = __importStar(require("fs"));
|
|
31
31
|
const path = __importStar(require("node:path"));
|
|
32
32
|
const os = __importStar(require("node:os"));
|
|
33
|
+
const json_schema_ref_parser_1 = __importDefault(require("@apidevtools/json-schema-ref-parser"));
|
|
33
34
|
const node_fetch_1 = __importDefault(require("node-fetch"));
|
|
34
35
|
const js_yaml_1 = __importDefault(require("js-yaml"));
|
|
35
36
|
const nunjucks_1 = __importDefault(require("nunjucks"));
|
|
@@ -37,9 +38,9 @@ const glob_1 = require("glob");
|
|
|
37
38
|
const path_1 = require("path");
|
|
38
39
|
const sync_1 = require("csv-parse/sync");
|
|
39
40
|
const sync_2 = require("csv-stringify/sync");
|
|
40
|
-
const
|
|
41
|
-
const
|
|
42
|
-
const
|
|
41
|
+
const logger_1 = __importDefault(require("./logger"));
|
|
42
|
+
const esm_1 = require("./esm");
|
|
43
|
+
const assertions_1 = require("./assertions");
|
|
43
44
|
const PROMPT_DELIMITER = '---';
|
|
44
45
|
function parseJson(json) {
|
|
45
46
|
try {
|
|
@@ -49,14 +50,14 @@ function parseJson(json) {
|
|
|
49
50
|
return undefined;
|
|
50
51
|
}
|
|
51
52
|
}
|
|
52
|
-
function maybeReadConfig(configPath) {
|
|
53
|
+
async function maybeReadConfig(configPath) {
|
|
53
54
|
if (!fs.existsSync(configPath)) {
|
|
54
55
|
return undefined;
|
|
55
56
|
}
|
|
56
57
|
return readConfig(configPath);
|
|
57
58
|
}
|
|
58
59
|
exports.maybeReadConfig = maybeReadConfig;
|
|
59
|
-
function readConfig(configPath) {
|
|
60
|
+
async function readConfig(configPath) {
|
|
60
61
|
const ext = path.parse(configPath).ext;
|
|
61
62
|
switch (ext) {
|
|
62
63
|
case '.json':
|
|
@@ -65,34 +66,64 @@ function readConfig(configPath) {
|
|
|
65
66
|
case '.js':
|
|
66
67
|
return require(configPath);
|
|
67
68
|
case '.yaml':
|
|
68
|
-
|
|
69
|
+
case '.yml':
|
|
70
|
+
let ret = js_yaml_1.default.load(fs.readFileSync(configPath, 'utf-8'));
|
|
71
|
+
ret = (await json_schema_ref_parser_1.default.dereference(ret));
|
|
72
|
+
return ret;
|
|
69
73
|
default:
|
|
70
74
|
throw new Error(`Unsupported configuration file format: ${ext}`);
|
|
71
75
|
}
|
|
72
76
|
}
|
|
73
77
|
exports.readConfig = readConfig;
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
+
var PromptInputType;
|
|
79
|
+
(function (PromptInputType) {
|
|
80
|
+
PromptInputType[PromptInputType["STRING"] = 1] = "STRING";
|
|
81
|
+
PromptInputType[PromptInputType["ARRAY"] = 2] = "ARRAY";
|
|
82
|
+
PromptInputType[PromptInputType["NAMED"] = 3] = "NAMED";
|
|
83
|
+
})(PromptInputType || (PromptInputType = {}));
|
|
84
|
+
function readPrompts(promptPathOrGlobs) {
|
|
85
|
+
let promptPaths = [];
|
|
78
86
|
let promptContents = [];
|
|
87
|
+
let inputType;
|
|
88
|
+
if (typeof promptPathOrGlobs === 'string') {
|
|
89
|
+
promptPaths = [promptPathOrGlobs];
|
|
90
|
+
inputType = PromptInputType.STRING;
|
|
91
|
+
}
|
|
92
|
+
else if (Array.isArray(promptPathOrGlobs)) {
|
|
93
|
+
promptPaths = promptPathOrGlobs.flatMap((pathOrGlob) => (0, glob_1.globSync)(pathOrGlob));
|
|
94
|
+
inputType = PromptInputType.ARRAY;
|
|
95
|
+
}
|
|
96
|
+
else if (typeof promptPathOrGlobs === 'object') {
|
|
97
|
+
promptPaths = Object.keys(promptPathOrGlobs);
|
|
98
|
+
inputType = PromptInputType.NAMED;
|
|
99
|
+
}
|
|
79
100
|
for (const promptPath of promptPaths) {
|
|
80
101
|
const stat = fs.statSync(promptPath);
|
|
81
102
|
if (stat.isDirectory()) {
|
|
82
103
|
const filesInDirectory = fs.readdirSync(promptPath);
|
|
83
104
|
const fileContents = filesInDirectory.map((fileName) => fs.readFileSync(path.join(promptPath, fileName), 'utf-8'));
|
|
84
|
-
promptContents.push(...fileContents);
|
|
105
|
+
promptContents.push(...fileContents.map((content) => ({ raw: content, display: content })));
|
|
85
106
|
}
|
|
86
107
|
else {
|
|
87
108
|
const fileContent = fs.readFileSync(promptPath, 'utf-8');
|
|
88
|
-
|
|
109
|
+
let display;
|
|
110
|
+
if (inputType === PromptInputType.NAMED) {
|
|
111
|
+
display = promptPathOrGlobs[promptPath];
|
|
112
|
+
}
|
|
113
|
+
else {
|
|
114
|
+
display = fileContent.length > 200 ? promptPath : fileContent;
|
|
115
|
+
}
|
|
116
|
+
promptContents.push({ raw: fileContent, display });
|
|
89
117
|
}
|
|
90
118
|
}
|
|
91
|
-
if (promptContents.length === 1) {
|
|
92
|
-
|
|
119
|
+
if (promptContents.length === 1 && inputType !== PromptInputType.NAMED) {
|
|
120
|
+
const content = promptContents[0].raw;
|
|
121
|
+
promptContents = content
|
|
122
|
+
.split(PROMPT_DELIMITER)
|
|
123
|
+
.map((p) => ({ raw: p.trim(), display: p.trim() }));
|
|
93
124
|
}
|
|
94
125
|
if (promptContents.length === 0) {
|
|
95
|
-
throw new Error(`There are no prompts in ${
|
|
126
|
+
throw new Error(`There are no prompts in ${JSON.stringify(promptPathOrGlobs)}`);
|
|
96
127
|
}
|
|
97
128
|
return promptContents;
|
|
98
129
|
}
|
|
@@ -166,7 +197,7 @@ function writeOutput(outputPath, summary) {
|
|
|
166
197
|
fs.writeFileSync(outputPath, js_yaml_1.default.dump(summary));
|
|
167
198
|
}
|
|
168
199
|
else if (outputExtension === 'html') {
|
|
169
|
-
const template = fs.readFileSync(`${(0,
|
|
200
|
+
const template = fs.readFileSync(`${(0, esm_1.getDirectory)()}/tableOutput.html`, 'utf-8');
|
|
170
201
|
const table = [
|
|
171
202
|
[...summary.table.head.prompts, ...summary.table.head.vars],
|
|
172
203
|
...summary.table.body.map((row) => [...row.outputs, ...row.vars]),
|
|
@@ -223,7 +254,7 @@ function writeLatestResults(results) {
|
|
|
223
254
|
fs.writeFileSync(latestResultsPath, JSON.stringify(results, null, 2));
|
|
224
255
|
}
|
|
225
256
|
catch (err) {
|
|
226
|
-
|
|
257
|
+
logger_1.default.error(`Failed to write latest results to ${latestResultsPath}:\n${err}`);
|
|
227
258
|
}
|
|
228
259
|
}
|
|
229
260
|
exports.writeLatestResults = writeLatestResults;
|
|
@@ -242,7 +273,7 @@ function testCaseFromCsvRow(row) {
|
|
|
242
273
|
const asserts = [];
|
|
243
274
|
for (const [key, value] of Object.entries(row)) {
|
|
244
275
|
if (key === '__expected') {
|
|
245
|
-
asserts.push((0,
|
|
276
|
+
asserts.push((0, assertions_1.assertionFromString)(value));
|
|
246
277
|
}
|
|
247
278
|
else {
|
|
248
279
|
vars[key] = value;
|
package/dist/util.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"util.js","sourceRoot":"","sources":["../src/util.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,uCAAyB;AACzB,gDAAkC;AAClC,4CAA8B;AAE9B,4DAA+B;AAC/B,sDAA2B;AAC3B,wDAAgC;AAChC,+BAAgC;AAChC,+BAA0C;AAC1C,yCAAmD;AACnD,6CAA+C;AAE/C,
|
|
1
|
+
{"version":3,"file":"util.js","sourceRoot":"","sources":["../src/util.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,uCAAyB;AACzB,gDAAkC;AAClC,4CAA8B;AAE9B,iGAA6D;AAC7D,4DAA+B;AAC/B,sDAA2B;AAC3B,wDAAgC;AAChC,+BAAgC;AAChC,+BAA0C;AAC1C,yCAAmD;AACnD,6CAA+C;AAE/C,sDAA8B;AAC9B,+BAAqC;AAKrC,6CAAmD;AAEnD,MAAM,gBAAgB,GAAG,KAAK,CAAC;AAE/B,SAAS,SAAS,CAAC,IAAY;IAC7B,IAAI;QACF,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;KACzB;IAAC,OAAO,GAAG,EAAE;QACZ,OAAO,SAAS,CAAC;KAClB;AACH,CAAC;AAEM,KAAK,UAAU,eAAe,CAAC,UAAkB;IACtD,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE;QAC9B,OAAO,SAAS,CAAC;KAClB;IACD,OAAO,UAAU,CAAC,UAAU,CAAC,CAAC;AAChC,CAAC;AALD,0CAKC;AAEM,KAAK,UAAU,UAAU,CAAC,UAAkB;IACjD,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,GAAG,CAAC;IACvC,QAAQ,GAAG,EAAE;QACX,KAAK,OAAO;YACV,MAAM,OAAO,GAAG,EAAE,CAAC,YAAY,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;YACrD,OAAO,IAAI,CAAC,KAAK,CAAC,OAAO,CAAkB,CAAC;QAC9C,KAAK,KAAK;YACR,OAAO,OAAO,CAAC,UAAU,CAAkB,CAAC;QAC9C,KAAK,OAAO,CAAC;QACb,KAAK,MAAM;YACT,IAAI,GAAG,GAAG,iBAAI,CAAC,IAAI,CAAC,EAAE,CAAC,YAAY,CAAC,UAAU,EAAE,OAAO,CAAC,CAAkB,CAAC;YAC3E,GAAG,GAAG,CAAC,MAAM,gCAAU,CAAC,WAAW,CAAC,GAAG,CAAC,CAAkB,CAAC;YAC3D,OAAO,GAAG,CAAC;QACb;YACE,MAAM,IAAI,KAAK,CAAC,0CAA0C,GAAG,EAAE,CAAC,CAAC;KACpE;AACH,CAAC;AAhBD,gCAgBC;AAED,IAAK,eAIJ;AAJD,WAAK,eAAe;IAClB,yDAAU,CAAA;IACV,uDAAS,CAAA;IACT,uDAAS,CAAA;AACX,CAAC,EAJI,eAAe,KAAf,eAAe,QAInB;AAED,SAAgB,WAAW,CACzB,iBAA6D;IAE7D,IAAI,WAAW,GAAa,EAAE,CAAC;IAC/B,IAAI,cAAc,GAAa,EAAE,CAAC;IAElC,IAAI,SAAsC,CAAC;IAC3C,IAAI,OAAO,iBAAiB,KAAK,QAAQ,EAAE;QACzC,WAAW,GAAG,CAAC,iBAAiB,CAAC,CAAC;QAClC,SAAS,GAAG,eAAe,CAAC,MAAM,CAAC;KACpC;SAAM,IAAI,KAAK,CAAC,OAAO,CAAC,iBAAiB,CAAC,EAAE;QAC3C,WAAW,GAAG,iBAAiB,CAAC,OAAO,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,IAAA,eAAQ,EAAC,UAAU,CAAC,CAAC,CAAC;QAC9E,SAAS,GAAG,eAAe,CAAC,KAAK,CAAC;KACnC;SAAM,IAAI,OAAO,iBAAiB,KAAK,QAAQ,EAAE;QAChD,WAAW,GAAG,MAAM,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;QAC7C,SAAS,GAAG,eAAe,CAAC,KAAK,CAAC;KACnC;IAED,KAAK,MAAM,UAAU,IAAI,WAAW,EAAE;QACpC,MAAM,IAAI,GAAG,EAAE,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;QACrC,IAAI,IAAI,CAAC,WAAW,EAAE,EAAE;YACtB,MAAM,gBAAgB,GAAG,EAAE,CAAC,WAAW,CAAC,UAAU,CAAC,CAAC;YACpD,MAAM,YAAY,GAAG,gBAAgB,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CACrD,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,QAAQ,CAAC,EAAE,OAAO,CAAC,CAC1D,CAAC;YACF,cAAc,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC;SAC7F;aAAM;YACL,MAAM,WAAW,GAAG,EAAE,CAAC,YAAY,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;YACzD,IAAI,OAAO,CAAC;YACZ,IAAI,SAAS,KAAK,eAAe,CAAC,KAAK,EAAE;gBACvC,OAAO,GAAI,iBAA4C,CAAC,UAAU,CAAC,CAAC;aACrE;iBAAM;gBACL,OAAO,GAAG,WAAW,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,WAAW,CAAC;aAC/D;YACD,cAAc,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,WAAW,EAAE,OAAO,EAAE,CAAC,CAAC;SACpD;KACF;IAED,IAAI,cAAc,CAAC,MAAM,KAAK,CAAC,IAAI,SAAS,KAAK,eAAe,CAAC,KAAK,EAAE;QACtE,MAAM,OAAO,GAAG,cAAc,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;QACtC,cAAc,GAAG,OAAO;aACrB,KAAK,CAAC,gBAAgB,CAAC;aACvB,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,IAAI,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC;KACvD;IACD,IAAI,cAAc,CAAC,MAAM,KAAK,CAAC,EAAE;QAC/B,MAAM,IAAI,KAAK,CAAC,2BAA2B,IAAI,CAAC,SAAS,CAAC,iBAAiB,CAAC,EAAE,CAAC,CAAC;KACjF;IACD,OAAO,cAAc,CAAC;AACxB,CAAC;AAhDD,kCAgDC;AAEM,KAAK,UAAU,uBAAuB,CAAC,GAAW;IACvD,MAAM,MAAM,GAAG,GAAG,CAAC,OAAO,CAAC,WAAW,EAAE,oBAAoB,CAAC,CAAC;IAC9D,MAAM,QAAQ,GAAG,MAAM,IAAA,oBAAK,EAAC,MAAM,CAAC,CAAC;IACrC,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE;QAC3B,MAAM,IAAI,KAAK,CAAC,+CAA+C,GAAG,EAAE,CAAC,CAAC;KACvE;IACD,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;IACtC,OAAO,OAAO,CAAC;AACjB,CAAC;AARD,0DAQC;AAEM,KAAK,UAAU,QAAQ,CAAC,QAAgB;IAC7C,MAAM,aAAa,GAAG,IAAA,YAAS,EAAC,QAAQ,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IACvD,IAAI,IAAI,GAAa,EAAE,CAAC;IAExB,IAAI,aAAa,KAAK,KAAK,EAAE;QAC3B,IAAI,QAAQ,CAAC,UAAU,CAAC,uCAAuC,CAAC,EAAE;YAChE,MAAM,OAAO,GAAG,MAAM,uBAAuB,CAAC,QAAQ,CAAC,CAAC;YACxD,IAAI,GAAG,IAAA,YAAQ,EAAC,OAAO,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;SAC7C;aAAM;YACL,IAAI,GAAG,IAAA,YAAQ,EAAC,EAAE,CAAC,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;SACxE;KACF;SAAM,IAAI,aAAa,KAAK,MAAM,EAAE;QACnC,IAAI,GAAG,SAAS,CAAC,EAAE,CAAC,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAC;KACtD;SAAM,IAAI,aAAa,KAAK,MAAM,IAAI,aAAa,KAAK,KAAK,EAAE;QAC9D,IAAI,GAAG,iBAAI,CAAC,IAAI,CAAC,EAAE,CAAC,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAmB,CAAC;KACxE;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAlBD,4BAkBC;AAEM,KAAK,UAAU,SAAS,CAAC,KAAsC;IACpE,IAAI,CAAC,KAAK,EAAE;QACV,OAAO,EAAE,CAAC;KACX;IAED,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE;QAC7B,iCAAiC;QACjC,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,KAAK,CAAC,CAAC;QACnC,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE;YAC3B,MAAM,IAAI,GAAG,kBAAkB,CAAC,GAAG,CAAC,CAAC;YACrC,IAAI,CAAC,WAAW,GAAG,QAAQ,GAAG,GAAG,CAAC,EAAE,CAAC;YACrC,OAAO,IAAI,CAAC;QACd,CAAC,CAAC,CAAC;KACJ;IAED,wCAAwC;IACxC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE;QACxB,IAAI,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE;YAC9B,MAAM,IAAI,KAAK,CACb,uEAAuE,IAAI,CAAC,SAAS,CACnF,IAAI,EACJ,IAAI,EACJ,CAAC,CACF,EAAE,CACJ,CAAC;SACH;KACF;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AA7BD,8BA6BC;AAED,SAAgB,WAAW,CAAC,UAAkB,EAAE,OAAwB;IACtE,MAAM,eAAe,GAAG,UAAU,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,WAAW,EAAE,CAAC;IAEnE,IAAI,eAAe,KAAK,KAAK,IAAI,eAAe,KAAK,KAAK,EAAE;QAC1D,MAAM,SAAS,GAAG,IAAA,gBAAS,EAAC;YAC1B,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC;YAC3D,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,GAAG,GAAG,CAAC,OAAO,EAAE,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC;SAClE,CAAC,CAAC;QACH,EAAE,CAAC,aAAa,CAAC,UAAU,EAAE,SAAS,CAAC,CAAC;KACzC;SAAM,IAAI,eAAe,KAAK,MAAM,EAAE;QACrC,EAAE,CAAC,aAAa,CAAC,UAAU,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;KAChE;SAAM,IAAI,eAAe,KAAK,MAAM,IAAI,eAAe,KAAK,KAAK,EAAE;QAClE,EAAE,CAAC,aAAa,CAAC,UAAU,EAAE,iBAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;KAClD;SAAM,IAAI,eAAe,KAAK,MAAM,EAAE;QACrC,MAAM,QAAQ,GAAG,EAAE,CAAC,YAAY,CAAC,GAAG,IAAA,kBAAY,GAAE,mBAAmB,EAAE,OAAO,CAAC,CAAC;QAChF,MAAM,KAAK,GAAG;YACZ,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC;YAC3D,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,GAAG,GAAG,CAAC,OAAO,EAAE,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC;SAClE,CAAC;QACF,MAAM,UAAU,GAAG,kBAAQ,CAAC,YAAY,CAAC,QAAQ,EAAE;YACjD,KAAK;YACL,OAAO,EAAE,OAAO,CAAC,OAAO;SACzB,CAAC,CAAC;QACH,EAAE,CAAC,aAAa,CAAC,UAAU,EAAE,UAAU,CAAC,CAAC;KAC1C;SAAM;QACL,MAAM,IAAI,KAAK,CAAC,yDAAyD,CAAC,CAAC;KAC5E;AACH,CAAC;AA3BD,kCA2BC;AAED,SAAgB,gBAAgB,CAC9B,GAAgB,EAChB,UAAuB,EAAE,EACzB,OAAe;IAEf,OAAO,IAAI,OAAO,CAAC,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE;QAC3C,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;QACzC,MAAM,EAAE,MAAM,EAAE,GAAG,UAAU,CAAC;QAC9B,OAAO,CAAC,MAAM,GAAG,MAAM,CAAC;QAExB,MAAM,SAAS,GAAG,UAAU,CAAC,GAAG,EAAE;YAChC,UAAU,CAAC,KAAK,EAAE,CAAC;YACnB,MAAM,CAAC,IAAI,KAAK,CAAC,2BAA2B,OAAO,KAAK,CAAC,CAAC,CAAC;QAC7D,CAAC,EAAE,OAAO,CAAC,CAAC;QAEZ,IAAI;YACF,MAAM,QAAQ,GAAG,MAAM,IAAA,oBAAK,EAAC,GAAG,EAAE,OAAO,CAAC,CAAC;YAC3C,YAAY,CAAC,SAAS,CAAC,CAAC;YACxB,OAAO,CAAC,QAAQ,CAAC,CAAC;SACnB;QAAC,OAAO,KAAK,EAAE;YACd,IAAI,KAAK,YAAY,KAAK,IAAI,KAAK,CAAC,IAAI,KAAK,YAAY,EAAE;gBACzD,qDAAqD;aACtD;iBAAM;gBACL,YAAY,CAAC,SAAS,CAAC,CAAC;gBACxB,MAAM,CAAC,KAAK,CAAC,CAAC;aACf;SACF;IACH,CAAC,CAAC,CAAC;AACL,CAAC;AA5BD,4CA4BC;AAED,SAAgB,sBAAsB;IACpC,OAAO,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,EAAE,YAAY,CAAC,CAAC;AAC/C,CAAC;AAFD,wDAEC;AAED,SAAgB,oBAAoB;IAClC,OAAO,IAAI,CAAC,IAAI,CAAC,sBAAsB,EAAE,EAAE,QAAQ,EAAE,aAAa,CAAC,CAAC;AACtE,CAAC;AAFD,oDAEC;AAED,SAAgB,kBAAkB,CAAC,OAAwB;IACzD,MAAM,iBAAiB,GAAG,oBAAoB,EAAE,CAAC;IACjD,IAAI;QACF,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,iBAAiB,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACnE,EAAE,CAAC,aAAa,CAAC,iBAAiB,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;KACvE;IAAC,OAAO,GAAG,EAAE;QACZ,gBAAM,CAAC,KAAK,CAAC,qCAAqC,iBAAiB,MAAM,GAAG,EAAE,CAAC,CAAC;KACjF;AACH,CAAC;AARD,gDAQC;AAED,SAAgB,gBAAgB,CAAC,IAAc,EAAE,IAAc;IAC7D,IAAI,IAAI,CAAC,MAAM,KAAK,IAAI,CAAC,MAAM,EAAE;QAC/B,MAAM,IAAI,KAAK,CAAC,iCAAiC,CAAC,CAAC;KACpD;IACD,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC,GAAG,GAAG,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;IAC5E,MAAM,aAAa,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC,GAAG,GAAG,GAAG,GAAG,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC;IAC/E,MAAM,aAAa,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC,GAAG,GAAG,GAAG,GAAG,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC;IAC/E,OAAO,UAAU,GAAG,CAAC,aAAa,GAAG,aAAa,CAAC,CAAC;AACtD,CAAC;AARD,4CAQC;AAED,SAAgB,kBAAkB,CAAC,GAAW;IAC5C,MAAM,IAAI,GAA2B,EAAE,CAAC;IACxC,MAAM,OAAO,GAAgB,EAAE,CAAC;IAChC,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE;QAC9C,IAAI,GAAG,KAAK,YAAY,EAAE;YACxB,OAAO,CAAC,IAAI,CAAC,IAAA,gCAAmB,EAAC,KAAK,CAAC,CAAC,CAAC;SAC1C;aAAM;YACL,IAAI,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;SACnB;KACF;IAED,OAAO;QACL,IAAI;QACJ,MAAM,EAAE,OAAO;KAChB,CAAC;AACJ,CAAC;AAfD,gDAeC"}
|