vskill 0.2.68 → 0.2.70
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/eval/generate-all.js +2 -2
- package/dist/commands/eval/generate-all.js.map +1 -1
- package/dist/commands/eval/init.js +2 -2
- package/dist/commands/eval/init.js.map +1 -1
- package/dist/commands/eval/run.js +2 -2
- package/dist/commands/eval/run.js.map +1 -1
- package/dist/eval/__tests__/activation-tester.test.js +4 -1
- package/dist/eval/__tests__/activation-tester.test.js.map +1 -1
- package/dist/eval/__tests__/comparator.test.js +4 -1
- package/dist/eval/__tests__/comparator.test.js.map +1 -1
- package/dist/eval/__tests__/judge.test.js +4 -1
- package/dist/eval/__tests__/judge.test.js.map +1 -1
- package/dist/eval/__tests__/llm.test.js +7 -3
- package/dist/eval/__tests__/llm.test.js.map +1 -1
- package/dist/eval/activation-tester.js +1 -1
- package/dist/eval/activation-tester.js.map +1 -1
- package/dist/eval/benchmark.d.ts +2 -0
- package/dist/eval/benchmark.js.map +1 -1
- package/dist/eval/comparator.d.ts +4 -0
- package/dist/eval/comparator.js +14 -8
- package/dist/eval/comparator.js.map +1 -1
- package/dist/eval/judge.js +1 -1
- package/dist/eval/judge.js.map +1 -1
- package/dist/eval/llm.d.ts +7 -1
- package/dist/eval/llm.js +19 -3
- package/dist/eval/llm.js.map +1 -1
- package/dist/eval-server/api-routes.js +16 -3
- package/dist/eval-server/api-routes.js.map +1 -1
- package/dist/eval-ui/assets/index-fiTwcc3L.js +70 -0
- package/dist/eval-ui/index.html +1 -1
- package/package.json +1 -1
- package/dist/eval-ui/assets/index-sP1kJUeE.js +0 -70
|
@@ -34,8 +34,8 @@ export async function runEvalGenerateAll(root, force) {
|
|
|
34
34
|
try {
|
|
35
35
|
const skillContent = readFileSync(skillMdPath, "utf-8");
|
|
36
36
|
const prompt = buildEvalInitPrompt(skillContent);
|
|
37
|
-
const
|
|
38
|
-
const evalsFile = parseGeneratedEvals(
|
|
37
|
+
const genResult = await client.generate("You generate eval test cases for AI skills. Output only valid JSON in a code fence.", prompt);
|
|
38
|
+
const evalsFile = parseGeneratedEvals(genResult.text);
|
|
39
39
|
mkdirSync(join(skill.dir, "evals"), { recursive: true });
|
|
40
40
|
writeFileSync(evalsPath, JSON.stringify(evalsFile, null, 2), "utf-8");
|
|
41
41
|
generated++;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"generate-all.js","sourceRoot":"","sources":["../../../src/commands/eval/generate-all.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,uEAAuE;AACvE,8EAA8E;AAE9E,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAC7E,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,UAAU,EAAE,MAAM,6BAA6B,CAAC;AACzD,OAAO,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AACpD,OAAO,EAAE,mBAAmB,EAAE,mBAAmB,EAAE,MAAM,8BAA8B,CAAC;AACxF,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,EAAE,MAAM,uBAAuB,CAAC;AAEtE,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACtC,IAAY,EACZ,KAAc;IAEd,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,IAAI,CAAC,CAAC;IAEtC,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACxB,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,qBAAqB,GAAG,IAAI,CAAC,CAAC,CAAC;QAC/C,OAAO;IACT,CAAC;IAED,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;IACjC,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,MAAM,WAAW,GAAa,EAAE,CAAC;IAEjC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,OAAO,EAAE,YAAY,CAAC,CAAC;QAEzD,8CAA8C;QAC9C,IAAI,KAAK,CAAC,QAAQ,IAAI,CAAC,KAAK,EAAE,CAAC;YAC7B,OAAO,EAAE,CAAC;YACV,SAAS;QACX,CAAC;QAED,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,UAAU,CAAC,CAAC;QAChD,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC,EAAE,CAAC;YAC7B,MAAM,EAAE,CAAC;YACT,WAAW,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,MAAM,IAAI,KAAK,CAAC,KAAK,gBAAgB,CAAC,CAAC;YACjE,SAAS;QACX,CAAC;QAED,IAAI,CAAC;YACH,MAAM,YAAY,GAAG,YAAY,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC;YACxD,MAAM,MAAM,GAAG,mBAAmB,CAAC,YAAY,CAAC,CAAC;YAEjD,MAAM,
|
|
1
|
+
{"version":3,"file":"generate-all.js","sourceRoot":"","sources":["../../../src/commands/eval/generate-all.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,uEAAuE;AACvE,8EAA8E;AAE9E,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAC7E,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,UAAU,EAAE,MAAM,6BAA6B,CAAC;AACzD,OAAO,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AACpD,OAAO,EAAE,mBAAmB,EAAE,mBAAmB,EAAE,MAAM,8BAA8B,CAAC;AACxF,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,EAAE,MAAM,uBAAuB,CAAC;AAEtE,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACtC,IAAY,EACZ,KAAc;IAEd,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,IAAI,CAAC,CAAC;IAEtC,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACxB,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,qBAAqB,GAAG,IAAI,CAAC,CAAC,CAAC;QAC/C,OAAO;IACT,CAAC;IAED,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;IACjC,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,MAAM,WAAW,GAAa,EAAE,CAAC;IAEjC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,OAAO,EAAE,YAAY,CAAC,CAAC;QAEzD,8CAA8C;QAC9C,IAAI,KAAK,CAAC,QAAQ,IAAI,CAAC,KAAK,EAAE,CAAC;YAC7B,OAAO,EAAE,CAAC;YACV,SAAS;QACX,CAAC;QAED,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,UAAU,CAAC,CAAC;QAChD,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC,EAAE,CAAC;YAC7B,MAAM,EAAE,CAAC;YACT,WAAW,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,MAAM,IAAI,KAAK,CAAC,KAAK,gBAAgB,CAAC,CAAC;YACjE,SAAS;QACX,CAAC;QAED,IAAI,CAAC;YACH,MAAM,YAAY,GAAG,YAAY,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC;YACxD,MAAM,MAAM,GAAG,mBAAmB,CAAC,YAAY,CAAC,CAAC;YAEjD,MAAM,SAAS,GAAG,MAAM,MAAM,CAAC,QAAQ,CACrC,qFAAqF,EACrF,MAAM,CACP,CAAC;YAEF,MAAM,SAAS,GAAG,mBAAmB,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;YAEtD,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,OAAO,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YACzD,aAAa,CAAC,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;YAEtE,SAAS,EAAE,CAAC;YACZ,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,gBAAgB,KAAK,CAAC,MAAM,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;YAElE,iEAAiE;YACjE,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC;QAChD,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,EAAE,CAAC;YACT,WAAW,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,MAAM,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC;YACnD,OAAO,CAAC,KAAK,CACX,GAAG,CAAC,aAAa,KAAK,CAAC,MAAM,IAAI,KAAK,CAAC,KAAK,KAAK,CAAC;gBAChD,GAAG,CAAE,GAAa,CAAC,OAAO,CAAC,CAC9B,CAAC;QACJ,CAAC;IACH,CAAC;IAED,gBAAgB;IAChB,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,4BAA4B,CAAC,CAAC,CAAC;IAChD,OAAO,CAAC,GAAG,CAAC,cAAc,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;IAC3C,OAAO,CAAC,GAAG,CAAC,KAAK,KAAK,CAAC,cAAc,SAAS,EAAE,CAAC,EAAE,CAAC,CAAC;IACrD,OAAO,CAAC,GAAG,CAAC,KAAK,MAAM,CAAC,YAAY,OAAO,EAAE,CAAC,EAAE,CAAC,CAAC;IAClD,OAAO,CAAC,GAAG,CAAC,KAAK,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,WAAW,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC,WAAW,MAAM,EAAE,EAAE,CAAC,CAAC;IAEhF,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC3B,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,kBAAkB,CAAC,CAAC,CAAC;QACrC,KAAK,MAAM,IAAI,IAAI,WAAW,EAAE,CAAC;YAC/B,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC,CAAC;QAClC,CAAC;IACH,CAAC;AACH,CAAC"}
|
|
@@ -24,8 +24,8 @@ export async function runEvalInit(skillDir, force) {
|
|
|
24
24
|
const prompt = buildEvalInitPrompt(skillContent);
|
|
25
25
|
try {
|
|
26
26
|
const client = createLlmClient();
|
|
27
|
-
const
|
|
28
|
-
const evalsFile = parseGeneratedEvals(
|
|
27
|
+
const genResult = await client.generate("You generate eval test cases for AI skills. Output only valid JSON in a code fence.", prompt);
|
|
28
|
+
const evalsFile = parseGeneratedEvals(genResult.text);
|
|
29
29
|
mkdirSync(evalsDir, { recursive: true });
|
|
30
30
|
writeFileSync(evalsPath, JSON.stringify(evalsFile, null, 2), "utf-8");
|
|
31
31
|
console.log(green(`Created ${evalsPath}`));
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"init.js","sourceRoot":"","sources":["../../../src/commands/eval/init.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,gEAAgE;AAChE,8EAA8E;AAE9E,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAC7E,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AACpD,OAAO,EAAE,mBAAmB,EAAE,mBAAmB,EAAE,MAAM,8BAA8B,CAAC;AACxF,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAC;AAEhE,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,QAAgB,EAChB,KAAc;IAEd,MAAM,WAAW,GAAG,IAAI,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;IAC/C,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IACzC,MAAM,SAAS,GAAG,IAAI,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;IAE/C,wBAAwB;IACxB,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC,EAAE,CAAC;QAC7B,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,yBAAyB,WAAW,EAAE,CAAC,CAAC,CAAC;QAC3D,OAAO;IACT,CAAC;IAED,4BAA4B;IAC5B,IAAI,UAAU,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;QACpC,OAAO,CAAC,GAAG,CACT,MAAM,CAAC,qDAAqD,CAAC,CAC9D,CAAC;QACF,OAAO;IACT,CAAC;IAED,MAAM,YAAY,GAAG,YAAY,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC;IACxD,MAAM,MAAM,GAAG,mBAAmB,CAAC,YAAY,CAAC,CAAC;IAEjD,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;QACjC,MAAM,
|
|
1
|
+
{"version":3,"file":"init.js","sourceRoot":"","sources":["../../../src/commands/eval/init.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,gEAAgE;AAChE,8EAA8E;AAE9E,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAC7E,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AACpD,OAAO,EAAE,mBAAmB,EAAE,mBAAmB,EAAE,MAAM,8BAA8B,CAAC;AACxF,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAC;AAEhE,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,QAAgB,EAChB,KAAc;IAEd,MAAM,WAAW,GAAG,IAAI,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;IAC/C,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IACzC,MAAM,SAAS,GAAG,IAAI,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;IAE/C,wBAAwB;IACxB,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC,EAAE,CAAC;QAC7B,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,yBAAyB,WAAW,EAAE,CAAC,CAAC,CAAC;QAC3D,OAAO;IACT,CAAC;IAED,4BAA4B;IAC5B,IAAI,UAAU,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;QACpC,OAAO,CAAC,GAAG,CACT,MAAM,CAAC,qDAAqD,CAAC,CAC9D,CAAC;QACF,OAAO;IACT,CAAC;IAED,MAAM,YAAY,GAAG,YAAY,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC;IACxD,MAAM,MAAM,GAAG,mBAAmB,CAAC,YAAY,CAAC,CAAC;IAEjD,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;QACjC,MAAM,SAAS,GAAG,MAAM,MAAM,CAAC,QAAQ,CACrC,qFAAqF,EACrF,MAAM,CACP,CAAC;QAEF,MAAM,SAAS,GAAG,mBAAmB,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;QAEtD,SAAS,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACzC,aAAa,CAAC,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;QAEtE,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,WAAW,SAAS,EAAE,CAAC,CAAC,CAAC;QAC3C,OAAO,CAAC,GAAG,CACT,GAAG,CAAC,KAAK,SAAS,CAAC,KAAK,CAAC,MAAM,gBAAgB,SAAS,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC,aAAa,CAAC,CAC9H,CAAC;IACJ,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,OAAO,CAAC,KAAK,CACX,GAAG,CAAC,4BAA4B,CAAC,GAAG,GAAG,CAAE,GAAa,CAAC,OAAO,CAAC,CAChE,CAAC;IACJ,CAAC;AACH,CAAC"}
|
|
@@ -54,13 +54,13 @@ export async function runEvalRun(skillDir) {
|
|
|
54
54
|
try {
|
|
55
55
|
// Step 1: Send prompt to LLM
|
|
56
56
|
process.stdout.write(dim(`[${i + 1}/${total}] ${evalCase.name} — generating...`));
|
|
57
|
-
const
|
|
57
|
+
const genResult = await client.generate(systemPrompt, evalCase.prompt);
|
|
58
58
|
process.stdout.write(dim(` judging ${evalCase.assertions.length} assertions...`));
|
|
59
59
|
// Step 2: Judge each assertion
|
|
60
60
|
const assertionResults = [];
|
|
61
61
|
let passCount = 0;
|
|
62
62
|
for (const assertion of evalCase.assertions) {
|
|
63
|
-
const result = await judgeAssertion(
|
|
63
|
+
const result = await judgeAssertion(genResult.text, assertion, client);
|
|
64
64
|
assertionResults.push(result);
|
|
65
65
|
if (result.pass)
|
|
66
66
|
passCount++;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"run.js","sourceRoot":"","sources":["../../../src/commands/eval/run.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,6DAA6D;AAC7D,8EAA8E;AAE9E,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AACnD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,oBAAoB,EAAE,mBAAmB,EAAE,MAAM,sBAAsB,CAAC;AACjF,OAAO,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AACpD,OAAO,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AACrD,OAAO,EAAE,cAAc,EAAE,MAAM,yBAAyB,CAAC;AAEzD,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,EAAE,KAAK,EAAE,MAAM,uBAAuB,CAAC;AAE7E,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,QAAgB;IAC/C,+BAA+B;IAC/B,IAAI,SAAS,CAAC;IACd,IAAI,CAAC;QACH,SAAS,GAAG,oBAAoB,CAAC,QAAQ,CAAC,CAAC;IAC7C,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,IAAI,GAAG,YAAY,mBAAmB,EAAE,CAAC;YACvC,MAAM,QAAQ,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,OAAO,IAAI,EAAE,CAAC;YAC9C,IAAI,QAAQ,CAAC,QAAQ,CAAC,eAAe,CAAC,EAAE,CAAC;gBACvC,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,0BAA0B,QAAQ,mBAAmB,CAAC,CAAC,CAAC;YAC5E,CAAC;iBAAM,CAAC;gBACN,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,uBAAuB,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;YAC3D,CAAC;QACH,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,wBAAyB,GAAa,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;QACvE,CAAC;QACD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAChB,OAAO;IACT,CAAC;IAED,8CAA8C;IAC9C,MAAM,WAAW,GAAG,IAAI,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;IAC/C,IAAI,YAAY,GAAG,EAAE,CAAC;IACtB,IAAI,UAAU,CAAC,WAAW,CAAC,EAAE,CAAC;QAC5B,YAAY,GAAG,YAAY,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC;IACpD,CAAC;SAAM,CAAC;QACN,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,iCAAiC,WAAW,wCAAwC,CAAC,CAAC,CAAC;IAC9G,CAAC;IAED,MAAM,YAAY,GAAG,YAAY;QAC/B,CAAC,CAAC,8HAA8H,YAAY,OAAO;QACnJ,CAAC,CAAC,sDAAsD,CAAC;IAE3D,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;IACjC,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC;IAC3B,MAAM,KAAK,GAAG,SAAS,CAAC,KAAK,CAAC,MAAM,CAAC;IACrC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,aAAa,KAAK,MAAM,KAAK,aAAa,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;IACrF,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,UAAU,YAAY,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,CAAC;IAEtE,MAAM,cAAc,GAAoB,EAAE,CAAC;IAC3C,MAAM,SAAS,GAAe,EAAE,CAAC;IAEjC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAChD,MAAM,QAAQ,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QACpC,IAAI,CAAC;YACH,6BAA6B;YAC7B,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,KAAK,KAAK,QAAQ,CAAC,IAAI,kBAAkB,CAAC,CAAC,CAAC;YAClF,MAAM,
|
|
1
|
+
{"version":3,"file":"run.js","sourceRoot":"","sources":["../../../src/commands/eval/run.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,6DAA6D;AAC7D,8EAA8E;AAE9E,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AACnD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,oBAAoB,EAAE,mBAAmB,EAAE,MAAM,sBAAsB,CAAC;AACjF,OAAO,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AACpD,OAAO,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AACrD,OAAO,EAAE,cAAc,EAAE,MAAM,yBAAyB,CAAC;AAEzD,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,EAAE,KAAK,EAAE,MAAM,uBAAuB,CAAC;AAE7E,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,QAAgB;IAC/C,+BAA+B;IAC/B,IAAI,SAAS,CAAC;IACd,IAAI,CAAC;QACH,SAAS,GAAG,oBAAoB,CAAC,QAAQ,CAAC,CAAC;IAC7C,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,IAAI,GAAG,YAAY,mBAAmB,EAAE,CAAC;YACvC,MAAM,QAAQ,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,OAAO,IAAI,EAAE,CAAC;YAC9C,IAAI,QAAQ,CAAC,QAAQ,CAAC,eAAe,CAAC,EAAE,CAAC;gBACvC,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,0BAA0B,QAAQ,mBAAmB,CAAC,CAAC,CAAC;YAC5E,CAAC;iBAAM,CAAC;gBACN,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,uBAAuB,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;YAC3D,CAAC;QACH,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,wBAAyB,GAAa,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;QACvE,CAAC;QACD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAChB,OAAO;IACT,CAAC;IAED,8CAA8C;IAC9C,MAAM,WAAW,GAAG,IAAI,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;IAC/C,IAAI,YAAY,GAAG,EAAE,CAAC;IACtB,IAAI,UAAU,CAAC,WAAW,CAAC,EAAE,CAAC;QAC5B,YAAY,GAAG,YAAY,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC;IACpD,CAAC;SAAM,CAAC;QACN,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,iCAAiC,WAAW,wCAAwC,CAAC,CAAC,CAAC;IAC9G,CAAC;IAED,MAAM,YAAY,GAAG,YAAY;QAC/B,CAAC,CAAC,8HAA8H,YAAY,OAAO;QACnJ,CAAC,CAAC,sDAAsD,CAAC;IAE3D,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;IACjC,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC;IAC3B,MAAM,KAAK,GAAG,SAAS,CAAC,KAAK,CAAC,MAAM,CAAC;IACrC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,aAAa,KAAK,MAAM,KAAK,aAAa,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;IACrF,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,UAAU,YAAY,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,CAAC;IAEtE,MAAM,cAAc,GAAoB,EAAE,CAAC;IAC3C,MAAM,SAAS,GAAe,EAAE,CAAC;IAEjC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAChD,MAAM,QAAQ,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QACpC,IAAI,CAAC;YACH,6BAA6B;YAC7B,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,KAAK,KAAK,QAAQ,CAAC,IAAI,kBAAkB,CAAC,CAAC,CAAC;YAClF,MAAM,SAAS,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC,YAAY,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;YACvE,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,YAAY,QAAQ,CAAC,UAAU,CAAC,MAAM,gBAAgB,CAAC,CAAC,CAAC;YAElF,+BAA+B;YAC/B,MAAM,gBAAgB,GAAG,EAAE,CAAC;YAC5B,IAAI,SAAS,GAAG,CAAC,CAAC;YAElB,KAAK,MAAM,SAAS,IAAI,QAAQ,CAAC,UAAU,EAAE,CAAC;gBAC5C,MAAM,MAAM,GAAG,MAAM,cAAc,CAAC,SAAS,CAAC,IAAI,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC;gBACvE,gBAAgB,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;gBAC9B,IAAI,MAAM,CAAC,IAAI;oBAAE,SAAS,EAAE,CAAC;gBAE7B,MAAM,aAAa,GACjB,SAAS,CAAC,IAAI,CAAC,MAAM,GAAG,EAAE;oBACxB,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,KAAK;oBACrC,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC;gBAErB,SAAS,CAAC,IAAI,CAAC;oBACb,QAAQ,CAAC,IAAI;oBACb,SAAS,CAAC,EAAE;oBACZ,aAAa;oBACb,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC;iBAC1C,CAAC,CAAC;YACL,CAAC;YAED,MAAM,QAAQ,GAAG,QAAQ,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC;gBAC7C,CAAC,CAAC,SAAS,GAAG,QAAQ,CAAC,UAAU,CAAC,MAAM;gBACxC,CAAC,CAAC,CAAC,CAAC;YACN,MAAM,SAAS,GAAG,SAAS,KAAK,QAAQ,CAAC,UAAU,CAAC,MAAM,CAAC;YAC3D,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,SAAS,IAAI,QAAQ,CAAC,UAAU,CAAC,MAAM,SAAS,CAAC,CAAC,CAAC;YAEpG,cAAc,CAAC,IAAI,CAAC;gBAClB,OAAO,EAAE,QAAQ,CAAC,EAAE;gBACpB,SAAS,EAAE,QAAQ,CAAC,IAAI;gBACxB,MAAM,EAAE,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM;gBACnC,aAAa,EAAE,IAAI;gBACnB,SAAS,EAAE,QAAQ;gBACnB,UAAU,EAAE,gBAAgB;aAC7B,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC;YAC9B,cAAc,CAAC,IAAI,CAAC;gBAClB,OAAO,EAAE,QAAQ,CAAC,EAAE;gBACpB,SAAS,EAAE,QAAQ,CAAC,IAAI;gBACxB,MAAM,EAAE,OAAO;gBACf,aAAa,EAAG,GAAa,CAAC,OAAO;gBACrC,SAAS,EAAE,CAAC;gBACZ,UAAU,EAAE,EAAE;aACf,CAAC,CAAC;YAEH,SAAS,CAAC,IAAI,CAAC;gBACb,QAAQ,CAAC,IAAI;gBACb,GAAG;gBACH,GAAG,CAAC,SAAS,GAAI,GAAa,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;gBACpD,MAAM,CAAC,OAAO,CAAC;aAChB,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,sBAAsB;IACtB,MAAM,OAAO,GAAG,CAAC,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC;IACxD,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,mBAAmB,SAAS,CAAC,UAAU,IAAI,CAAC,CAAC,CAAC;IAC/D,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC,CAAC;IAEvC,kBAAkB;IAClB,MAAM,MAAM,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC,MAAM,CAAC;IACxE,MAAM,MAAM,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC,MAAM,CAAC;IACxE,MAAM,MAAM,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,OAAO,CAAC,CAAC,MAAM,CAAC;IACzE,OAAO,CAAC,GAAG,CACT,KAAK,KAAK,CAAC,GAAG,MAAM,SAAS,CAAC,IAAI,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,MAAM,SAAS,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,MAAM,SAAS,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,EAAE,CACrI,CAAC;IAEF,uBAAuB;IACvB,MAAM,SAAS,GAAoB;QACjC,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACnC,KAAK;QACL,UAAU,EAAE,SAAS,CAAC,UAAU;QAChC,KAAK,EAAE,cAAc;KACtB,CAAC;IAEF,MAAM,cAAc,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;IAC1C,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,0BAA0B,QAAQ,uBAAuB,CAAC,CAAC,CAAC;AAC9E,CAAC"}
|
|
@@ -4,7 +4,10 @@ function mockClient(responses) {
|
|
|
4
4
|
let i = 0;
|
|
5
5
|
return {
|
|
6
6
|
model: "test-model",
|
|
7
|
-
generate: vi.fn(async () =>
|
|
7
|
+
generate: vi.fn(async () => {
|
|
8
|
+
const text = responses[i++] ?? "";
|
|
9
|
+
return { text, durationMs: 100, inputTokens: null, outputTokens: null };
|
|
10
|
+
}),
|
|
8
11
|
};
|
|
9
12
|
}
|
|
10
13
|
const PROMPTS = [
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"activation-tester.test.js","sourceRoot":"","sources":["../../../src/eval/__tests__/activation-tester.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAC;AAClD,OAAO,EAAE,cAAc,EAAE,MAAM,yBAAyB,CAAC;AAIzD,SAAS,UAAU,CAAC,SAAmB;IACrC,IAAI,CAAC,GAAG,CAAC,CAAC;IACV,OAAO;QACL,KAAK,EAAE,YAAY;QACnB,QAAQ,EAAE,EAAE,CAAC,EAAE,CAAC,KAAK,IAAI,EAAE,
|
|
1
|
+
{"version":3,"file":"activation-tester.test.js","sourceRoot":"","sources":["../../../src/eval/__tests__/activation-tester.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAC;AAClD,OAAO,EAAE,cAAc,EAAE,MAAM,yBAAyB,CAAC;AAIzD,SAAS,UAAU,CAAC,SAAmB;IACrC,IAAI,CAAC,GAAG,CAAC,CAAC;IACV,OAAO;QACL,KAAK,EAAE,YAAY;QACnB,QAAQ,EAAE,EAAE,CAAC,EAAE,CAAC,KAAK,IAAI,EAAE;YACzB,MAAM,IAAI,GAAG,SAAS,CAAC,CAAC,EAAE,CAAC,IAAI,EAAE,CAAC;YAClC,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,GAAG,EAAE,WAAW,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC;QAC1E,CAAC,CAAC;KACH,CAAC;AACJ,CAAC;AAED,MAAM,OAAO,GAAuB;IAClC,EAAE,MAAM,EAAE,wBAAwB,EAAE,QAAQ,EAAE,iBAAiB,EAAE;IACjE,EAAE,MAAM,EAAE,sBAAsB,EAAE,QAAQ,EAAE,qBAAqB,EAAE;CACpE,CAAC;AAEF,QAAQ,CAAC,gBAAgB,EAAE,GAAG,EAAE;IAC9B,EAAE,CAAC,0DAA0D,EAAE,KAAK,IAAI,EAAE;QACxE,MAAM,MAAM,GAAG,UAAU,CAAC;YACxB,IAAI,CAAC,SAAS,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,EAAE,SAAS,EAAE,cAAc,EAAE,CAAC;YACjF,IAAI,CAAC,SAAS,CAAC,EAAE,QAAQ,EAAE,KAAK,EAAE,UAAU,EAAE,MAAM,EAAE,SAAS,EAAE,aAAa,EAAE,CAAC;SAClF,CAAC,CAAC;QAEH,MAAM,OAAO,GAAG,MAAM,cAAc,CAAC,wBAAwB,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;QAChF,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC3B,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC3B,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC3B,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC3B,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC/B,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACpC,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAChC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,8DAA8D,EAAE,KAAK,IAAI,EAAE;QAC5E,MAAM,MAAM,GAAG,UAAU,CAAC;YACxB,IAAI,CAAC,SAAS,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,UAAU,EAAE,QAAQ,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC;YAC1E,IAAI,CAAC,SAAS,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,UAAU,EAAE,KAAK,EAAE,SAAS,EAAE,mBAAmB,EAAE,CAAC;SACtF,CAAC,CAAC;QAEH,MAAM,OAAO,GAAG,MAAM,cAAc,CAAC,MAAM,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;QAC9D,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC3B,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC3B,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACpC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACjC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,qDAAqD,EAAE,KAAK,IAAI,EAAE;QACnE,MAAM,MAAM,GAAG,UAAU,CAAC;YACxB,IAAI,CAAC,SAAS,CAAC,EAAE,QAAQ,EAAE,KAAK,EAAE,UAAU,EAAE,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAC;YAC5E,IAAI,CAAC,SAAS,CAAC,EAAE,QAAQ,EAAE,KAAK,EAAE,UAAU,EAAE,MAAM,EAAE,SAAS,EAAE,SAAS,EAAE,CAAC;SAC9E,CAAC,CAAC;QAEH,MAAM,OAAO,GAAG,MAAM,cAAc,CAAC,MAAM,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;QAC9D,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC3B,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC3B,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACjC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,+BAA+B,EAAE,KAAK,IAAI,EAAE;QAC7C,MAAM,MAAM,GAAc;YACxB,KAAK,EAAE,MAAM;YACb,QAAQ,EAAE,EAAE,CAAC,EAAE,CAAC,KAAK,IAAI,EAAE;gBACzB,MAAM,IAAI,KAAK,CAAC,aAAa,CAAC,CAAC;YACjC,CAAC,CAAC;SACH,CAAC;QAEF,MAAM,OAAO,GAAG,MAAM,cAAc,CAAC,MAAM,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;QAC9D,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC9B,8EAA8E;QAC9E,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC3B,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC3B,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC;IAChE,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,yCAAyC,EAAE,KAAK,IAAI,EAAE;QACvD,MAAM,MAAM,GAAG,UAAU,CAAC;YACxB,IAAI,CAAC,SAAS,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC;YACxE,IAAI,CAAC,SAAS,CAAC,EAAE,QAAQ,EAAE,KAAK,EAAE,UAAU,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC;SACzE,CAAC,CAAC;QAEH,MAAM,OAAO,GAAU,EAAE,CAAC;QAC1B,MAAM,cAAc,CAAC,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;QACtE,MAAM,CAAC,OAAO,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAChC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC7C,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC/C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,uCAAuC,EAAE,KAAK,IAAI,EAAE;QACrD,MAAM,MAAM,GAAG,UAAU,CAAC;YACxB,yFAAyF;YACzF,iFAAiF;SAClF,CAAC,CAAC;QAEH,MAAM,OAAO,GAAG,MAAM,cAAc,CAAC,MAAM,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;QAC9D,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC3B,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAC7B,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,6BAA6B,EAAE,KAAK,IAAI,EAAE;QAC3C,MAAM,MAAM,GAAG,UAAU,CAAC,EAAE,CAAC,CAAC;QAC9B,MAAM,OAAO,GAAG,MAAM,cAAc,CAAC,MAAM,EAAE,EAAE,EAAE,MAAM,CAAC,CAAC;QACzD,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC9B,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC/B,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACtC,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
|
@@ -4,7 +4,10 @@ function mockClient(responses) {
|
|
|
4
4
|
let callIndex = 0;
|
|
5
5
|
return {
|
|
6
6
|
model: "test-model",
|
|
7
|
-
generate: vi.fn(async () =>
|
|
7
|
+
generate: vi.fn(async () => {
|
|
8
|
+
const text = responses[callIndex++] ?? "";
|
|
9
|
+
return { text, durationMs: 100, inputTokens: 50, outputTokens: 100 };
|
|
10
|
+
}),
|
|
8
11
|
};
|
|
9
12
|
}
|
|
10
13
|
describe("generateComparisonOutputs", () => {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"comparator.test.js","sourceRoot":"","sources":["../../../src/eval/__tests__/comparator.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAC;AAClD,OAAO,EACL,yBAAyB,EACzB,eAAe,EACf,aAAa,GACd,MAAM,kBAAkB,CAAC;AAG1B,SAAS,UAAU,CAAC,SAAmB;IACrC,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,OAAO;QACL,KAAK,EAAE,YAAY;QACnB,QAAQ,EAAE,EAAE,CAAC,EAAE,CAAC,KAAK,IAAI,EAAE,
|
|
1
|
+
{"version":3,"file":"comparator.test.js","sourceRoot":"","sources":["../../../src/eval/__tests__/comparator.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAC;AAClD,OAAO,EACL,yBAAyB,EACzB,eAAe,EACf,aAAa,GACd,MAAM,kBAAkB,CAAC;AAG1B,SAAS,UAAU,CAAC,SAAmB;IACrC,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,OAAO;QACL,KAAK,EAAE,YAAY;QACnB,QAAQ,EAAE,EAAE,CAAC,EAAE,CAAC,KAAK,IAAI,EAAE;YACzB,MAAM,IAAI,GAAG,SAAS,CAAC,SAAS,EAAE,CAAC,IAAI,EAAE,CAAC;YAC1C,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,GAAG,EAAE,WAAW,EAAE,EAAE,EAAE,YAAY,EAAE,GAAG,EAAE,CAAC;QACvE,CAAC,CAAC;KACH,CAAC;AACJ,CAAC;AAED,QAAQ,CAAC,2BAA2B,EAAE,GAAG,EAAE;IACzC,EAAE,CAAC,mDAAmD,EAAE,KAAK,IAAI,EAAE;QACjE,MAAM,MAAM,GAAG,UAAU,CAAC,CAAC,gBAAgB,EAAE,mBAAmB,CAAC,CAAC,CAAC;QACnE,MAAM,MAAM,GAAG,MAAM,yBAAyB,CAAC,aAAa,EAAE,iBAAiB,EAAE,MAAM,CAAC,CAAC;QAEzF,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;QAClD,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;QACxD,MAAM,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,CAAC;QACzD,MAAM,CAAC,MAAM,CAAC,kBAAkB,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,CAAC;QAC5D,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC;QAEjD,0CAA0C;QAC1C,MAAM,SAAS,GAAI,MAAM,CAAC,QAAgB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QACzD,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,eAAe,CAAC,CAAC;QAEhD,gCAAgC;QAChC,MAAM,UAAU,GAAI,MAAM,CAAC,QAAgB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QAC1D,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,sBAAsB,CAAC,CAAC;IAC1D,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,iBAAiB,EAAE,GAAG,EAAE;IAC/B,EAAE,CAAC,sCAAsC,EAAE,KAAK,IAAI,EAAE;QACpD,MAAM,MAAM,GAAG,UAAU,CAAC;YACxB,IAAI,CAAC,SAAS,CAAC;gBACb,eAAe,EAAE,CAAC;gBAClB,iBAAiB,EAAE,CAAC;gBACpB,eAAe,EAAE,CAAC;gBAClB,iBAAiB,EAAE,CAAC;gBACpB,MAAM,EAAE,QAAQ;gBAChB,SAAS,EAAE,aAAa;aACzB,CAAC;SACH,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,MAAM,eAAe,CAAC,UAAU,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC;QAC/E,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACrC,MAAM,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACvC,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACrC,MAAM,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACvC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IACvC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,6BAA6B,EAAE,KAAK,IAAI,EAAE;QAC3C,MAAM,MAAM,GAAG,UAAU,CAAC;YACxB,6HAA6H;SAC9H,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,MAAM,eAAe,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,MAAM,CAAC,CAAC;QAC5D,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAClC,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACvC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,4BAA4B,EAAE,KAAK,IAAI,EAAE;QAC1C,MAAM,MAAM,GAAG,UAAU,CAAC;YACxB,IAAI,CAAC,SAAS,CAAC;gBACb,eAAe,EAAE,CAAC;gBAClB,iBAAiB,EAAE,EAAE;gBACrB,eAAe,EAAE,CAAC,CAAC;gBACnB,iBAAiB,EAAE,CAAC;gBACpB,MAAM,EAAE,OAAO;aAChB,CAAC;SACH,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,MAAM,eAAe,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,MAAM,CAAC,CAAC;QAC5D,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACrC,MAAM,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACvC,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACrC,MAAM,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACzC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,gCAAgC,EAAE,KAAK,IAAI,EAAE;QAC9C,MAAM,MAAM,GAAG,UAAU,CAAC;YACxB,IAAI,CAAC,SAAS,CAAC;gBACb,eAAe,EAAE,CAAC;gBAClB,iBAAiB,EAAE,CAAC;gBACpB,eAAe,EAAE,CAAC;gBAClB,iBAAiB,EAAE,CAAC;gBACpB,MAAM,EAAE,SAAS;aAClB,CAAC;SACH,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,MAAM,eAAe,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,MAAM,CAAC,CAAC;QAC5D,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACpC,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,eAAe,EAAE,GAAG,EAAE;IAC7B,EAAE,CAAC,8CAA8C,EAAE,KAAK,IAAI,EAAE;QAC5D,oEAAoE;QACpE,MAAM,MAAM,GAAG,UAAU,CAAC;YACxB,mBAAmB;YACnB,sBAAsB;YACtB,IAAI,CAAC,SAAS,CAAC;gBACb,eAAe,EAAE,CAAC;gBAClB,iBAAiB,EAAE,CAAC;gBACpB,eAAe,EAAE,CAAC;gBAClB,iBAAiB,EAAE,CAAC;gBACpB,MAAM,EAAE,OAAO;gBACf,SAAS,EAAE,aAAa;aACzB,CAAC;SACH,CAAC,CAAC;QAEH,wCAAwC;QACxC,EAAE,CAAC,KAAK,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC,CAAC,qBAAqB;QAEpE,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,aAAa,EAAE,eAAe,EAAE,MAAM,CAAC,CAAC;QAE3E,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QAC1C,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;QACrD,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,IAAI,CAAC,sBAAsB,CAAC,CAAC;QAC3D,qCAAqC;QACrC,MAAM,CAAC,MAAM,CAAC,iBAAiB,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACzC,MAAM,CAAC,MAAM,CAAC,mBAAmB,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC3C,MAAM,CAAC,MAAM,CAAC,oBAAoB,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC5C,MAAM,CAAC,MAAM,CAAC,sBAAsB,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC9C,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAEpC,EAAE,CAAC,eAAe,EAAE,CAAC;IACvB,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,0CAA0C,EAAE,KAAK,IAAI,EAAE;QACxD,MAAM,MAAM,GAAG,UAAU,CAAC;YACxB,WAAW;YACX,cAAc;YACd,IAAI,CAAC,SAAS,CAAC;gBACb,eAAe,EAAE,CAAC;gBAClB,iBAAiB,EAAE,CAAC;gBACpB,eAAe,EAAE,CAAC;gBAClB,iBAAiB,EAAE,CAAC;gBACpB,MAAM,EAAE,QAAQ;aACjB,CAAC;SACH,CAAC,CAAC;QAEH,qBAAqB;QACrB,EAAE,CAAC,KAAK,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC;QAE9C,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,GAAG,EAAE,GAAG,EAAE,MAAM,CAAC,CAAC;QACrD,6CAA6C;QAC7C,MAAM,CAAC,MAAM,CAAC,iBAAiB,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACzC,MAAM,CAAC,MAAM,CAAC,oBAAoB,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC5C,8BAA8B;QAC9B,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAEpC,EAAE,CAAC,eAAe,EAAE,CAAC;IACvB,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
|
@@ -3,8 +3,11 @@ import { judgeAssertion } from "../judge.js";
|
|
|
3
3
|
// ---------------------------------------------------------------------------
|
|
4
4
|
// Helpers
|
|
5
5
|
// ---------------------------------------------------------------------------
|
|
6
|
+
function mockResult(text) {
|
|
7
|
+
return { text, durationMs: 100, inputTokens: null, outputTokens: null };
|
|
8
|
+
}
|
|
6
9
|
function mockClient(response) {
|
|
7
|
-
return { generate: vi.fn().mockResolvedValue(response), model: "test-model" };
|
|
10
|
+
return { generate: vi.fn().mockResolvedValue(mockResult(response)), model: "test-model" };
|
|
8
11
|
}
|
|
9
12
|
const ASSERTION = {
|
|
10
13
|
id: "assert-1",
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"judge.test.js","sourceRoot":"","sources":["../../../src/eval/__tests__/judge.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAc,MAAM,QAAQ,CAAC;AAG9D,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAE7C,8EAA8E;AAC9E,UAAU;AACV,8EAA8E;AAE9E,SAAS,UAAU,CAAC,QAAgB;IAClC,OAAO,EAAE,QAAQ,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,QAAQ,CAAC,EAAE,KAAK,EAAE,YAAY,EAAE,CAAC;
|
|
1
|
+
{"version":3,"file":"judge.test.js","sourceRoot":"","sources":["../../../src/eval/__tests__/judge.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAc,MAAM,QAAQ,CAAC;AAG9D,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAE7C,8EAA8E;AAC9E,UAAU;AACV,8EAA8E;AAE9E,SAAS,UAAU,CAAC,IAAY;IAC9B,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,GAAG,EAAE,WAAW,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC;AAC1E,CAAC;AAED,SAAS,UAAU,CAAC,QAAgB;IAClC,OAAO,EAAE,QAAQ,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,EAAE,KAAK,EAAE,YAAY,EAAE,CAAC;AAC5F,CAAC;AAED,MAAM,SAAS,GAAc;IAC3B,EAAE,EAAE,UAAU;IACd,IAAI,EAAE,6BAA6B;IACnC,IAAI,EAAE,SAAS;CAChB,CAAC;AAEF,8EAA8E;AAC9E,QAAQ;AACR,8EAA8E;AAE9E,QAAQ,CAAC,gBAAgB,EAAE,GAAG,EAAE;IAC9B,EAAE,CAAC,8CAA8C,EAAE,KAAK,IAAI,EAAE;QAC5D,MAAM,MAAM,GAAG,UAAU,CACvB,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,SAAS,EAAE,2BAA2B,EAAE,CAAC,CACvE,CAAC;QAEF,MAAM,MAAM,GAAG,MAAM,cAAc,CACjC,6CAA6C,EAC7C,SAAS,EACT,MAAM,CACP,CAAC;QAEF,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC/B,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC;QAC3D,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QACnC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,6BAA6B,CAAC,CAAC;IAC1D,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,8CAA8C,EAAE,KAAK,IAAI,EAAE;QAC5D,MAAM,MAAM,GAAG,UAAU,CACvB,IAAI,CAAC,SAAS,CAAC;YACb,IAAI,EAAE,KAAK;YACX,SAAS,EAAE,8BAA8B;SAC1C,CAAC,CACH,CAAC;QAEF,MAAM,MAAM,GAAG,MAAM,cAAc,CAAC,aAAa,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC;QAEtE,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAChC,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,8BAA8B,CAAC,CAAC;IAChE,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,oCAAoC,EAAE,KAAK,IAAI,EAAE;QAClD,MAAM,MAAM,GAAG,UAAU,CAAC,kBAAkB,CAAC,CAAC;QAE9C,MAAM,MAAM,CACV,cAAc,CAAC,aAAa,EAAE,SAAS,EAAE,MAAM,CAAC,CACjD,CAAC,OAAO,CAAC,OAAO,CAAC,uBAAuB,CAAC,CAAC;IAC7C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,oCAAoC,EAAE,KAAK,IAAI,EAAE;QAClD,MAAM,MAAM,GAAG,UAAU,CACvB,yDAAyD,CAC1D,CAAC;QAEF,MAAM,MAAM,GAAG,MAAM,cAAc,CAAC,aAAa,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC;QACtE,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACjC,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
|
@@ -124,10 +124,14 @@ describe("createLlmClient", () => {
|
|
|
124
124
|
it("returns text content on successful generate call", async () => {
|
|
125
125
|
mockCreate.mockResolvedValue({
|
|
126
126
|
content: [{ type: "text", text: "Generated response" }],
|
|
127
|
+
usage: { input_tokens: 50, output_tokens: 100 },
|
|
127
128
|
});
|
|
128
129
|
const client = createLlmClient();
|
|
129
130
|
const result = await client.generate("system prompt", "user prompt");
|
|
130
|
-
expect(result).toBe("Generated response");
|
|
131
|
+
expect(result.text).toBe("Generated response");
|
|
132
|
+
expect(result.durationMs).toBeGreaterThanOrEqual(0);
|
|
133
|
+
expect(result.inputTokens).toBe(50);
|
|
134
|
+
expect(result.outputTokens).toBe(100);
|
|
131
135
|
expect(mockCreate).toHaveBeenCalledOnce();
|
|
132
136
|
});
|
|
133
137
|
it("uses default model claude-sonnet-4-20250514", async () => {
|
|
@@ -180,7 +184,7 @@ describe("createLlmClient", () => {
|
|
|
180
184
|
mockSpawn.mockReturnValue(createFakeProc("CLI response\n"));
|
|
181
185
|
const client = createLlmClient();
|
|
182
186
|
const result = await client.generate("system prompt", "user prompt");
|
|
183
|
-
expect(result).toBe("CLI response");
|
|
187
|
+
expect(result.text).toBe("CLI response");
|
|
184
188
|
expect(mockSpawn).toHaveBeenCalledWith("claude", ["-p", "--model", "sonnet"], expect.objectContaining({ stdio: ["pipe", "pipe", "pipe"] }));
|
|
185
189
|
// Verify prompt was piped via stdin
|
|
186
190
|
const proc = mockSpawn.mock.results[0].value;
|
|
@@ -245,7 +249,7 @@ describe("createLlmClient", () => {
|
|
|
245
249
|
const mockFetch = vi.spyOn(globalThis, "fetch").mockResolvedValue(new Response(JSON.stringify({ response: "Ollama reply" }), { status: 200 }));
|
|
246
250
|
const client = createLlmClient();
|
|
247
251
|
const result = await client.generate("system prompt", "user prompt");
|
|
248
|
-
expect(result).toBe("Ollama reply");
|
|
252
|
+
expect(result.text).toBe("Ollama reply");
|
|
249
253
|
expect(mockFetch).toHaveBeenCalledWith("http://localhost:11434/api/generate", expect.objectContaining({
|
|
250
254
|
method: "POST",
|
|
251
255
|
body: expect.stringContaining('"model":"llama3.1:8b"'),
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"llm.test.js","sourceRoot":"","sources":["../../../src/eval/__tests__/llm.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AAEzE,8EAA8E;AAC9E,QAAQ;AACR,8EAA8E;AAE9E,MAAM,UAAU,GAAG,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;AAC7C,MAAM,SAAS,GAAG,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;AAE5C,EAAE,CAAC,IAAI,CAAC,mBAAmB,EAAE,GAAG,EAAE,CAAC,CAAC;IAClC,OAAO,EAAE,MAAM,aAAa;QAC1B,QAAQ,GAAG,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC;KACnC;CACF,CAAC,CAAC,CAAC;AAEJ,EAAE,CAAC,IAAI,CAAC,oBAAoB,EAAE,GAAG,EAAE,CAAC,CAAC;IACnC,KAAK,EAAE,SAAS;CACjB,CAAC,CAAC,CAAC;AAEJ,8EAA8E;AAC9E,yDAAyD;AACzD,8EAA8E;AAC9E,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAE3C,SAAS,cAAc,CAAC,UAAkB,EAAE,QAAQ,GAAG,CAAC,EAAE,UAAU,GAAG,EAAE;IACvE,MAAM,IAAI,GAAG,IAAI,YAAY,EAAS,CAAC;IACvC,IAAI,CAAC,MAAM,GAAG,IAAI,YAAY,EAAE,CAAC;IACjC,IAAI,CAAC,MAAM,GAAG,IAAI,YAAY,EAAE,CAAC;IACjC,IAAI,CAAC,KAAK,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC;IAC9C,IAAI,CAAC,IAAI,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;IAEpB,4CAA4C;IAC5C,UAAU,CAAC,GAAG,EAAE;QACd,IAAI,UAAU;YAAE,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC;QAClE,IAAI,UAAU;YAAE,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC;QAClE,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;IAC/B,CAAC,EAAE,CAAC,CAAC,CAAC;IAEN,OAAO,IAAI,CAAC;AACd,CAAC;AAED,8EAA8E;AAC9E,uCAAuC;AACvC,8EAA8E;AAE9E,MAAM,EAAE,eAAe,EAAE,GAAG,MAAM,MAAM,CAAC,WAAW,CAAC,CAAC;AAEtD,8EAA8E;AAC9E,QAAQ;AACR,8EAA8E;AAE9E,QAAQ,CAAC,iBAAiB,EAAE,GAAG,EAAE;IAC/B,MAAM,OAAO,GAAG,EAAE,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC;IAEnC,UAAU,CAAC,GAAG,EAAE;QACd,EAAE,CAAC,aAAa,EAAE,CAAC;QACnB,OAAO,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC;QACxC,OAAO,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC;QACrC,OAAO,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC;QACrC,OAAO,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC;QACnC,OAAO,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC;IAChC,CAAC,CAAC,CAAC;IAEH,SAAS,CAAC,GAAG,EAAE;QACb,OAAO,CAAC,GAAG,GAAG,EAAE,GAAG,OAAO,EAAE,CAAC;IAC/B,CAAC,CAAC,CAAC;IAEH,4EAA4E;IAC5E,iBAAiB;IACjB,4EAA4E;IAE5E,EAAE,CAAC,8CAA8C,EAAE,GAAG,EAAE;QACtD,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;QACjC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;IAC7C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,gDAAgD,EAAE,GAAG,EAAE;QACxD,OAAO,CAAC,GAAG,CAAC,UAAU,GAAG,GAAG,CAAC;QAC7B,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;QACjC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;IAC3C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,sDAAsD,EAAE,GAAG,EAAE;QAC9D,OAAO,CAAC,GAAG,CAAC,iBAAiB,GAAG,UAAU,CAAC;QAC3C,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;QACjC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;IACxD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,qEAAqE,EAAE,GAAG,EAAE;QAC7E,OAAO,CAAC,GAAG,CAAC,UAAU,GAAG,GAAG,CAAC;QAC7B,OAAO,CAAC,GAAG,CAAC,iBAAiB,GAAG,UAAU,CAAC;QAC3C,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;QACjC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;IAC3C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,wDAAwD,EAAE,GAAG,EAAE;QAChE,OAAO,CAAC,GAAG,CAAC,oBAAoB,GAAG,QAAQ,CAAC;QAC5C,OAAO,CAAC,GAAG,CAAC,iBAAiB,GAAG,UAAU,CAAC;QAC3C,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;QACjC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;IAC3C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,4BAA4B,EAAE,GAAG,EAAE;QACpC,OAAO,CAAC,GAAG,CAAC,oBAAoB,GAAG,WAAW,CAAC;QAC/C,MAAM,CAAC,GAAG,EAAE,CAAC,eAAe,EAAE,CAAC,CAAC,OAAO,CAAC,2CAA2C,CAAC,CAAC;IACvF,CAAC,CAAC,CAAC;IAEH,4EAA4E;IAC5E,kBAAkB;IAClB,4EAA4E;IAE5E,EAAE,CAAC,iDAAiD,EAAE,GAAG,EAAE;QACzD,OAAO,CAAC,GAAG,CAAC,oBAAoB,GAAG,QAAQ,CAAC;QAC5C,OAAO,CAAC,GAAG,CAAC,iBAAiB,GAAG,UAAU,CAAC;QAC3C,MAAM,MAAM,GAAG,eAAe,CAAC,EAAE,QAAQ,EAAE,WAAW,EAAE,CAAC,CAAC;QAC1D,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;IACxD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,8CAA8C,EAAE,GAAG,EAAE;QACtD,OAAO,CAAC,GAAG,CAAC,iBAAiB,GAAG,UAAU,CAAC;QAC3C,MAAM,MAAM,GAAG,eAAe,CAAC,EAAE,QAAQ,EAAE,WAAW,EAAE,KAAK,EAAE,wBAAwB,EAAE,CAAC,CAAC;QAC3F,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAC;IACtD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,gDAAgD,EAAE,GAAG,EAAE;QACxD,MAAM,MAAM,GAAG,eAAe,CAAC,EAAE,QAAQ,EAAE,YAAY,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC;QAC1E,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;IAC3C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,4CAA4C,EAAE,GAAG,EAAE;QACpD,MAAM,MAAM,GAAG,eAAe,CAAC,EAAE,QAAQ,EAAE,QAAQ,EAAE,KAAK,EAAE,aAAa,EAAE,CAAC,CAAC;QAC7E,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;IAC3C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,gEAAgE,EAAE,GAAG,EAAE;QACxE,OAAO,CAAC,GAAG,CAAC,UAAU,GAAG,GAAG,CAAC;QAC7B,8CAA8C;QAC9C,MAAM,MAAM,GAAG,eAAe,CAAC,EAAE,QAAQ,EAAE,YAAY,EAAE,CAAC,CAAC;QAC3D,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;IAC7C,CAAC,CAAC,CAAC;IAEH,4EAA4E;IAC5E,qBAAqB;IACrB,4EAA4E;IAE5E,QAAQ,CAAC,oBAAoB,EAAE,GAAG,EAAE;QAClC,UAAU,CAAC,GAAG,EAAE;YACd,OAAO,CAAC,GAAG,CAAC,oBAAoB,GAAG,WAAW,CAAC;YAC/C,OAAO,CAAC,GAAG,CAAC,iBAAiB,GAAG,UAAU,CAAC;QAC7C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,kDAAkD,EAAE,KAAK,IAAI,EAAE;YAChE,UAAU,CAAC,iBAAiB,CAAC;gBAC3B,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,oBAAoB,EAAE,CAAC;aACxD,CAAC,CAAC;YAEH,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;YACjC,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC,eAAe,EAAE,aAAa,CAAC,CAAC;YAErE,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;YAC1C,MAAM,CAAC,UAAU,CAAC,CAAC,oBAAoB,EAAE,CAAC;QAC5C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,6CAA6C,EAAE,KAAK,IAAI,EAAE;YAC3D,UAAU,CAAC,iBAAiB,CAAC;gBAC3B,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;aACxC,CAAC,CAAC;YAEH,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;YACjC,MAAM,MAAM,CAAC,QAAQ,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;YAEpC,MAAM,CAAC,UAAU,CAAC,CAAC,oBAAoB,CACrC,MAAM,CAAC,gBAAgB,CAAC,EAAE,KAAK,EAAE,0BAA0B,EAAE,CAAC,EAC9D,MAAM,CAAC,QAAQ,EAAE,CAClB,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,0CAA0C,EAAE,KAAK,IAAI,EAAE;YACxD,OAAO,CAAC,GAAG,CAAC,iBAAiB,GAAG,wBAAwB,CAAC;YACzD,UAAU,CAAC,iBAAiB,CAAC;gBAC3B,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;aACxC,CAAC,CAAC;YAEH,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;YACjC,MAAM,MAAM,CAAC,QAAQ,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;YAEpC,MAAM,CAAC,UAAU,CAAC,CAAC,oBAAoB,CACrC,MAAM,CAAC,gBAAgB,CAAC,EAAE,KAAK,EAAE,wBAAwB,EAAE,CAAC,EAC5D,MAAM,CAAC,QAAQ,EAAE,CAClB,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,mCAAmC,EAAE,KAAK,IAAI,EAAE;YACjD,UAAU,CAAC,iBAAiB,CAAC,IAAI,KAAK,CAAC,oBAAoB,CAAC,CAAC,CAAC;YAE9D,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;YACjC,MAAM,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CACzD,oBAAoB,CACrB,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,0CAA0C,EAAE,KAAK,IAAI,EAAE;YACxD,UAAU,CAAC,iBAAiB,CAAC;gBAC3B,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;aACxC,CAAC,CAAC;YAEH,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;YACjC,MAAM,MAAM,CAAC,QAAQ,CAAC,kBAAkB,EAAE,gBAAgB,CAAC,CAAC;YAE5D,MAAM,CAAC,UAAU,CAAC,CAAC,oBAAoB,CACrC,MAAM,CAAC,gBAAgB,CAAC;gBACtB,MAAM,EAAE,kBAAkB;gBAC1B,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,gBAAgB,EAAE,CAAC;gBACvD,UAAU,EAAE,IAAI;aACjB,CAAC,EACF,MAAM,CAAC,QAAQ,EAAE,CAClB,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,0CAA0C,EAAE,GAAG,EAAE;YAClD,OAAO,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC;YACrC,MAAM,CAAC,GAAG,EAAE,CAAC,eAAe,EAAE,CAAC,CAAC,OAAO,CAAC,8BAA8B,CAAC,CAAC;QAC1E,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,4EAA4E;IAC5E,sBAAsB;IACtB,4EAA4E;IAE5E,QAAQ,CAAC,qBAAqB,EAAE,GAAG,EAAE;QACnC,UAAU,CAAC,GAAG,EAAE;YACd,OAAO,CAAC,GAAG,CAAC,oBAAoB,GAAG,YAAY,CAAC;QAClD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,+DAA+D,EAAE,KAAK,IAAI,EAAE;YAC7E,SAAS,CAAC,eAAe,CAAC,cAAc,CAAC,gBAAgB,CAAC,CAAC,CAAC;YAE5D,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;YACjC,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC,eAAe,EAAE,aAAa,CAAC,CAAC;YAErE,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;YACpC,MAAM,CAAC,SAAS,CAAC,CAAC,oBAAoB,CACpC,QAAQ,EACR,CAAC,IAAI,EAAE,SAAS,EAAE,QAAQ,CAAC,EAC3B,MAAM,CAAC,gBAAgB,CAAC,EAAE,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC,CAC7D,CAAC;YACF,oCAAoC;YACpC,MAAM,IAAI,GAAG,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;YAC7C,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,oBAAoB,CAAC,8BAA8B,CAAC,CAAC;QAC9E,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,0BAA0B,EAAE,GAAG,EAAE;YAClC,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;YACjC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;QAC7C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,4CAA4C,EAAE,KAAK,IAAI,EAAE;YAC1D,OAAO,CAAC,GAAG,CAAC,iBAAiB,GAAG,MAAM,CAAC;YACvC,SAAS,CAAC,eAAe,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC;YAElD,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;YACjC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;YACzC,MAAM,MAAM,CAAC,QAAQ,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;YAEpC,MAAM,CAAC,SAAS,CAAC,CAAC,oBAAoB,CACpC,QAAQ,EACR,CAAC,IAAI,EAAE,SAAS,EAAE,MAAM,CAAC,EACzB,MAAM,CAAC,QAAQ,EAAE,CAClB,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,gDAAgD,EAAE,KAAK,IAAI,EAAE;YAC9D,MAAM,IAAI,GAAG,IAAI,YAAY,EAAS,CAAC;YACvC,IAAI,CAAC,MAAM,GAAG,IAAI,YAAY,EAAE,CAAC;YACjC,IAAI,CAAC,MAAM,GAAG,IAAI,YAAY,EAAE,CAAC;YACjC,IAAI,CAAC,KAAK,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC;YAC9C,IAAI,CAAC,IAAI,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;YACpB,SAAS,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC;YAEhC,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;YACjC,MAAM,OAAO,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;YAE9C,oBAAoB;YACpB,UAAU,CAAC,GAAG,EAAE;gBACd,MAAM,GAAG,GAAG,IAAI,KAAK,CAAC,qBAAqB,CAA0B,CAAC;gBACtE,GAAG,CAAC,IAAI,GAAG,QAAQ,CAAC;gBACpB,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;YAC1B,CAAC,EAAE,CAAC,CAAC,CAAC;YAEN,MAAM,MAAM,CAAC,OAAO,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,sBAAsB,CAAC,CAAC;QAChE,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,8CAA8C,EAAE,KAAK,IAAI,EAAE;YAC5D,SAAS,CAAC,eAAe,CAAC,cAAc,CAAC,EAAE,EAAE,CAAC,EAAE,sBAAsB,CAAC,CAAC,CAAC;YAEzE,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;YACjC,MAAM,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CACzD,qDAAqD,CACtD,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,4DAA4D,EAAE,GAAG,EAAE;YACpE,OAAO,CAAC,GAAG,CAAC,UAAU,GAAG,GAAG,CAAC;YAC7B,MAAM,CAAC,GAAG,EAAE,CAAC,eAAe,EAAE,CAAC,CAAC,OAAO,CACrC,6DAA6D,CAC9D,CAAC;QACJ,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,4EAA4E;IAC5E,kBAAkB;IAClB,4EAA4E;IAE5E,QAAQ,CAAC,iBAAiB,EAAE,GAAG,EAAE;QAC/B,UAAU,CAAC,GAAG,EAAE;YACd,OAAO,CAAC,GAAG,CAAC,oBAAoB,GAAG,QAAQ,CAAC;QAC9C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,gCAAgC,EAAE,GAAG,EAAE;YACxC,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;YACjC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QAC3C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,0CAA0C,EAAE,GAAG,EAAE;YAClD,OAAO,CAAC,GAAG,CAAC,iBAAiB,GAAG,aAAa,CAAC;YAC9C,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;YACjC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QAC3C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,4CAA4C,EAAE,KAAK,IAAI,EAAE;YAC1D,MAAM,SAAS,GAAG,EAAE,CAAC,KAAK,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC,iBAAiB,CAC/D,IAAI,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,QAAQ,EAAE,cAAc,EAAE,CAAC,EAAE,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,CAC5E,CAAC;YAEF,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;YACjC,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC,eAAe,EAAE,aAAa,CAAC,CAAC;YAErE,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;YACpC,MAAM,CAAC,SAAS,CAAC,CAAC,oBAAoB,CACpC,qCAAqC,EACrC,MAAM,CAAC,gBAAgB,CAAC;gBACtB,MAAM,EAAE,MAAM;gBACd,IAAI,EAAE,MAAM,CAAC,gBAAgB,CAAC,uBAAuB,CAAC;aACvD,CAAC,CACH,CAAC;YAEF,SAAS,CAAC,WAAW,EAAE,CAAC;QAC1B,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,2CAA2C,EAAE,KAAK,IAAI,EAAE;YACzD,OAAO,CAAC,GAAG,CAAC,eAAe,GAAG,yBAAyB,CAAC;YAExD,MAAM,SAAS,GAAG,EAAE,CAAC,KAAK,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC,iBAAiB,CAC/D,IAAI,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,EAAE,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,CAClE,CAAC;YAEF,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;YACjC,MAAM,MAAM,CAAC,QAAQ,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;YAEpC,MAAM,CAAC,SAAS,CAAC,CAAC,oBAAoB,CACpC,sCAAsC,EACtC,MAAM,CAAC,QAAQ,EAAE,CAClB,CAAC;YAEF,SAAS,CAAC,WAAW,EAAE,CAAC;QAC1B,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
|
1
|
+
{"version":3,"file":"llm.test.js","sourceRoot":"","sources":["../../../src/eval/__tests__/llm.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AAEzE,8EAA8E;AAC9E,QAAQ;AACR,8EAA8E;AAE9E,MAAM,UAAU,GAAG,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;AAC7C,MAAM,SAAS,GAAG,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;AAE5C,EAAE,CAAC,IAAI,CAAC,mBAAmB,EAAE,GAAG,EAAE,CAAC,CAAC;IAClC,OAAO,EAAE,MAAM,aAAa;QAC1B,QAAQ,GAAG,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC;KACnC;CACF,CAAC,CAAC,CAAC;AAEJ,EAAE,CAAC,IAAI,CAAC,oBAAoB,EAAE,GAAG,EAAE,CAAC,CAAC;IACnC,KAAK,EAAE,SAAS;CACjB,CAAC,CAAC,CAAC;AAEJ,8EAA8E;AAC9E,yDAAyD;AACzD,8EAA8E;AAC9E,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAE3C,SAAS,cAAc,CAAC,UAAkB,EAAE,QAAQ,GAAG,CAAC,EAAE,UAAU,GAAG,EAAE;IACvE,MAAM,IAAI,GAAG,IAAI,YAAY,EAAS,CAAC;IACvC,IAAI,CAAC,MAAM,GAAG,IAAI,YAAY,EAAE,CAAC;IACjC,IAAI,CAAC,MAAM,GAAG,IAAI,YAAY,EAAE,CAAC;IACjC,IAAI,CAAC,KAAK,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC;IAC9C,IAAI,CAAC,IAAI,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;IAEpB,4CAA4C;IAC5C,UAAU,CAAC,GAAG,EAAE;QACd,IAAI,UAAU;YAAE,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC;QAClE,IAAI,UAAU;YAAE,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC;QAClE,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;IAC/B,CAAC,EAAE,CAAC,CAAC,CAAC;IAEN,OAAO,IAAI,CAAC;AACd,CAAC;AAED,8EAA8E;AAC9E,uCAAuC;AACvC,8EAA8E;AAE9E,MAAM,EAAE,eAAe,EAAE,GAAG,MAAM,MAAM,CAAC,WAAW,CAAC,CAAC;AAEtD,8EAA8E;AAC9E,QAAQ;AACR,8EAA8E;AAE9E,QAAQ,CAAC,iBAAiB,EAAE,GAAG,EAAE;IAC/B,MAAM,OAAO,GAAG,EAAE,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC;IAEnC,UAAU,CAAC,GAAG,EAAE;QACd,EAAE,CAAC,aAAa,EAAE,CAAC;QACnB,OAAO,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC;QACxC,OAAO,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC;QACrC,OAAO,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC;QACrC,OAAO,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC;QACnC,OAAO,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC;IAChC,CAAC,CAAC,CAAC;IAEH,SAAS,CAAC,GAAG,EAAE;QACb,OAAO,CAAC,GAAG,GAAG,EAAE,GAAG,OAAO,EAAE,CAAC;IAC/B,CAAC,CAAC,CAAC;IAEH,4EAA4E;IAC5E,iBAAiB;IACjB,4EAA4E;IAE5E,EAAE,CAAC,8CAA8C,EAAE,GAAG,EAAE;QACtD,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;QACjC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;IAC7C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,gDAAgD,EAAE,GAAG,EAAE;QACxD,OAAO,CAAC,GAAG,CAAC,UAAU,GAAG,GAAG,CAAC;QAC7B,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;QACjC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;IAC3C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,sDAAsD,EAAE,GAAG,EAAE;QAC9D,OAAO,CAAC,GAAG,CAAC,iBAAiB,GAAG,UAAU,CAAC;QAC3C,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;QACjC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;IACxD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,qEAAqE,EAAE,GAAG,EAAE;QAC7E,OAAO,CAAC,GAAG,CAAC,UAAU,GAAG,GAAG,CAAC;QAC7B,OAAO,CAAC,GAAG,CAAC,iBAAiB,GAAG,UAAU,CAAC;QAC3C,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;QACjC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;IAC3C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,wDAAwD,EAAE,GAAG,EAAE;QAChE,OAAO,CAAC,GAAG,CAAC,oBAAoB,GAAG,QAAQ,CAAC;QAC5C,OAAO,CAAC,GAAG,CAAC,iBAAiB,GAAG,UAAU,CAAC;QAC3C,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;QACjC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;IAC3C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,4BAA4B,EAAE,GAAG,EAAE;QACpC,OAAO,CAAC,GAAG,CAAC,oBAAoB,GAAG,WAAW,CAAC;QAC/C,MAAM,CAAC,GAAG,EAAE,CAAC,eAAe,EAAE,CAAC,CAAC,OAAO,CAAC,2CAA2C,CAAC,CAAC;IACvF,CAAC,CAAC,CAAC;IAEH,4EAA4E;IAC5E,kBAAkB;IAClB,4EAA4E;IAE5E,EAAE,CAAC,iDAAiD,EAAE,GAAG,EAAE;QACzD,OAAO,CAAC,GAAG,CAAC,oBAAoB,GAAG,QAAQ,CAAC;QAC5C,OAAO,CAAC,GAAG,CAAC,iBAAiB,GAAG,UAAU,CAAC;QAC3C,MAAM,MAAM,GAAG,eAAe,CAAC,EAAE,QAAQ,EAAE,WAAW,EAAE,CAAC,CAAC;QAC1D,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;IACxD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,8CAA8C,EAAE,GAAG,EAAE;QACtD,OAAO,CAAC,GAAG,CAAC,iBAAiB,GAAG,UAAU,CAAC;QAC3C,MAAM,MAAM,GAAG,eAAe,CAAC,EAAE,QAAQ,EAAE,WAAW,EAAE,KAAK,EAAE,wBAAwB,EAAE,CAAC,CAAC;QAC3F,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAC;IACtD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,gDAAgD,EAAE,GAAG,EAAE;QACxD,MAAM,MAAM,GAAG,eAAe,CAAC,EAAE,QAAQ,EAAE,YAAY,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC;QAC1E,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;IAC3C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,4CAA4C,EAAE,GAAG,EAAE;QACpD,MAAM,MAAM,GAAG,eAAe,CAAC,EAAE,QAAQ,EAAE,QAAQ,EAAE,KAAK,EAAE,aAAa,EAAE,CAAC,CAAC;QAC7E,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;IAC3C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,gEAAgE,EAAE,GAAG,EAAE;QACxE,OAAO,CAAC,GAAG,CAAC,UAAU,GAAG,GAAG,CAAC;QAC7B,8CAA8C;QAC9C,MAAM,MAAM,GAAG,eAAe,CAAC,EAAE,QAAQ,EAAE,YAAY,EAAE,CAAC,CAAC;QAC3D,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;IAC7C,CAAC,CAAC,CAAC;IAEH,4EAA4E;IAC5E,qBAAqB;IACrB,4EAA4E;IAE5E,QAAQ,CAAC,oBAAoB,EAAE,GAAG,EAAE;QAClC,UAAU,CAAC,GAAG,EAAE;YACd,OAAO,CAAC,GAAG,CAAC,oBAAoB,GAAG,WAAW,CAAC;YAC/C,OAAO,CAAC,GAAG,CAAC,iBAAiB,GAAG,UAAU,CAAC;QAC7C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,kDAAkD,EAAE,KAAK,IAAI,EAAE;YAChE,UAAU,CAAC,iBAAiB,CAAC;gBAC3B,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,oBAAoB,EAAE,CAAC;gBACvD,KAAK,EAAE,EAAE,YAAY,EAAE,EAAE,EAAE,aAAa,EAAE,GAAG,EAAE;aAChD,CAAC,CAAC;YAEH,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;YACjC,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC,eAAe,EAAE,aAAa,CAAC,CAAC;YAErE,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;YAC/C,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,CAAC;YACpD,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACpC,MAAM,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACtC,MAAM,CAAC,UAAU,CAAC,CAAC,oBAAoB,EAAE,CAAC;QAC5C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,6CAA6C,EAAE,KAAK,IAAI,EAAE;YAC3D,UAAU,CAAC,iBAAiB,CAAC;gBAC3B,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;aACxC,CAAC,CAAC;YAEH,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;YACjC,MAAM,MAAM,CAAC,QAAQ,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;YAEpC,MAAM,CAAC,UAAU,CAAC,CAAC,oBAAoB,CACrC,MAAM,CAAC,gBAAgB,CAAC,EAAE,KAAK,EAAE,0BAA0B,EAAE,CAAC,EAC9D,MAAM,CAAC,QAAQ,EAAE,CAClB,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,0CAA0C,EAAE,KAAK,IAAI,EAAE;YACxD,OAAO,CAAC,GAAG,CAAC,iBAAiB,GAAG,wBAAwB,CAAC;YACzD,UAAU,CAAC,iBAAiB,CAAC;gBAC3B,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;aACxC,CAAC,CAAC;YAEH,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;YACjC,MAAM,MAAM,CAAC,QAAQ,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;YAEpC,MAAM,CAAC,UAAU,CAAC,CAAC,oBAAoB,CACrC,MAAM,CAAC,gBAAgB,CAAC,EAAE,KAAK,EAAE,wBAAwB,EAAE,CAAC,EAC5D,MAAM,CAAC,QAAQ,EAAE,CAClB,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,mCAAmC,EAAE,KAAK,IAAI,EAAE;YACjD,UAAU,CAAC,iBAAiB,CAAC,IAAI,KAAK,CAAC,oBAAoB,CAAC,CAAC,CAAC;YAE9D,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;YACjC,MAAM,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CACzD,oBAAoB,CACrB,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,0CAA0C,EAAE,KAAK,IAAI,EAAE;YACxD,UAAU,CAAC,iBAAiB,CAAC;gBAC3B,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;aACxC,CAAC,CAAC;YAEH,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;YACjC,MAAM,MAAM,CAAC,QAAQ,CAAC,kBAAkB,EAAE,gBAAgB,CAAC,CAAC;YAE5D,MAAM,CAAC,UAAU,CAAC,CAAC,oBAAoB,CACrC,MAAM,CAAC,gBAAgB,CAAC;gBACtB,MAAM,EAAE,kBAAkB;gBAC1B,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,gBAAgB,EAAE,CAAC;gBACvD,UAAU,EAAE,IAAI;aACjB,CAAC,EACF,MAAM,CAAC,QAAQ,EAAE,CAClB,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,0CAA0C,EAAE,GAAG,EAAE;YAClD,OAAO,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC;YACrC,MAAM,CAAC,GAAG,EAAE,CAAC,eAAe,EAAE,CAAC,CAAC,OAAO,CAAC,8BAA8B,CAAC,CAAC;QAC1E,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,4EAA4E;IAC5E,sBAAsB;IACtB,4EAA4E;IAE5E,QAAQ,CAAC,qBAAqB,EAAE,GAAG,EAAE;QACnC,UAAU,CAAC,GAAG,EAAE;YACd,OAAO,CAAC,GAAG,CAAC,oBAAoB,GAAG,YAAY,CAAC;QAClD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,+DAA+D,EAAE,KAAK,IAAI,EAAE;YAC7E,SAAS,CAAC,eAAe,CAAC,cAAc,CAAC,gBAAgB,CAAC,CAAC,CAAC;YAE5D,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;YACjC,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC,eAAe,EAAE,aAAa,CAAC,CAAC;YAErE,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;YACzC,MAAM,CAAC,SAAS,CAAC,CAAC,oBAAoB,CACpC,QAAQ,EACR,CAAC,IAAI,EAAE,SAAS,EAAE,QAAQ,CAAC,EAC3B,MAAM,CAAC,gBAAgB,CAAC,EAAE,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC,CAC7D,CAAC;YACF,oCAAoC;YACpC,MAAM,IAAI,GAAG,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;YAC7C,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,oBAAoB,CAAC,8BAA8B,CAAC,CAAC;QAC9E,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,0BAA0B,EAAE,GAAG,EAAE;YAClC,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;YACjC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;QAC7C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,4CAA4C,EAAE,KAAK,IAAI,EAAE;YAC1D,OAAO,CAAC,GAAG,CAAC,iBAAiB,GAAG,MAAM,CAAC;YACvC,SAAS,CAAC,eAAe,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC;YAElD,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;YACjC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;YACzC,MAAM,MAAM,CAAC,QAAQ,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;YAEpC,MAAM,CAAC,SAAS,CAAC,CAAC,oBAAoB,CACpC,QAAQ,EACR,CAAC,IAAI,EAAE,SAAS,EAAE,MAAM,CAAC,EACzB,MAAM,CAAC,QAAQ,EAAE,CAClB,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,gDAAgD,EAAE,KAAK,IAAI,EAAE;YAC9D,MAAM,IAAI,GAAG,IAAI,YAAY,EAAS,CAAC;YACvC,IAAI,CAAC,MAAM,GAAG,IAAI,YAAY,EAAE,CAAC;YACjC,IAAI,CAAC,MAAM,GAAG,IAAI,YAAY,EAAE,CAAC;YACjC,IAAI,CAAC,KAAK,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC;YAC9C,IAAI,CAAC,IAAI,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;YACpB,SAAS,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC;YAEhC,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;YACjC,MAAM,OAAO,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;YAE9C,oBAAoB;YACpB,UAAU,CAAC,GAAG,EAAE;gBACd,MAAM,GAAG,GAAG,IAAI,KAAK,CAAC,qBAAqB,CAA0B,CAAC;gBACtE,GAAG,CAAC,IAAI,GAAG,QAAQ,CAAC;gBACpB,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;YAC1B,CAAC,EAAE,CAAC,CAAC,CAAC;YAEN,MAAM,MAAM,CAAC,OAAO,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,sBAAsB,CAAC,CAAC;QAChE,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,8CAA8C,EAAE,KAAK,IAAI,EAAE;YAC5D,SAAS,CAAC,eAAe,CAAC,cAAc,CAAC,EAAE,EAAE,CAAC,EAAE,sBAAsB,CAAC,CAAC,CAAC;YAEzE,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;YACjC,MAAM,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CACzD,qDAAqD,CACtD,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,4DAA4D,EAAE,GAAG,EAAE;YACpE,OAAO,CAAC,GAAG,CAAC,UAAU,GAAG,GAAG,CAAC;YAC7B,MAAM,CAAC,GAAG,EAAE,CAAC,eAAe,EAAE,CAAC,CAAC,OAAO,CACrC,6DAA6D,CAC9D,CAAC;QACJ,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,4EAA4E;IAC5E,kBAAkB;IAClB,4EAA4E;IAE5E,QAAQ,CAAC,iBAAiB,EAAE,GAAG,EAAE;QAC/B,UAAU,CAAC,GAAG,EAAE;YACd,OAAO,CAAC,GAAG,CAAC,oBAAoB,GAAG,QAAQ,CAAC;QAC9C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,gCAAgC,EAAE,GAAG,EAAE;YACxC,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;YACjC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QAC3C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,0CAA0C,EAAE,GAAG,EAAE;YAClD,OAAO,CAAC,GAAG,CAAC,iBAAiB,GAAG,aAAa,CAAC;YAC9C,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;YACjC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QAC3C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,4CAA4C,EAAE,KAAK,IAAI,EAAE;YAC1D,MAAM,SAAS,GAAG,EAAE,CAAC,KAAK,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC,iBAAiB,CAC/D,IAAI,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,QAAQ,EAAE,cAAc,EAAE,CAAC,EAAE,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,CAC5E,CAAC;YAEF,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;YACjC,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC,eAAe,EAAE,aAAa,CAAC,CAAC;YAErE,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;YACzC,MAAM,CAAC,SAAS,CAAC,CAAC,oBAAoB,CACpC,qCAAqC,EACrC,MAAM,CAAC,gBAAgB,CAAC;gBACtB,MAAM,EAAE,MAAM;gBACd,IAAI,EAAE,MAAM,CAAC,gBAAgB,CAAC,uBAAuB,CAAC;aACvD,CAAC,CACH,CAAC;YAEF,SAAS,CAAC,WAAW,EAAE,CAAC;QAC1B,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,2CAA2C,EAAE,KAAK,IAAI,EAAE;YACzD,OAAO,CAAC,GAAG,CAAC,eAAe,GAAG,yBAAyB,CAAC;YAExD,MAAM,SAAS,GAAG,EAAE,CAAC,KAAK,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC,iBAAiB,CAC/D,IAAI,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,EAAE,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,CAClE,CAAC;YAEF,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;YACjC,MAAM,MAAM,CAAC,QAAQ,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;YAEpC,MAAM,CAAC,SAAS,CAAC,CAAC,oBAAoB,CACpC,sCAAsC,EACtC,MAAM,CAAC,QAAQ,EAAE,CAClB,CAAC;YAEF,SAAS,CAAC,WAAW,EAAE,CAAC;QAC1B,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
|
@@ -25,7 +25,7 @@ ${p.prompt}
|
|
|
25
25
|
|
|
26
26
|
Would this user prompt trigger this skill?`;
|
|
27
27
|
try {
|
|
28
|
-
const response = await client.generate(ACTIVATION_SYSTEM_PROMPT, userPrompt);
|
|
28
|
+
const { text: response } = await client.generate(ACTIVATION_SYSTEM_PROMPT, userPrompt);
|
|
29
29
|
const jsonMatch = response.match(/```(?:json)?\s*([\s\S]*?)```/) || [null, response];
|
|
30
30
|
const json = JSON.parse(jsonMatch[1].trim());
|
|
31
31
|
const activate = !!json.activate;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"activation-tester.js","sourceRoot":"","sources":["../../src/eval/activation-tester.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,4EAA4E;AAC5E,8EAA8E;AA8B9E,MAAM,wBAAwB,GAAG;;;;;;;;;;;;EAY/B,CAAC;AAEH,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,gBAAwB,EACxB,OAA2B,EAC3B,MAAiB,EACjB,QAA6C;IAE7C,MAAM,OAAO,GAAuB,EAAE,CAAC;IAEvC,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACxB,MAAM,UAAU,GAAG;EACrB,gBAAgB;;;EAGhB,CAAC,CAAC,MAAM;;2CAEiC,CAAC;QAExC,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC,wBAAwB,EAAE,UAAU,CAAC,CAAC;
|
|
1
|
+
{"version":3,"file":"activation-tester.js","sourceRoot":"","sources":["../../src/eval/activation-tester.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,4EAA4E;AAC5E,8EAA8E;AA8B9E,MAAM,wBAAwB,GAAG;;;;;;;;;;;;EAY/B,CAAC;AAEH,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,gBAAwB,EACxB,OAA2B,EAC3B,MAAiB,EACjB,QAA6C;IAE7C,MAAM,OAAO,GAAuB,EAAE,CAAC;IAEvC,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACxB,MAAM,UAAU,GAAG;EACrB,gBAAgB;;;EAGhB,CAAC,CAAC,MAAM;;2CAEiC,CAAC;QAExC,IAAI,CAAC;YACH,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC,wBAAwB,EAAE,UAAU,CAAC,CAAC;YACvF,MAAM,SAAS,GAAG,QAAQ,CAAC,KAAK,CAAC,8BAA8B,CAAC,IAAI,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;YACrF,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAE,CAAC,IAAI,EAAE,CAAC,CAAC;YAE9C,MAAM,QAAQ,GAAG,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC;YACjC,MAAM,UAAU,GAAG,CAAC,MAAM,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,UAAU,CAAC;gBACpE,CAAC,CAAE,IAAI,CAAC,UAAwC;gBAChD,CAAC,CAAC,KAAK,CAAC;YAEV,MAAM,cAAc,GAAG,cAAc,CAAC,CAAC,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;YAE5D,MAAM,MAAM,GAAqB;gBAC/B,MAAM,EAAE,CAAC,CAAC,MAAM;gBAChB,QAAQ,EAAE,CAAC,CAAC,QAAQ;gBACpB,QAAQ;gBACR,UAAU;gBACV,SAAS,EAAE,MAAM,CAAC,IAAI,CAAC,SAAS,IAAI,EAAE,CAAC;gBACvC,cAAc;aACf,CAAC;YACF,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACrB,QAAQ,EAAE,CAAC,MAAM,CAAC,CAAC;QACrB,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,MAAM,GAAqB;gBAC/B,MAAM,EAAE,CAAC,CAAC,MAAM;gBAChB,QAAQ,EAAE,CAAC,CAAC,QAAQ;gBACpB,QAAQ,EAAE,KAAK;gBACf,UAAU,EAAE,KAAK;gBACjB,SAAS,EAAE,UAAU,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE;gBACvE,cAAc,EAAE,CAAC,CAAC,QAAQ,KAAK,iBAAiB,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI;aAC/D,CAAC;YACF,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACrB,QAAQ,EAAE,CAAC,MAAM,CAAC,CAAC;QACrB,CAAC;IACH,CAAC;IAED,OAAO,cAAc,CAAC,OAAO,CAAC,CAAC;AACjC,CAAC;AAED,SAAS,cAAc,CACrB,QAAmD,EACnD,MAAe;IAEf,IAAI,QAAQ,KAAK,iBAAiB,IAAI,MAAM;QAAE,OAAO,IAAI,CAAC;IAC1D,IAAI,QAAQ,KAAK,iBAAiB,IAAI,CAAC,MAAM;QAAE,OAAO,IAAI,CAAC;IAC3D,IAAI,QAAQ,KAAK,qBAAqB,IAAI,CAAC,MAAM;QAAE,OAAO,IAAI,CAAC;IAC/D,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,cAAc,CAAC,OAA2B;IACjD,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,cAAc,KAAK,IAAI,CAAC,CAAC,MAAM,CAAC;IACnE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,cAAc,KAAK,IAAI,CAAC,CAAC,MAAM,CAAC;IACnE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,cAAc,KAAK,IAAI,CAAC,CAAC,MAAM,CAAC;IACnE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,cAAc,KAAK,IAAI,CAAC,CAAC,MAAM,CAAC;IACnE,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC;IAE7B,OAAO;QACL,OAAO;QACP,SAAS,EAAE,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QAC3C,MAAM,EAAE,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QACxC,WAAW,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;QAC9C,KAAK;QACL,EAAE;QACF,EAAE;QACF,EAAE;QACF,EAAE;KACH,CAAC;AACJ,CAAC"}
|
package/dist/eval/benchmark.d.ts
CHANGED
|
@@ -10,6 +10,8 @@ export interface BenchmarkCase {
|
|
|
10
10
|
status: "pass" | "fail" | "error";
|
|
11
11
|
error_message: string | null;
|
|
12
12
|
pass_rate: number;
|
|
13
|
+
durationMs?: number;
|
|
14
|
+
tokens?: number | null;
|
|
13
15
|
assertions: BenchmarkAssertionResult[];
|
|
14
16
|
}
|
|
15
17
|
export interface BenchmarkResult {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"benchmark.js","sourceRoot":"","sources":["../../src/eval/benchmark.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,4BAA4B;AAC5B,8EAA8E;AAE9E,OAAO,EAAE,aAAa,EAAE,YAAY,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAC7E,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;
|
|
1
|
+
{"version":3,"file":"benchmark.js","sourceRoot":"","sources":["../../src/eval/benchmark.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,4BAA4B;AAC5B,8EAA8E;AAE9E,OAAO,EAAE,aAAa,EAAE,YAAY,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAC7E,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AA4BjC,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,QAAgB,EAChB,MAAuB;IAEvB,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IACzC,SAAS,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACzC,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,EAAE,gBAAgB,CAAC,CAAC;IAClD,aAAa,CAAC,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;AACpE,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,QAAgB;IAEhB,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,EAAE,OAAO,EAAE,gBAAgB,CAAC,CAAC;IAC3D,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC;QAAE,OAAO,IAAI,CAAC;IAEvC,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAC5C,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAoB,CAAC;IAC5C,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC"}
|
|
@@ -2,8 +2,10 @@ import type { LlmClient } from "./llm.js";
|
|
|
2
2
|
export interface ComparisonOutput {
|
|
3
3
|
skillOutput: string;
|
|
4
4
|
skillDurationMs: number;
|
|
5
|
+
skillTokens: number | null;
|
|
5
6
|
baselineOutput: string;
|
|
6
7
|
baselineDurationMs: number;
|
|
8
|
+
baselineTokens: number | null;
|
|
7
9
|
}
|
|
8
10
|
export interface ComparisonScore {
|
|
9
11
|
contentScoreA: number;
|
|
@@ -16,8 +18,10 @@ export interface ComparisonResult {
|
|
|
16
18
|
prompt: string;
|
|
17
19
|
skillOutput: string;
|
|
18
20
|
skillDurationMs: number;
|
|
21
|
+
skillTokens: number | null;
|
|
19
22
|
baselineOutput: string;
|
|
20
23
|
baselineDurationMs: number;
|
|
24
|
+
baselineTokens: number | null;
|
|
21
25
|
skillContentScore: number;
|
|
22
26
|
skillStructureScore: number;
|
|
23
27
|
baselineContentScore: number;
|
package/dist/eval/comparator.js
CHANGED
|
@@ -22,13 +22,17 @@ export async function generateComparisonOutputs(prompt, skillContent, client) {
|
|
|
22
22
|
const skillSystemPrompt = `You are an AI assistant enhanced with the following skill:\n\n${skillContent}`;
|
|
23
23
|
const baselineSystemPrompt = "You are a helpful AI assistant.";
|
|
24
24
|
// Run sequentially (claude-cli can only handle one at a time)
|
|
25
|
-
const
|
|
26
|
-
const
|
|
27
|
-
const
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
25
|
+
const skillResult = await client.generate(skillSystemPrompt, prompt);
|
|
26
|
+
const baselineResult = await client.generate(baselineSystemPrompt, prompt);
|
|
27
|
+
const totalTokens = (r) => r.inputTokens != null && r.outputTokens != null ? r.inputTokens + r.outputTokens : null;
|
|
28
|
+
return {
|
|
29
|
+
skillOutput: skillResult.text,
|
|
30
|
+
skillDurationMs: skillResult.durationMs,
|
|
31
|
+
skillTokens: totalTokens(skillResult),
|
|
32
|
+
baselineOutput: baselineResult.text,
|
|
33
|
+
baselineDurationMs: baselineResult.durationMs,
|
|
34
|
+
baselineTokens: totalTokens(baselineResult),
|
|
35
|
+
};
|
|
32
36
|
}
|
|
33
37
|
export async function scoreComparison(outputA, outputB, prompt, client) {
|
|
34
38
|
const userPrompt = `## Original Prompt
|
|
@@ -41,7 +45,7 @@ ${outputA}
|
|
|
41
45
|
${outputB}
|
|
42
46
|
|
|
43
47
|
Evaluate both responses.`;
|
|
44
|
-
const response = await client.generate(COMPARATOR_SYSTEM_PROMPT, userPrompt);
|
|
48
|
+
const { text: response } = await client.generate(COMPARATOR_SYSTEM_PROMPT, userPrompt);
|
|
45
49
|
// Parse JSON from response (may be in code fence)
|
|
46
50
|
const jsonMatch = response.match(/```(?:json)?\s*([\s\S]*?)```/) || [null, response];
|
|
47
51
|
const json = JSON.parse(jsonMatch[1].trim());
|
|
@@ -82,8 +86,10 @@ export async function runComparison(prompt, skillContent, client) {
|
|
|
82
86
|
prompt,
|
|
83
87
|
skillOutput: outputs.skillOutput,
|
|
84
88
|
skillDurationMs: outputs.skillDurationMs,
|
|
89
|
+
skillTokens: outputs.skillTokens,
|
|
85
90
|
baselineOutput: outputs.baselineOutput,
|
|
86
91
|
baselineDurationMs: outputs.baselineDurationMs,
|
|
92
|
+
baselineTokens: outputs.baselineTokens,
|
|
87
93
|
skillContentScore,
|
|
88
94
|
skillStructureScore,
|
|
89
95
|
baselineContentScore,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"comparator.js","sourceRoot":"","sources":["../../src/eval/comparator.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,6DAA6D;AAC7D,8EAA8E;
|
|
1
|
+
{"version":3,"file":"comparator.js","sourceRoot":"","sources":["../../src/eval/comparator.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,6DAA6D;AAC7D,8EAA8E;AAoC9E,MAAM,wBAAwB,GAAG;;;;;;;;;;;;;;;;EAgB/B,CAAC;AAEH,MAAM,CAAC,KAAK,UAAU,yBAAyB,CAC7C,MAAc,EACd,YAAoB,EACpB,MAAiB;IAEjB,MAAM,iBAAiB,GAAG,iEAAiE,YAAY,EAAE,CAAC;IAC1G,MAAM,oBAAoB,GAAG,iCAAiC,CAAC;IAE/D,8DAA8D;IAC9D,MAAM,WAAW,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC,iBAAiB,EAAE,MAAM,CAAC,CAAC;IACrE,MAAM,cAAc,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC,oBAAoB,EAAE,MAAM,CAAC,CAAC;IAE3E,MAAM,WAAW,GAAG,CAAC,CAAqB,EAAE,EAAE,CAC5C,CAAC,CAAC,WAAW,IAAI,IAAI,IAAI,CAAC,CAAC,YAAY,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,WAAW,GAAG,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC;IAE1F,OAAO;QACL,WAAW,EAAE,WAAW,CAAC,IAAI;QAC7B,eAAe,EAAE,WAAW,CAAC,UAAU;QACvC,WAAW,EAAE,WAAW,CAAC,WAAW,CAAC;QACrC,cAAc,EAAE,cAAc,CAAC,IAAI;QACnC,kBAAkB,EAAE,cAAc,CAAC,UAAU;QAC7C,cAAc,EAAE,WAAW,CAAC,cAAc,CAAC;KAC5C,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,OAAe,EACf,OAAe,EACf,MAAc,EACd,MAAiB;IAEjB,MAAM,UAAU,GAAG;EACnB,MAAM;;;EAGN,OAAO;;;EAGP,OAAO;;yBAEgB,CAAC;IAExB,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC,wBAAwB,EAAE,UAAU,CAAC,CAAC;IAEvF,kDAAkD;IAClD,MAAM,SAAS,GAAG,QAAQ,CAAC,KAAK,CAAC,8BAA8B,CAAC,IAAI,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;IACrF,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAE,CAAC,IAAI,EAAE,CAAC,CAAC;IAE9C,OAAO;QACL,aAAa,EAAE,UAAU,CAAC,IAAI,CAAC,eAAe,CAAC;QAC/C,eAAe,EAAE,UAAU,CAAC,IAAI,CAAC,iBAAiB,CAAC;QACnD,aAAa,EAAE,UAAU,CAAC,IAAI,CAAC,eAAe,CAAC;QAC/C,eAAe,EAAE,UAAU,CAAC,IAAI,CAAC,iBAAiB,CAAC;QACnD,MAAM,EAAE,IAAI,CAAC,MAAM,KAAK,OAAO,IAAI,IAAI,CAAC,MAAM,KAAK,QAAQ,IAAI,IAAI,CAAC,MAAM,KAAK,KAAK;YAClF,CAAC,CAAC,IAAI,CAAC,MAAM;YACb,CAAC,CAAC,KAAK;KACV,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,MAAc,EACd,YAAoB,EACpB,MAAiB;IAEjB,MAAM,OAAO,GAAG,MAAM,yBAAyB,CAAC,MAAM,EAAE,YAAY,EAAE,MAAM,CAAC,CAAC;IAE9E,uCAAuC;IACvC,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,EAAE,GAAG,GAAG,CAAC;IACrC,MAAM,OAAO,GAAG,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC,OAAO,CAAC,cAAc,CAAC;IACxE,MAAM,OAAO,GAAG,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,OAAO,CAAC,WAAW,CAAC;IAExE,MAAM,MAAM,GAAG,MAAM,eAAe,CAAC,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;IAEvE,oCAAoC;IACpC,MAAM,iBAAiB,GAAG,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC;IACjF,MAAM,mBAAmB,GAAG,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC,CAAC,MAAM,CAAC,eAAe,CAAC;IACvF,MAAM,oBAAoB,GAAG,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC;IACpF,MAAM,sBAAsB,GAAG,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC,CAAC,MAAM,CAAC,eAAe,CAAC;IAE1F,kBAAkB;IAClB,IAAI,MAAoC,CAAC;IACzC,IAAI,MAAM,CAAC,MAAM,KAAK,KAAK,EAAE,CAAC;QAC5B,MAAM,GAAG,KAAK,CAAC;IACjB,CAAC;SAAM,IAAI,MAAM,CAAC,MAAM,KAAK,OAAO,EAAE,CAAC;QACrC,MAAM,GAAG,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,UAAU,CAAC;IAC3C,CAAC;SAAM,CAAC;QACN,MAAM,GAAG,QAAQ,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,OAAO,CAAC;IAC3C,CAAC;IAED,OAAO;QACL,MAAM;QACN,WAAW,EAAE,OAAO,CAAC,WAAW;QAChC,eAAe,EAAE,OAAO,CAAC,eAAe;QACxC,WAAW,EAAE,OAAO,CAAC,WAAW;QAChC,cAAc,EAAE,OAAO,CAAC,cAAc;QACtC,kBAAkB,EAAE,OAAO,CAAC,kBAAkB;QAC9C,cAAc,EAAE,OAAO,CAAC,cAAc;QACtC,iBAAiB;QACjB,mBAAmB;QACnB,oBAAoB;QACpB,sBAAsB;QACtB,MAAM;KACP,CAAC;AACJ,CAAC;AAED,SAAS,UAAU,CAAC,GAAY;IAC9B,MAAM,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC;IACtB,IAAI,KAAK,CAAC,CAAC,CAAC;QAAE,OAAO,CAAC,CAAC;IACvB,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AACjD,CAAC"}
|
package/dist/eval/judge.js
CHANGED
|
@@ -10,7 +10,7 @@ ${output}
|
|
|
10
10
|
${assertion.text}
|
|
11
11
|
|
|
12
12
|
Does the LLM output satisfy this assertion? Respond with JSON only: { "pass": boolean, "reasoning": "..." }`;
|
|
13
|
-
const raw = await client.generate(JUDGE_SYSTEM, userPrompt);
|
|
13
|
+
const { text: raw } = await client.generate(JUDGE_SYSTEM, userPrompt);
|
|
14
14
|
const parsed = parseJudgeResponse(raw);
|
|
15
15
|
return {
|
|
16
16
|
id: assertion.id,
|
package/dist/eval/judge.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"judge.js","sourceRoot":"","sources":["../../src/eval/judge.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,4BAA4B;AAC5B,8EAA8E;AAY9E,MAAM,YAAY,GAAG,qNAAqN,CAAC;AAE3O,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,MAAc,EACd,SAAoB,EACpB,MAAiB;IAEjB,MAAM,UAAU,GAAG;EACnB,MAAM;;;EAGN,SAAS,CAAC,IAAI;;4GAE4F,CAAC;IAE3G,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC,YAAY,EAAE,UAAU,CAAC,CAAC;
|
|
1
|
+
{"version":3,"file":"judge.js","sourceRoot":"","sources":["../../src/eval/judge.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,4BAA4B;AAC5B,8EAA8E;AAY9E,MAAM,YAAY,GAAG,qNAAqN,CAAC;AAE3O,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,MAAc,EACd,SAAoB,EACpB,MAAiB;IAEjB,MAAM,UAAU,GAAG;EACnB,MAAM;;;EAGN,SAAS,CAAC,IAAI;;4GAE4F,CAAC;IAE3G,MAAM,EAAE,IAAI,EAAE,GAAG,EAAE,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC,YAAY,EAAE,UAAU,CAAC,CAAC;IAEtE,MAAM,MAAM,GAAG,kBAAkB,CAAC,GAAG,CAAC,CAAC;IAEvC,OAAO;QACL,EAAE,EAAE,SAAS,CAAC,EAAE;QAChB,IAAI,EAAE,SAAS,CAAC,IAAI;QACpB,IAAI,EAAE,MAAM,CAAC,IAAI;QACjB,SAAS,EAAE,MAAM,CAAC,SAAS;KAC5B,CAAC;AACJ,CAAC;AAED,SAAS,kBAAkB,CAAC,GAAW;IACrC,4CAA4C;IAC5C,MAAM,UAAU,GAAG,GAAG,CAAC,KAAK,CAAC,kCAAkC,CAAC,CAAC;IACjE,MAAM,OAAO,GAAG,UAAU,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;IAEjD,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QACnC,IAAI,OAAO,MAAM,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;YACrC,MAAM,IAAI,KAAK,CAAC,oBAAoB,CAAC,CAAC;QACxC,CAAC;QACD,OAAO;YACL,IAAI,EAAE,MAAM,CAAC,IAAI;YACjB,SAAS,EAAE,OAAO,MAAM,CAAC,SAAS,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE;SACxE,CAAC;IACJ,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,KAAK,CACb,sEAAsE,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAC1F,CAAC;IACJ,CAAC;AACH,CAAC"}
|
package/dist/eval/llm.d.ts
CHANGED
|
@@ -1,5 +1,11 @@
|
|
|
1
|
+
export interface GenerateResult {
|
|
2
|
+
text: string;
|
|
3
|
+
durationMs: number;
|
|
4
|
+
inputTokens: number | null;
|
|
5
|
+
outputTokens: number | null;
|
|
6
|
+
}
|
|
1
7
|
export interface LlmClient {
|
|
2
|
-
generate(systemPrompt: string, userPrompt: string): Promise<
|
|
8
|
+
generate(systemPrompt: string, userPrompt: string): Promise<GenerateResult>;
|
|
3
9
|
readonly model: string;
|
|
4
10
|
}
|
|
5
11
|
export type ProviderName = "anthropic" | "claude-cli" | "ollama";
|
package/dist/eval/llm.js
CHANGED
|
@@ -59,6 +59,7 @@ function createAnthropicClient(modelOverride) {
|
|
|
59
59
|
}
|
|
60
60
|
const controller = new AbortController();
|
|
61
61
|
const timeout = setTimeout(() => controller.abort(), 120_000);
|
|
62
|
+
const start = Date.now();
|
|
62
63
|
try {
|
|
63
64
|
const response = await clientInstance.messages.create({
|
|
64
65
|
model,
|
|
@@ -66,8 +67,15 @@ function createAnthropicClient(modelOverride) {
|
|
|
66
67
|
messages: [{ role: "user", content: userPrompt }],
|
|
67
68
|
max_tokens: 4096,
|
|
68
69
|
}, { signal: controller.signal });
|
|
70
|
+
const durationMs = Date.now() - start;
|
|
69
71
|
const textBlock = response.content.find((b) => b.type === "text");
|
|
70
|
-
|
|
72
|
+
const text = textBlock && "text" in textBlock ? textBlock.text : "";
|
|
73
|
+
return {
|
|
74
|
+
text,
|
|
75
|
+
durationMs,
|
|
76
|
+
inputTokens: response.usage?.input_tokens ?? null,
|
|
77
|
+
outputTokens: response.usage?.output_tokens ?? null,
|
|
78
|
+
};
|
|
71
79
|
}
|
|
72
80
|
finally {
|
|
73
81
|
clearTimeout(timeout);
|
|
@@ -92,7 +100,8 @@ function createClaudeCliClient(modelOverride, forced = false) {
|
|
|
92
100
|
model: `claude-${model}`,
|
|
93
101
|
async generate(systemPrompt, userPrompt) {
|
|
94
102
|
const combinedPrompt = `${systemPrompt}\n\n${userPrompt}`;
|
|
95
|
-
|
|
103
|
+
const start = Date.now();
|
|
104
|
+
const text = await new Promise((resolve, reject) => {
|
|
96
105
|
// Strip all CLAUDE* env vars so the child process doesn't detect nesting
|
|
97
106
|
const cleanEnv = {};
|
|
98
107
|
for (const [k, v] of Object.entries(process.env)) {
|
|
@@ -133,6 +142,7 @@ function createClaudeCliClient(modelOverride, forced = false) {
|
|
|
133
142
|
// Pipe prompt via stdin — avoids ARG_MAX limits for large SKILL.md files
|
|
134
143
|
proc.stdin.end(combinedPrompt);
|
|
135
144
|
});
|
|
145
|
+
return { text, durationMs: Date.now() - start, inputTokens: null, outputTokens: null };
|
|
136
146
|
},
|
|
137
147
|
};
|
|
138
148
|
}
|
|
@@ -146,6 +156,7 @@ function createOllamaClient(modelOverride) {
|
|
|
146
156
|
model,
|
|
147
157
|
async generate(systemPrompt, userPrompt) {
|
|
148
158
|
const fullPrompt = `${systemPrompt}\n\n${userPrompt}`;
|
|
159
|
+
const start = Date.now();
|
|
149
160
|
const response = await fetch(`${baseUrl}/api/generate`, {
|
|
150
161
|
method: "POST",
|
|
151
162
|
headers: { "Content-Type": "application/json" },
|
|
@@ -168,7 +179,12 @@ function createOllamaClient(modelOverride) {
|
|
|
168
179
|
throw new Error(`Ollama request failed: ${error}`);
|
|
169
180
|
}
|
|
170
181
|
const data = (await response.json());
|
|
171
|
-
return
|
|
182
|
+
return {
|
|
183
|
+
text: data.response || "",
|
|
184
|
+
durationMs: Date.now() - start,
|
|
185
|
+
inputTokens: data.prompt_eval_count ?? null,
|
|
186
|
+
outputTokens: data.eval_count ?? null,
|
|
187
|
+
};
|
|
172
188
|
},
|
|
173
189
|
};
|
|
174
190
|
}
|
package/dist/eval/llm.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"llm.js","sourceRoot":"","sources":["../../src/eval/llm.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,gFAAgF;AAChF,EAAE;AACF,uDAAuD;AACvD,wEAAwE;AACxE,8DAA8D;AAC9D,uEAAuE;AACvE,EAAE;AACF,uDAAuD;AACvD,4DAA4D;AAC5D,6CAA6C;AAC7C,2DAA2D;AAC3D,EAAE;AACF,iDAAiD;AACjD,+DAA+D;AAC/D,mEAAmE;AACnE,mDAAmD;AACnD,8EAA8E;AAE9E,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;
|
|
1
|
+
{"version":3,"file":"llm.js","sourceRoot":"","sources":["../../src/eval/llm.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,gFAAgF;AAChF,EAAE;AACF,uDAAuD;AACvD,wEAAwE;AACxE,8DAA8D;AAC9D,uEAAuE;AACvE,EAAE;AACF,uDAAuD;AACvD,4DAA4D;AAC5D,6CAA6C;AAC7C,2DAA2D;AAC3D,EAAE;AACF,iDAAiD;AACjD,+DAA+D;AAC/D,mEAAmE;AACnE,mDAAmD;AACnD,8EAA8E;AAE9E,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAgB3C,SAAS,cAAc;IACrB,IAAI,OAAO,CAAC,GAAG,CAAC,UAAU;QAAE,OAAO,QAAQ,CAAC;IAC5C,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB;QAAE,OAAO,WAAW,CAAC;IACtD,OAAO,YAAY,CAAC;AACtB,CAAC;AAOD,MAAM,UAAU,eAAe,CAAC,SAAwB;IACtD,MAAM,QAAQ,GAAG,CAAC,SAAS,EAAE,QAAQ,IAAI,OAAO,CAAC,GAAG,CAAC,oBAAoB,IAAI,cAAc,EAAE,CAAiB,CAAC;IAC/G,MAAM,aAAa,GAAG,SAAS,EAAE,KAAK,CAAC;IACvC,MAAM,MAAM,GAAG,CAAC,CAAC,SAAS,EAAE,QAAQ,CAAC;IAErC,QAAQ,QAAQ,EAAE,CAAC;QACjB,KAAK,WAAW;YACd,OAAO,qBAAqB,CAAC,aAAa,CAAC,CAAC;QAC9C,KAAK,YAAY;YACf,OAAO,qBAAqB,CAAC,aAAa,EAAE,MAAM,CAAC,CAAC;QACtD,KAAK,QAAQ;YACX,OAAO,kBAAkB,CAAC,aAAa,CAAC,CAAC;QAC3C;YACE,MAAM,IAAI,KAAK,CACb,kCAAkC,QAAQ,gDAAgD,CAC3F,CAAC;IACN,CAAC;AACH,CAAC;AAED,8EAA8E;AAC9E,0BAA0B;AAC1B,8EAA8E;AAC9E,SAAS,qBAAqB,CAAC,aAAsB;IACnD,MAAM,aAAa,GAAG,0BAA0B,CAAC;IAEjD,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC;IAC7C,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,MAAM,IAAI,KAAK,CACb,sRAAsR,CACvR,CAAC;IACJ,CAAC;IAED,MAAM,KAAK,GAAG,aAAa,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,aAAa,CAAC;IAC9E,IAAI,cAAc,GAAQ,IAAI,CAAC;IAE/B,OAAO;QACL,KAAK;QACL,KAAK,CAAC,QAAQ,CAAC,YAAoB,EAAE,UAAkB;YACrD,IAAI,CAAC,cAAc,EAAE,CAAC;gBACpB,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,GAAG,MAAM,MAAM,CAAC,mBAAmB,CAAC,CAAC;gBACjE,cAAc,GAAG,IAAI,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;YAC7C,CAAC;YAED,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;YACzC,MAAM,OAAO,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,OAAO,CAAC,CAAC;YAC9D,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YACzB,IAAI,CAAC;gBACH,MAAM,QAAQ,GAAG,MAAM,cAAc,CAAC,QAAQ,CAAC,MAAM,CACnD;oBACE,KAAK;oBACL,MAAM,EAAE,YAAY;oBACpB,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,CAAC;oBACjD,UAAU,EAAE,IAAI;iBACjB,EACD,EAAE,MAAM,EAAE,UAAU,CAAC,MAAM,EAAE,CAC9B,CAAC;gBACF,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC;gBAEtC,MAAM,SAAS,GAAG,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC;gBACvE,MAAM,IAAI,GAAG,SAAS,IAAI,MAAM,IAAI,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;gBACpE,OAAO;oBACL,IAAI;oBACJ,UAAU;oBACV,WAAW,EAAE,QAAQ,CAAC,KAAK,EAAE,YAAY,IAAI,IAAI;oBACjD,YAAY,EAAE,QAAQ,CAAC,KAAK,EAAE,aAAa,IAAI,IAAI;iBACpD,CAAC;YACJ,CAAC;oBAAS,CAAC;gBACT,YAAY,CAAC,OAAO,CAAC,CAAC;YACxB,CAAC;QACH,CAAC;KACF,CAAC;AACJ,CAAC;AAED,8EAA8E;AAC9E,4EAA4E;AAC5E,EAAE;AACF,uEAAuE;AACvE,EAAE;AACF,6DAA6D;AAC7D,oFAAoF;AACpF,8EAA8E;AAC9E,SAAS,qBAAqB,CAAC,aAAsB,EAAE,MAAM,GAAG,KAAK;IACnE,IAAI,OAAO,CAAC,GAAG,CAAC,UAAU,IAAI,CAAC,MAAM,EAAE,CAAC;QACtC,MAAM,IAAI,KAAK,CACb,6JAA6J,CAC9J,CAAC;IACJ,CAAC;IAED,MAAM,KAAK,GAAG,aAAa,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,QAAQ,CAAC;IAEzE,OAAO;QACL,KAAK,EAAE,UAAU,KAAK,EAAE;QACxB,KAAK,CAAC,QAAQ,CAAC,YAAoB,EAAE,UAAkB;YACrD,MAAM,cAAc,GAAG,GAAG,YAAY,OAAO,UAAU,EAAE,CAAC;YAC1D,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAEzB,MAAM,IAAI,GAAG,MAAM,IAAI,OAAO,CAAS,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;gBACzD,yEAAyE;gBACzE,MAAM,QAAQ,GAA2B,EAAE,CAAC;gBAC5C,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;oBACjD,IAAI,CAAC,KAAK,SAAS,IAAI,CAAC,CAAC,CAAC,UAAU,CAAC,QAAQ,CAAC;wBAAE,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;gBAClE,CAAC;gBACD,MAAM,IAAI,GAAG,KAAK,CAAC,QAAQ,EAAE,CAAC,IAAI,EAAE,SAAS,EAAE,KAAK,CAAC,EAAE;oBACrD,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC;oBAC/B,GAAG,EAAE,QAAQ;iBACd,CAAC,CAAC;gBAEH,IAAI,MAAM,GAAG,EAAE,CAAC;gBAChB,IAAI,MAAM,GAAG,EAAE,CAAC;gBAChB,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,CAAS,EAAE,EAAE,GAAG,MAAM,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;gBACnE,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,CAAS,EAAE,EAAE,GAAG,MAAM,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;gBAEnE,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE;oBAC5B,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;oBACrB,MAAM,CAAC,IAAI,KAAK,CAAC,iCAAiC,CAAC,CAAC,CAAC;gBACvD,CAAC,EAAE,OAAO,CAAC,CAAC;gBAEZ,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAA0B,EAAE,EAAE;oBAC9C,YAAY,CAAC,KAAK,CAAC,CAAC;oBACpB,IAAI,GAAG,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;wBAC1B,MAAM,CAAC,IAAI,KAAK,CACd,qJAAqJ,CACtJ,CAAC,CAAC;oBACL,CAAC;yBAAM,CAAC;wBACN,MAAM,CAAC,IAAI,KAAK,CAAC,sBAAsB,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;oBACzD,CAAC;gBACH,CAAC,CAAC,CAAC;gBAEH,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,EAAE;oBACxB,YAAY,CAAC,KAAK,CAAC,CAAC;oBACpB,IAAI,IAAI,KAAK,CAAC,EAAE,CAAC;wBACf,OAAO,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;oBACzB,CAAC;yBAAM,CAAC;wBACN,MAAM,MAAM,GAAG,CAAC,MAAM,IAAI,MAAM,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;wBAChD,MAAM,CAAC,IAAI,KAAK,CAAC,+BAA+B,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;oBACzF,CAAC;gBACH,CAAC,CAAC,CAAC;gBAEH,yEAAyE;gBACzE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;YACjC,CAAC,CAAC,CAAC;YAEH,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,EAAE,WAAW,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC;QACzF,CAAC;KACF,CAAC;AACJ,CAAC;AAED,8EAA8E;AAC9E,qDAAqD;AACrD,8EAA8E;AAC9E,SAAS,kBAAkB,CAAC,aAAsB;IAChD,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,eAAe,IAAI,wBAAwB,CAAC;IACxE,MAAM,KAAK,GAAG,aAAa,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,aAAa,CAAC;IAE9E,OAAO;QACL,KAAK;QACL,KAAK,CAAC,QAAQ,CAAC,YAAoB,EAAE,UAAkB;YACrD,MAAM,UAAU,GAAG,GAAG,YAAY,OAAO,UAAU,EAAE,CAAC;YACtD,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAEzB,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,OAAO,eAAe,EAAE;gBACtD,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE;gBAC/C,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;oBACnB,KAAK;oBACL,MAAM,EAAE,UAAU;oBAClB,MAAM,EAAE,KAAK;oBACb,OAAO,EAAE;wBACP,WAAW,EAAE,IAAI;wBACjB,WAAW,EAAE,GAAG;qBACjB;iBACF,CAAC;gBACF,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,OAAO,CAAC;aACrC,CAAC,CAAC;YAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;gBACjB,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;gBACpC,IAAI,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;oBAC3D,MAAM,IAAI,KAAK,CACb,iBAAiB,KAAK,8CAA8C,KAAK,EAAE,CAC5E,CAAC;gBACJ,CAAC;gBACD,MAAM,IAAI,KAAK,CAAC,0BAA0B,KAAK,EAAE,CAAC,CAAC;YACrD,CAAC;YAED,MAAM,IAAI,GAAG,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAIlC,CAAC;YACF,OAAO;gBACL,IAAI,EAAE,IAAI,CAAC,QAAQ,IAAI,EAAE;gBACzB,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;gBAC9B,WAAW,EAAE,IAAI,CAAC,iBAAiB,IAAI,IAAI;gBAC3C,YAAY,EAAE,IAAI,CAAC,UAAU,IAAI,IAAI;aACtC,CAAC;QACJ,CAAC;KACF,CAAC;AACJ,CAAC"}
|