vskill 0.2.75 → 0.2.77
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +27 -0
- package/dist/commands/eval/__tests__/coverage.test.js +1 -1
- package/dist/commands/eval/__tests__/coverage.test.js.map +1 -1
- package/dist/commands/eval/serve.js +10 -4
- package/dist/commands/eval/serve.js.map +1 -1
- package/dist/eval/__tests__/benchmark.test.js +2 -2
- package/dist/eval/__tests__/benchmark.test.js.map +1 -1
- package/dist/eval/__tests__/llm.test.js +7 -7
- package/dist/eval/__tests__/llm.test.js.map +1 -1
- package/dist/eval/__tests__/mcp-detector.test.d.ts +1 -0
- package/dist/eval/__tests__/mcp-detector.test.js +125 -0
- package/dist/eval/__tests__/mcp-detector.test.js.map +1 -0
- package/dist/eval/__tests__/skill-scanner.test.js +2 -1
- package/dist/eval/__tests__/skill-scanner.test.js.map +1 -1
- package/dist/eval/benchmark-history.d.ts +31 -4
- package/dist/eval/benchmark-history.js +83 -4
- package/dist/eval/benchmark-history.js.map +1 -1
- package/dist/eval/benchmark.d.ts +32 -0
- package/dist/eval/benchmark.js.map +1 -1
- package/dist/eval/llm.js +2 -2
- package/dist/eval/llm.js.map +1 -1
- package/dist/eval/mcp-detector.d.ts +13 -0
- package/dist/eval/mcp-detector.js +123 -0
- package/dist/eval/mcp-detector.js.map +1 -0
- package/dist/eval/skill-scanner.js +14 -2
- package/dist/eval/skill-scanner.js.map +1 -1
- package/dist/eval-server/api-routes.js +183 -117
- package/dist/eval-server/api-routes.js.map +1 -1
- package/dist/eval-server/benchmark-runner.d.ts +16 -0
- package/dist/eval-server/benchmark-runner.js +114 -0
- package/dist/eval-server/benchmark-runner.js.map +1 -0
- package/dist/eval-server/eval-server.js +4 -0
- package/dist/eval-server/eval-server.js.map +1 -1
- package/dist/eval-server/improve-routes.d.ts +2 -0
- package/dist/eval-server/improve-routes.js +77 -0
- package/dist/eval-server/improve-routes.js.map +1 -0
- package/dist/eval-server/model-compare-routes.d.ts +2 -0
- package/dist/eval-server/model-compare-routes.js +119 -0
- package/dist/eval-server/model-compare-routes.js.map +1 -0
- package/dist/eval-server/skill-resolver.d.ts +1 -0
- package/dist/eval-server/skill-resolver.js +24 -0
- package/dist/eval-server/skill-resolver.js.map +1 -0
- package/dist/eval-ui/assets/index-CFJYUUyc.js +73 -0
- package/dist/eval-ui/assets/index-DSspu3L6.css +1 -0
- package/dist/eval-ui/index.html +2 -2
- package/dist/installer/canonical.js +1 -1
- package/dist/installer/canonical.js.map +1 -1
- package/dist/installer/canonical.test.js +8 -0
- package/dist/installer/canonical.test.js.map +1 -1
- package/package.json +1 -1
- package/dist/eval-ui/assets/index-BsNUxjb1.js +0 -70
- package/dist/eval-ui/assets/index-D5mEzX7i.css +0 -1
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
// ---------------------------------------------------------------------------
|
|
2
2
|
// benchmark-history.ts -- timestamped benchmark history with regression diffing
|
|
3
3
|
// ---------------------------------------------------------------------------
|
|
4
|
-
import { readdir, readFile, mkdir, writeFile } from "node:fs/promises";
|
|
4
|
+
import { readdir, readFile, mkdir, writeFile, unlink } from "node:fs/promises";
|
|
5
5
|
import { join } from "node:path";
|
|
6
6
|
import { writeBenchmark } from "./benchmark.js";
|
|
7
7
|
function toFilesafeTimestamp(iso) {
|
|
@@ -24,7 +24,18 @@ export async function writeHistoryEntry(skillDir, result) {
|
|
|
24
24
|
await writeBenchmark(skillDir, result);
|
|
25
25
|
return filename;
|
|
26
26
|
}
|
|
27
|
-
export async function
|
|
27
|
+
export async function deleteHistoryEntry(skillDir, timestamp) {
|
|
28
|
+
const historyDir = join(skillDir, "evals", "history");
|
|
29
|
+
const filename = `${toFilesafeTimestamp(timestamp)}.json`;
|
|
30
|
+
try {
|
|
31
|
+
await unlink(join(historyDir, filename));
|
|
32
|
+
return true;
|
|
33
|
+
}
|
|
34
|
+
catch {
|
|
35
|
+
return false;
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
export async function listHistory(skillDir, filter) {
|
|
28
39
|
const historyDir = join(skillDir, "evals", "history");
|
|
29
40
|
let files;
|
|
30
41
|
try {
|
|
@@ -33,20 +44,50 @@ export async function listHistory(skillDir) {
|
|
|
33
44
|
catch {
|
|
34
45
|
return [];
|
|
35
46
|
}
|
|
47
|
+
let jsonFiles = files.filter((f) => f.endsWith(".json")).sort().reverse();
|
|
48
|
+
// Pre-filter by date range using filename timestamps (fast, no JSON parse)
|
|
49
|
+
if (filter?.from || filter?.to) {
|
|
50
|
+
const fromSafe = filter.from ? toFilesafeTimestamp(filter.from) : undefined;
|
|
51
|
+
const toSafe = filter.to ? toFilesafeTimestamp(filter.to) : undefined;
|
|
52
|
+
jsonFiles = jsonFiles.filter((f) => {
|
|
53
|
+
const ts = f.replace(/\.json$/, "");
|
|
54
|
+
if (fromSafe && ts < fromSafe)
|
|
55
|
+
return false;
|
|
56
|
+
if (toSafe && ts > toSafe)
|
|
57
|
+
return false;
|
|
58
|
+
return true;
|
|
59
|
+
});
|
|
60
|
+
}
|
|
36
61
|
const entries = [];
|
|
37
|
-
for (const file of
|
|
62
|
+
for (const file of jsonFiles) {
|
|
38
63
|
try {
|
|
39
64
|
const content = await readFile(join(historyDir, file), "utf-8");
|
|
40
65
|
const data = JSON.parse(content);
|
|
66
|
+
const entryType = data.type || "benchmark";
|
|
67
|
+
// Post-filter by model and type
|
|
68
|
+
if (filter?.model && data.model !== filter.model)
|
|
69
|
+
continue;
|
|
70
|
+
if (filter?.type && entryType !== filter.type)
|
|
71
|
+
continue;
|
|
41
72
|
const totalAssertions = data.cases.reduce((sum, c) => sum + c.assertions.length, 0);
|
|
42
73
|
const passedAssertions = data.cases.reduce((sum, c) => sum + c.assertions.filter((a) => a.pass).length, 0);
|
|
74
|
+
const totalDurationMs = data.cases.reduce((s, c) => s + (c.durationMs ?? 0), 0);
|
|
75
|
+
const hasTokens = data.cases.some((c) => c.tokens != null);
|
|
76
|
+
const totalTokens = hasTokens
|
|
77
|
+
? data.cases.reduce((s, c) => s + (c.tokens ?? 0), 0)
|
|
78
|
+
: null;
|
|
43
79
|
entries.push({
|
|
44
80
|
timestamp: fromFilesafeTimestamp(file),
|
|
45
81
|
filename: file,
|
|
46
82
|
model: data.model,
|
|
47
83
|
skillName: data.skill_name,
|
|
48
84
|
passRate: totalAssertions > 0 ? passedAssertions / totalAssertions : 0,
|
|
49
|
-
type:
|
|
85
|
+
type: entryType,
|
|
86
|
+
caseCount: data.cases.length,
|
|
87
|
+
totalDurationMs,
|
|
88
|
+
totalTokens,
|
|
89
|
+
provider: data.provider,
|
|
90
|
+
verdict: data.verdict,
|
|
50
91
|
});
|
|
51
92
|
}
|
|
52
93
|
catch {
|
|
@@ -66,6 +107,44 @@ export async function readHistoryEntry(skillDir, timestamp) {
|
|
|
66
107
|
return null;
|
|
67
108
|
}
|
|
68
109
|
}
|
|
110
|
+
export async function getCaseHistory(skillDir, evalId, filter) {
|
|
111
|
+
const historyDir = join(skillDir, "evals", "history");
|
|
112
|
+
let files;
|
|
113
|
+
try {
|
|
114
|
+
files = await readdir(historyDir);
|
|
115
|
+
}
|
|
116
|
+
catch {
|
|
117
|
+
return [];
|
|
118
|
+
}
|
|
119
|
+
const entries = [];
|
|
120
|
+
for (const file of files.filter((f) => f.endsWith(".json")).sort().reverse()) {
|
|
121
|
+
try {
|
|
122
|
+
const content = await readFile(join(historyDir, file), "utf-8");
|
|
123
|
+
const data = JSON.parse(content);
|
|
124
|
+
if (filter?.model && data.model !== filter.model)
|
|
125
|
+
continue;
|
|
126
|
+
const matchingCase = data.cases.find((c) => c.eval_id === evalId);
|
|
127
|
+
if (!matchingCase)
|
|
128
|
+
continue;
|
|
129
|
+
entries.push({
|
|
130
|
+
timestamp: fromFilesafeTimestamp(file),
|
|
131
|
+
model: data.model,
|
|
132
|
+
type: data.type || "benchmark",
|
|
133
|
+
provider: data.provider,
|
|
134
|
+
pass_rate: matchingCase.pass_rate,
|
|
135
|
+
durationMs: matchingCase.durationMs,
|
|
136
|
+
tokens: matchingCase.tokens,
|
|
137
|
+
inputTokens: matchingCase.inputTokens,
|
|
138
|
+
outputTokens: matchingCase.outputTokens,
|
|
139
|
+
assertions: matchingCase.assertions,
|
|
140
|
+
});
|
|
141
|
+
}
|
|
142
|
+
catch {
|
|
143
|
+
// Skip malformed files
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
return entries;
|
|
147
|
+
}
|
|
69
148
|
export function computeRegressions(current, previous) {
|
|
70
149
|
const regressions = [];
|
|
71
150
|
// Build a map of previous assertion results by eval_id + assertion_id
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"benchmark-history.js","sourceRoot":"","sources":["../../src/eval/benchmark-history.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,gFAAgF;AAChF,8EAA8E;AAE9E,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;
|
|
1
|
+
{"version":3,"file":"benchmark-history.js","sourceRoot":"","sources":["../../src/eval/benchmark-history.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,gFAAgF;AAChF,8EAA8E;AAE9E,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAC/E,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAEjC,OAAO,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AA6ChD,SAAS,mBAAmB,CAAC,GAAW;IACtC,OAAO,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;AAChC,CAAC;AAED,SAAS,qBAAqB,CAAC,QAAgB;IAC7C,0CAA0C;IAC1C,MAAM,EAAE,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;IAC3C,yDAAyD;IACzD,OAAO,EAAE,CAAC,OAAO,CAAC,0BAA0B,EAAE,WAAW,CAAC,CAAC;AAC7D,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,QAAgB,EAChB,MAA4E;IAE5E,MAAM,UAAU,GAAG,IAAI,CAAC,QAAQ,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC;IACtD,MAAM,KAAK,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAE7C,MAAM,SAAS,GAAG,MAAM,CAAC,SAAS,IAAI,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAC/D,MAAM,QAAQ,GAAG,GAAG,mBAAmB,CAAC,SAAS,CAAC,OAAO,CAAC;IAC1D,MAAM,QAAQ,GAAG,IAAI,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;IAE5C,MAAM,SAAS,CAAC,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;IAE3D,uDAAuD;IACvD,MAAM,cAAc,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IAEvC,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACtC,QAAgB,EAChB,SAAiB;IAEjB,MAAM,UAAU,GAAG,IAAI,CAAC,QAAQ,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC;IACtD,MAAM,QAAQ,GAAG,GAAG,mBAAmB,CAAC,SAAS,CAAC,OAAO,CAAC;IAC1D,IAAI,CAAC;QACH,MAAM,MAAM,CAAC,IAAI,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC,CAAC;QACzC,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,QAAgB,EAChB,MAAsB;IAEtB,MAAM,UAAU,GAAG,IAAI,CAAC,QAAQ,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC;IACtD,IAAI,KAAe,CAAC;IACpB,IAAI,CAAC;QACH,KAAK,GAAG,MAAM,OAAO,CAAC,UAAU,CAAC,CAAC;IACpC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,IAAI,SAAS,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,OAAO,EAAE,CAAC;IAE1E,2EAA2E;IAC3E,IAAI,MAAM,EAAE,IAAI,IAAI,MAAM,EAAE,EAAE,EAAE,CAAC;QAC/B,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,mBAAmB,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QAC5E,MAAM,MAAM,GAAG,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,mBAAmB,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QACtE,SAAS,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE;YACjC,MAAM,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;YACpC,IAAI,QAAQ,IAAI,EAAE,GAAG,QAAQ;gBAAE,OAAO,KAAK,CAAC;YAC5C,IAAI,MAAM,IAAI,EAAE,GAAG,MAAM;gBAAE,OAAO,KAAK,CAAC;YACxC,OAAO,IAAI,CAAC;QACd,CAAC,CAAC,CAAC;IACL,CAAC;IAED,MAAM,OAAO,GAAqB,EAAE,CAAC;IACrC,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;QAC7B,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,EAAE,OAAO,CAAC,CAAC;YAChE,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAwC,CAAC;YAExE,MAAM,SAAS,GAAI,IAAI,CAAC,IAA+B,IAAI,WAAW,CAAC;YAEvE,gCAAgC;YAChC,IAAI,MAAM,EAAE,KAAK,IAAI,IAAI,CAAC,KAAK,KAAK,MAAM,CAAC,KAAK;gBAAE,SAAS;YAC3D,IAAI,MAAM,EAAE,IAAI,IAAI,SAAS,KAAK,MAAM,CAAC,IAAI;gBAAE,SAAS;YAExD,MAAM,eAAe,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;YACpF,MAAM,gBAAgB,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CACxC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EAC3D,CAAC,CACF,CAAC;YACF,MAAM,eAAe,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,UAAU,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;YAChF,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,IAAI,IAAI,CAAC,CAAC;YAC3D,MAAM,WAAW,GAAG,SAAS;gBAC3B,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC;gBACrD,CAAC,CAAC,IAAI,CAAC;YAET,OAAO,CAAC,IAAI,CAAC;gBACX,SAAS,EAAE,qBAAqB,CAAC,IAAI,CAAC;gBACtC,QAAQ,EAAE,IAAI;gBACd,KAAK,EAAE,IAAI,CAAC,KAAK;gBACjB,SAAS,EAAE,IAAI,CAAC,UAAU;gBAC1B,QAAQ,EAAE,eAAe,GAAG,CAAC,CAAC,CAAC,CAAC,gBAAgB,GAAG,eAAe,CAAC,CAAC,CAAC,CAAC;gBACtE,IAAI,EAAE,SAAS;gBACf,SAAS,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM;gBAC5B,eAAe;gBACf,WAAW;gBACX,QAAQ,EAAE,IAAI,CAAC,QAAQ;gBACvB,OAAO,EAAE,IAAI,CAAC,OAAO;aACtB,CAAC,CAAC;QACL,CAAC;QAAC,MAAM,CAAC;YACP,uBAAuB;QACzB,CAAC;IACH,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,QAAgB,EAChB,SAAiB;IAEjB,MAAM,UAAU,GAAG,IAAI,CAAC,QAAQ,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC;IACtD,MAAM,QAAQ,GAAG,GAAG,mBAAmB,CAAC,SAAS,CAAC,OAAO,CAAC;IAC1D,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,UAAU,EAAE,QAAQ,CAAC,EAAE,OAAO,CAAC,CAAC;QACpE,OAAO,IAAI,CAAC,KAAK,CAAC,OAAO,CAAoB,CAAC;IAChD,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,QAAgB,EAChB,MAAc,EACd,MAA2B;IAE3B,MAAM,UAAU,GAAG,IAAI,CAAC,QAAQ,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC;IACtD,IAAI,KAAe,CAAC;IACpB,IAAI,CAAC;QACH,KAAK,GAAG,MAAM,OAAO,CAAC,UAAU,CAAC,CAAC;IACpC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,MAAM,OAAO,GAAuB,EAAE,CAAC;IACvC,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC;QAC7E,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,EAAE,OAAO,CAAC,CAAC;YAChE,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAwC,CAAC;YAExE,IAAI,MAAM,EAAE,KAAK,IAAI,IAAI,CAAC,KAAK,KAAK,MAAM,CAAC,KAAK;gBAAE,SAAS;YAE3D,MAAM,YAAY,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,MAAM,CAAC,CAAC;YAClE,IAAI,CAAC,YAAY;gBAAE,SAAS;YAE5B,OAAO,CAAC,IAAI,CAAC;gBACX,SAAS,EAAE,qBAAqB,CAAC,IAAI,CAAC;gBACtC,KAAK,EAAE,IAAI,CAAC,KAAK;gBACjB,IAAI,EAAG,IAAI,CAAC,IAAiC,IAAI,WAAW;gBAC5D,QAAQ,EAAE,IAAI,CAAC,QAAQ;gBACvB,SAAS,EAAE,YAAY,CAAC,SAAS;gBACjC,UAAU,EAAE,YAAY,CAAC,UAAU;gBACnC,MAAM,EAAE,YAAY,CAAC,MAAM;gBAC3B,WAAW,EAAE,YAAY,CAAC,WAAW;gBACrC,YAAY,EAAE,YAAY,CAAC,YAAY;gBACvC,UAAU,EAAE,YAAY,CAAC,UAAU;aACpC,CAAC,CAAC;QACL,CAAC;QAAC,MAAM,CAAC;YACP,uBAAuB;QACzB,CAAC;IACH,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,MAAM,UAAU,kBAAkB,CAChC,OAAwB,EACxB,QAAyB;IAEzB,MAAM,WAAW,GAAsB,EAAE,CAAC;IAE1C,sEAAsE;IACtE,MAAM,OAAO,GAAG,IAAI,GAAG,EAAmB,CAAC;IAC3C,KAAK,MAAM,CAAC,IAAI,QAAQ,CAAC,KAAK,EAAE,CAAC;QAC/B,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,UAAU,EAAE,CAAC;YAC7B,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC;QAC9C,CAAC;IACH,CAAC;IAED,KAAK,MAAM,CAAC,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;QAC9B,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,UAAU,EAAE,CAAC;YAC7B,MAAM,GAAG,GAAG,GAAG,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC,EAAE,EAAE,CAAC;YACnC,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YAC9B,IAAI,IAAI,KAAK,SAAS;gBAAE,SAAS,CAAC,sBAAsB;YAExD,IAAI,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;gBACpB,WAAW,CAAC,IAAI,CAAC;oBACf,WAAW,EAAE,CAAC,CAAC,EAAE;oBACjB,MAAM,EAAE,CAAC,CAAC,OAAO;oBACjB,QAAQ,EAAE,CAAC,CAAC,SAAS;oBACrB,cAAc,EAAE,IAAI;oBACpB,aAAa,EAAE,KAAK;oBACpB,MAAM,EAAE,YAAY;iBACrB,CAAC,CAAC;YACL,CAAC;iBAAM,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC;gBAC3B,WAAW,CAAC,IAAI,CAAC;oBACf,WAAW,EAAE,CAAC,CAAC,EAAE;oBACjB,MAAM,EAAE,CAAC,CAAC,OAAO;oBACjB,QAAQ,EAAE,CAAC,CAAC,SAAS;oBACrB,cAAc,EAAE,KAAK;oBACrB,aAAa,EAAE,IAAI;oBACnB,MAAM,EAAE,aAAa;iBACtB,CAAC,CAAC;YACL,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,WAAW,CAAC;AACrB,CAAC"}
|
package/dist/eval/benchmark.d.ts
CHANGED
|
@@ -4,6 +4,21 @@ export interface BenchmarkAssertionResult {
|
|
|
4
4
|
pass: boolean;
|
|
5
5
|
reasoning: string;
|
|
6
6
|
}
|
|
7
|
+
export interface ComparisonCaseDetail {
|
|
8
|
+
skillDurationMs: number;
|
|
9
|
+
skillTokens: number | null;
|
|
10
|
+
skillInputTokens?: number | null;
|
|
11
|
+
skillOutputTokens?: number | null;
|
|
12
|
+
baselineDurationMs: number;
|
|
13
|
+
baselineTokens: number | null;
|
|
14
|
+
baselineInputTokens?: number | null;
|
|
15
|
+
baselineOutputTokens?: number | null;
|
|
16
|
+
skillContentScore: number;
|
|
17
|
+
skillStructureScore: number;
|
|
18
|
+
baselineContentScore: number;
|
|
19
|
+
baselineStructureScore: number;
|
|
20
|
+
winner: "skill" | "baseline" | "tie";
|
|
21
|
+
}
|
|
7
22
|
export interface BenchmarkCase {
|
|
8
23
|
eval_id: number;
|
|
9
24
|
eval_name: string;
|
|
@@ -12,7 +27,11 @@ export interface BenchmarkCase {
|
|
|
12
27
|
pass_rate: number;
|
|
13
28
|
durationMs?: number;
|
|
14
29
|
tokens?: number | null;
|
|
30
|
+
inputTokens?: number | null;
|
|
31
|
+
outputTokens?: number | null;
|
|
32
|
+
output?: string;
|
|
15
33
|
assertions: BenchmarkAssertionResult[];
|
|
34
|
+
comparisonDetail?: ComparisonCaseDetail;
|
|
16
35
|
}
|
|
17
36
|
export interface BenchmarkResult {
|
|
18
37
|
timestamp: string;
|
|
@@ -20,6 +39,19 @@ export interface BenchmarkResult {
|
|
|
20
39
|
skill_name: string;
|
|
21
40
|
cases: BenchmarkCase[];
|
|
22
41
|
overall_pass_rate?: number;
|
|
42
|
+
type?: "benchmark" | "comparison" | "baseline";
|
|
43
|
+
provider?: string;
|
|
44
|
+
totalDurationMs?: number;
|
|
45
|
+
totalInputTokens?: number | null;
|
|
46
|
+
totalOutputTokens?: number | null;
|
|
47
|
+
verdict?: string;
|
|
48
|
+
comparison?: {
|
|
49
|
+
skillPassRate: number;
|
|
50
|
+
baselinePassRate: number;
|
|
51
|
+
skillRubricAvg: number;
|
|
52
|
+
baselineRubricAvg: number;
|
|
53
|
+
delta: number;
|
|
54
|
+
};
|
|
23
55
|
}
|
|
24
56
|
export declare function writeBenchmark(skillDir: string, result: BenchmarkResult): Promise<void>;
|
|
25
57
|
export declare function readBenchmark(skillDir: string): Promise<BenchmarkResult | null>;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"benchmark.js","sourceRoot":"","sources":["../../src/eval/benchmark.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,4BAA4B;AAC5B,8EAA8E;AAE9E,OAAO,EAAE,aAAa,EAAE,YAAY,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAC7E,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;
|
|
1
|
+
{"version":3,"file":"benchmark.js","sourceRoot":"","sources":["../../src/eval/benchmark.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,4BAA4B;AAC5B,8EAA8E;AAE9E,OAAO,EAAE,aAAa,EAAE,YAAY,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAC7E,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AA6DjC,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,QAAgB,EAChB,MAAuB;IAEvB,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IACzC,SAAS,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACzC,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,EAAE,gBAAgB,CAAC,CAAC;IAClD,aAAa,CAAC,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;AACpE,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,QAAgB;IAEhB,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,EAAE,OAAO,EAAE,gBAAgB,CAAC,CAAC;IAC3D,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC;QAAE,OAAO,IAAI,CAAC;IAEvC,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAC5C,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAoB,CAAC;IAC5C,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC"}
|
package/dist/eval/llm.js
CHANGED
|
@@ -12,7 +12,7 @@
|
|
|
12
12
|
//
|
|
13
13
|
// Model selection via VSKILL_EVAL_MODEL env var:
|
|
14
14
|
// claude-cli: "sonnet" | "opus" | "haiku" (default: sonnet)
|
|
15
|
-
// anthropic: full model ID (default: claude-sonnet-4-
|
|
15
|
+
// anthropic: full model ID (default: claude-sonnet-4-6)
|
|
16
16
|
// ollama: model name (default: llama3.1:8b)
|
|
17
17
|
// ---------------------------------------------------------------------------
|
|
18
18
|
import { spawn } from "node:child_process";
|
|
@@ -37,7 +37,7 @@ export function createLlmClient(overrides) {
|
|
|
37
37
|
// Provider: Anthropic API
|
|
38
38
|
// ---------------------------------------------------------------------------
|
|
39
39
|
function createAnthropicClient(modelOverride) {
|
|
40
|
-
const DEFAULT_MODEL = "claude-sonnet-4-
|
|
40
|
+
const DEFAULT_MODEL = "claude-sonnet-4-6";
|
|
41
41
|
const apiKey = process.env.ANTHROPIC_API_KEY;
|
|
42
42
|
if (!apiKey) {
|
|
43
43
|
throw new Error("ANTHROPIC_API_KEY is not set. Export it before running eval commands:\n export ANTHROPIC_API_KEY=sk-ant-...\n\nOr use a different provider:\n export VSKILL_EVAL_PROVIDER=claude-cli # uses your Claude Max plan\n export VSKILL_EVAL_PROVIDER=ollama # uses local Ollama");
|
package/dist/eval/llm.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"llm.js","sourceRoot":"","sources":["../../src/eval/llm.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,gFAAgF;AAChF,EAAE;AACF,uDAAuD;AACvD,wEAAwE;AACxE,8DAA8D;AAC9D,uEAAuE;AACvE,EAAE;AACF,uDAAuD;AACvD,iFAAiF;AACjF,4EAA4E;AAC5E,EAAE;AACF,iDAAiD;AACjD,+DAA+D;AAC/D,
|
|
1
|
+
{"version":3,"file":"llm.js","sourceRoot":"","sources":["../../src/eval/llm.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,gFAAgF;AAChF,EAAE;AACF,uDAAuD;AACvD,wEAAwE;AACxE,8DAA8D;AAC9D,uEAAuE;AACvE,EAAE;AACF,uDAAuD;AACvD,iFAAiF;AACjF,4EAA4E;AAC5E,EAAE;AACF,iDAAiD;AACjD,+DAA+D;AAC/D,4DAA4D;AAC5D,mDAAmD;AACnD,8EAA8E;AAE9E,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAgB3C,SAAS,cAAc;IACrB,OAAO,YAAY,CAAC;AACtB,CAAC;AAOD,MAAM,UAAU,eAAe,CAAC,SAAwB;IACtD,MAAM,QAAQ,GAAG,CAAC,SAAS,EAAE,QAAQ,IAAI,OAAO,CAAC,GAAG,CAAC,oBAAoB,IAAI,cAAc,EAAE,CAAiB,CAAC;IAC/G,MAAM,aAAa,GAAG,SAAS,EAAE,KAAK,CAAC;IACvC,QAAQ,QAAQ,EAAE,CAAC;QACjB,KAAK,WAAW;YACd,OAAO,qBAAqB,CAAC,aAAa,CAAC,CAAC;QAC9C,KAAK,YAAY;YACf,OAAO,qBAAqB,CAAC,aAAa,CAAC,CAAC;QAC9C,KAAK,QAAQ;YACX,OAAO,kBAAkB,CAAC,aAAa,CAAC,CAAC;QAC3C;YACE,MAAM,IAAI,KAAK,CACb,kCAAkC,QAAQ,gDAAgD,CAC3F,CAAC;IACN,CAAC;AACH,CAAC;AAED,8EAA8E;AAC9E,0BAA0B;AAC1B,8EAA8E;AAC9E,SAAS,qBAAqB,CAAC,aAAsB;IACnD,MAAM,aAAa,GAAG,mBAAmB,CAAC;IAE1C,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC;IAC7C,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,MAAM,IAAI,KAAK,CACb,sRAAsR,CACvR,CAAC;IACJ,CAAC;IAED,MAAM,KAAK,GAAG,aAAa,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,aAAa,CAAC;IAC9E,IAAI,cAAc,GAAQ,IAAI,CAAC;IAE/B,OAAO;QACL,KAAK;QACL,KAAK,CAAC,QAAQ,CAAC,YAAoB,EAAE,UAAkB;YACrD,IAAI,CAAC,cAAc,EAAE,CAAC;gBACpB,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,GAAG,MAAM,MAAM,CAAC,mBAAmB,CAAC,CAAC;gBACjE,cAAc,GAAG,IAAI,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;YAC7C,CAAC;YAED,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;YACzC,MAAM,OAAO,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,OAAO,CAAC,CAAC;YAC9D,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YACzB,IAAI,CAAC;gBACH,MAAM,QAAQ,GAAG,MAAM,cAAc,CAAC,QAAQ,CAAC,MAAM,CACnD;oBACE,KAAK;oBACL,MAAM,EAAE,YAAY;oBACpB,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,CAAC;oBACjD,UAAU,EAAE,IAAI;iBACjB,EACD,EAAE,MAAM,EAAE,UAAU,CAAC,MAAM,EAAE,CAC9B,CAAC;gBACF,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC;gBAEtC,MAAM,SAAS,GAAG,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC;gBACvE,MAAM,IAAI,GAAG,SAAS,IAAI,MAAM,IAAI,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;gBACpE,OAAO;oBACL,IAAI;oBACJ,UAAU;oBACV,WAAW,EAAE,QAAQ,CAAC,KAAK,EAAE,YAAY,IAAI,IAAI;oBACjD,YAAY,EAAE,QAAQ,CAAC,KAAK,EAAE,aAAa,IAAI,IAAI;iBACpD,CAAC;YACJ,CAAC;oBAAS,CAAC;gBACT,YAAY,CAAC,OAAO,CAAC,CAAC;YACxB,CAAC;QACH,CAAC;KACF,CAAC;AACJ,CAAC;AAED,8EAA8E;AAC9E,4EAA4E;AAC5E,EAAE;AACF,uEAAuE;AACvE,EAAE;AACF,6DAA6D;AAC7D,oFAAoF;AACpF,8EAA8E;AAC9E,SAAS,qBAAqB,CAAC,aAAsB;IACnD,MAAM,KAAK,GAAG,aAAa,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,QAAQ,CAAC;IAEzE,OAAO;QACL,KAAK,EAAE,UAAU,KAAK,EAAE;QACxB,KAAK,CAAC,QAAQ,CAAC,YAAoB,EAAE,UAAkB;YACrD,MAAM,cAAc,GAAG,GAAG,YAAY,OAAO,UAAU,EAAE,CAAC;YAC1D,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAEzB,MAAM,IAAI,GAAG,MAAM,IAAI,OAAO,CAAS,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;gBACzD,yEAAyE;gBACzE,MAAM,QAAQ,GAA2B,EAAE,CAAC;gBAC5C,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;oBACjD,IAAI,CAAC,KAAK,SAAS,IAAI,CAAC,CAAC,CAAC,UAAU,CAAC,QAAQ,CAAC;wBAAE,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;gBAClE,CAAC;gBACD,MAAM,IAAI,GAAG,KAAK,CAAC,QAAQ,EAAE,CAAC,IAAI,EAAE,SAAS,EAAE,KAAK,CAAC,EAAE;oBACrD,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC;oBAC/B,GAAG,EAAE,QAAQ;iBACd,CAAC,CAAC;gBAEH,IAAI,MAAM,GAAG,EAAE,CAAC;gBAChB,IAAI,MAAM,GAAG,EAAE,CAAC;gBAChB,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,CAAS,EAAE,EAAE,GAAG,MAAM,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;gBACnE,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,CAAS,EAAE,EAAE,GAAG,MAAM,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;gBAEnE,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE;oBAC5B,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;oBACrB,MAAM,CAAC,IAAI,KAAK,CAAC,iCAAiC,CAAC,CAAC,CAAC;gBACvD,CAAC,EAAE,OAAO,CAAC,CAAC;gBAEZ,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAA0B,EAAE,EAAE;oBAC9C,YAAY,CAAC,KAAK,CAAC,CAAC;oBACpB,IAAI,GAAG,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;wBAC1B,MAAM,CAAC,IAAI,KAAK,CACd,qJAAqJ,CACtJ,CAAC,CAAC;oBACL,CAAC;yBAAM,CAAC;wBACN,MAAM,CAAC,IAAI,KAAK,CAAC,sBAAsB,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;oBACzD,CAAC;gBACH,CAAC,CAAC,CAAC;gBAEH,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,EAAE;oBACxB,YAAY,CAAC,KAAK,CAAC,CAAC;oBACpB,IAAI,IAAI,KAAK,CAAC,EAAE,CAAC;wBACf,OAAO,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;oBACzB,CAAC;yBAAM,CAAC;wBACN,MAAM,MAAM,GAAG,CAAC,MAAM,IAAI,MAAM,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;wBAChD,MAAM,CAAC,IAAI,KAAK,CAAC,+BAA+B,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;oBACzF,CAAC;gBACH,CAAC,CAAC,CAAC;gBAEH,yEAAyE;gBACzE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;YACjC,CAAC,CAAC,CAAC;YAEH,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,EAAE,WAAW,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC;QACzF,CAAC;KACF,CAAC;AACJ,CAAC;AAED,8EAA8E;AAC9E,qDAAqD;AACrD,8EAA8E;AAC9E,SAAS,kBAAkB,CAAC,aAAsB;IAChD,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,eAAe,IAAI,wBAAwB,CAAC;IACxE,MAAM,KAAK,GAAG,aAAa,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,aAAa,CAAC;IAE9E,OAAO;QACL,KAAK;QACL,KAAK,CAAC,QAAQ,CAAC,YAAoB,EAAE,UAAkB;YACrD,MAAM,UAAU,GAAG,GAAG,YAAY,OAAO,UAAU,EAAE,CAAC;YACtD,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAEzB,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,OAAO,eAAe,EAAE;gBACtD,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE;gBAC/C,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;oBACnB,KAAK;oBACL,MAAM,EAAE,UAAU;oBAClB,MAAM,EAAE,KAAK;oBACb,OAAO,EAAE;wBACP,WAAW,EAAE,IAAI;wBACjB,WAAW,EAAE,GAAG;qBACjB;iBACF,CAAC;gBACF,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,OAAO,CAAC;aACrC,CAAC,CAAC;YAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;gBACjB,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;gBACpC,IAAI,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;oBAC3D,MAAM,IAAI,KAAK,CACb,iBAAiB,KAAK,8CAA8C,KAAK,EAAE,CAC5E,CAAC;gBACJ,CAAC;gBACD,MAAM,IAAI,KAAK,CAAC,0BAA0B,KAAK,EAAE,CAAC,CAAC;YACrD,CAAC;YAED,MAAM,IAAI,GAAG,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAIlC,CAAC;YACF,OAAO;gBACL,IAAI,EAAE,IAAI,CAAC,QAAQ,IAAI,EAAE;gBACzB,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;gBAC9B,WAAW,EAAE,IAAI,CAAC,iBAAiB,IAAI,IAAI;gBAC3C,YAAY,EAAE,IAAI,CAAC,UAAU,IAAI,IAAI;aACtC,CAAC;QACJ,CAAC;KACF,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
export interface McpDependency {
|
|
2
|
+
server: string;
|
|
3
|
+
url: string;
|
|
4
|
+
transport: "http" | "stdio";
|
|
5
|
+
matchedTools: string[];
|
|
6
|
+
configSnippet: string;
|
|
7
|
+
}
|
|
8
|
+
export interface SkillDependency {
|
|
9
|
+
name: string;
|
|
10
|
+
source: "body-reference" | "frontmatter";
|
|
11
|
+
}
|
|
12
|
+
export declare function detectMcpDependencies(content: string): McpDependency[];
|
|
13
|
+
export declare function detectSkillDependencies(content: string): SkillDependency[];
|
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
// ---------------------------------------------------------------------------
|
|
2
|
+
// mcp-detector.ts -- pure function to detect MCP + skill dependencies from SKILL.md
|
|
3
|
+
// ---------------------------------------------------------------------------
|
|
4
|
+
const MCP_REGISTRY = [
|
|
5
|
+
{
|
|
6
|
+
server: "Slack",
|
|
7
|
+
prefixes: ["slack_"],
|
|
8
|
+
url: "https://mcp.slack.com/mcp",
|
|
9
|
+
transport: "http",
|
|
10
|
+
},
|
|
11
|
+
{
|
|
12
|
+
server: "GitHub",
|
|
13
|
+
prefixes: ["github_"],
|
|
14
|
+
url: "https://api.githubcopilot.com/mcp/",
|
|
15
|
+
transport: "http",
|
|
16
|
+
},
|
|
17
|
+
{
|
|
18
|
+
server: "Linear",
|
|
19
|
+
prefixes: ["linear_"],
|
|
20
|
+
url: "https://mcp.linear.app/mcp",
|
|
21
|
+
transport: "http",
|
|
22
|
+
},
|
|
23
|
+
{
|
|
24
|
+
server: "Google Workspace",
|
|
25
|
+
prefixes: ["gws_", "drive_", "gmail_", "sheets_", "calendar_", "chat_"],
|
|
26
|
+
url: "https://mcp.google.com/mcp",
|
|
27
|
+
transport: "http",
|
|
28
|
+
},
|
|
29
|
+
];
|
|
30
|
+
function buildConfigSnippet(serverKey, url) {
|
|
31
|
+
return JSON.stringify({ mcpServers: { [serverKey.toLowerCase()]: { type: "http", url } } }, null, 2);
|
|
32
|
+
}
|
|
33
|
+
function parseFrontmatterField(content, field) {
|
|
34
|
+
const fmMatch = content.match(/^---\n([\s\S]*?)\n---/);
|
|
35
|
+
if (!fmMatch)
|
|
36
|
+
return [];
|
|
37
|
+
const fm = fmMatch[1];
|
|
38
|
+
// Try YAML list: field:\n - val1\n - val2
|
|
39
|
+
const listMatch = fm.match(new RegExp(`${field}:\\s*\\n((?:\\s+-\\s+.+\\n?)+)`));
|
|
40
|
+
if (listMatch) {
|
|
41
|
+
return listMatch[1]
|
|
42
|
+
.split("\n")
|
|
43
|
+
.map((l) => l.replace(/^\s*-\s*/, "").trim())
|
|
44
|
+
.filter(Boolean);
|
|
45
|
+
}
|
|
46
|
+
// Try inline: field: [val1, val2] or field: val1, val2
|
|
47
|
+
const inlineMatch = fm.match(new RegExp(`${field}:\\s*\\[?([^\\]\\n]+)\\]?`));
|
|
48
|
+
if (inlineMatch) {
|
|
49
|
+
return inlineMatch[1]
|
|
50
|
+
.split(",")
|
|
51
|
+
.map((s) => s.trim().replace(/^["']|["']$/g, ""))
|
|
52
|
+
.filter(Boolean);
|
|
53
|
+
}
|
|
54
|
+
return [];
|
|
55
|
+
}
|
|
56
|
+
function extractToolNames(content) {
|
|
57
|
+
const tools = new Set();
|
|
58
|
+
// From frontmatter allowed-tools
|
|
59
|
+
for (const tool of parseFrontmatterField(content, "allowed-tools")) {
|
|
60
|
+
tools.add(tool);
|
|
61
|
+
}
|
|
62
|
+
// Scan body for tool-like patterns (word_word format matching known prefixes)
|
|
63
|
+
const body = content.replace(/^---\n[\s\S]*?\n---/, "");
|
|
64
|
+
for (const entry of MCP_REGISTRY) {
|
|
65
|
+
for (const prefix of entry.prefixes) {
|
|
66
|
+
const regex = new RegExp(`\\b(${prefix.replace("_", "_")}\\w+)\\b`, "g");
|
|
67
|
+
let match;
|
|
68
|
+
while ((match = regex.exec(body)) !== null) {
|
|
69
|
+
tools.add(match[1]);
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
return Array.from(tools);
|
|
74
|
+
}
|
|
75
|
+
export function detectMcpDependencies(content) {
|
|
76
|
+
const toolNames = extractToolNames(content);
|
|
77
|
+
const deps = [];
|
|
78
|
+
for (const entry of MCP_REGISTRY) {
|
|
79
|
+
const matched = toolNames.filter((tool) => entry.prefixes.some((prefix) => tool.startsWith(prefix)));
|
|
80
|
+
if (matched.length === 0)
|
|
81
|
+
continue;
|
|
82
|
+
deps.push({
|
|
83
|
+
server: entry.server,
|
|
84
|
+
url: entry.url,
|
|
85
|
+
transport: entry.transport,
|
|
86
|
+
matchedTools: [...new Set(matched)],
|
|
87
|
+
configSnippet: buildConfigSnippet(entry.server, entry.url),
|
|
88
|
+
});
|
|
89
|
+
}
|
|
90
|
+
return deps;
|
|
91
|
+
}
|
|
92
|
+
export function detectSkillDependencies(content) {
|
|
93
|
+
const deps = [];
|
|
94
|
+
const seen = new Set();
|
|
95
|
+
// From frontmatter dependencies field
|
|
96
|
+
for (const dep of parseFrontmatterField(content, "dependencies")) {
|
|
97
|
+
if (!seen.has(dep)) {
|
|
98
|
+
seen.add(dep);
|
|
99
|
+
deps.push({ name: dep, source: "frontmatter" });
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
// Scan body for skill references
|
|
103
|
+
const body = content.replace(/^---\n[\s\S]*?\n---/, "");
|
|
104
|
+
// Pattern: "use the X skill", "chains with X", "requires the X skill"
|
|
105
|
+
const patterns = [
|
|
106
|
+
/\buses?\s+the\s+([\w-]+)\s+skill\b/gi,
|
|
107
|
+
/\bchains?\s+with\s+([\w-]+)\b/gi,
|
|
108
|
+
/\brequires?\s+the\s+([\w-]+)\s+skill\b/gi,
|
|
109
|
+
/\bdepends?\s+on\s+the\s+([\w-]+)\s+skill\b/gi,
|
|
110
|
+
];
|
|
111
|
+
for (const pattern of patterns) {
|
|
112
|
+
let match;
|
|
113
|
+
while ((match = pattern.exec(body)) !== null) {
|
|
114
|
+
const name = match[1].toLowerCase();
|
|
115
|
+
if (!seen.has(name)) {
|
|
116
|
+
seen.add(name);
|
|
117
|
+
deps.push({ name, source: "body-reference" });
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
return deps;
|
|
122
|
+
}
|
|
123
|
+
//# sourceMappingURL=mcp-detector.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"mcp-detector.js","sourceRoot":"","sources":["../../src/eval/mcp-detector.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,oFAAoF;AACpF,8EAA8E;AAsB9E,MAAM,YAAY,GAAuB;IACvC;QACE,MAAM,EAAE,OAAO;QACf,QAAQ,EAAE,CAAC,QAAQ,CAAC;QACpB,GAAG,EAAE,2BAA2B;QAChC,SAAS,EAAE,MAAM;KAClB;IACD;QACE,MAAM,EAAE,QAAQ;QAChB,QAAQ,EAAE,CAAC,SAAS,CAAC;QACrB,GAAG,EAAE,oCAAoC;QACzC,SAAS,EAAE,MAAM;KAClB;IACD;QACE,MAAM,EAAE,QAAQ;QAChB,QAAQ,EAAE,CAAC,SAAS,CAAC;QACrB,GAAG,EAAE,4BAA4B;QACjC,SAAS,EAAE,MAAM;KAClB;IACD;QACE,MAAM,EAAE,kBAAkB;QAC1B,QAAQ,EAAE,CAAC,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,SAAS,EAAE,WAAW,EAAE,OAAO,CAAC;QACvE,GAAG,EAAE,4BAA4B;QACjC,SAAS,EAAE,MAAM;KAClB;CACF,CAAC;AAEF,SAAS,kBAAkB,CAAC,SAAiB,EAAE,GAAW;IACxD,OAAO,IAAI,CAAC,SAAS,CACnB,EAAE,UAAU,EAAE,EAAE,CAAC,SAAS,CAAC,WAAW,EAAE,CAAC,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,EAAE,EAAE,EACpE,IAAI,EACJ,CAAC,CACF,CAAC;AACJ,CAAC;AAED,SAAS,qBAAqB,CAAC,OAAe,EAAE,KAAa;IAC3D,MAAM,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,uBAAuB,CAAC,CAAC;IACvD,IAAI,CAAC,OAAO;QAAE,OAAO,EAAE,CAAC;IACxB,MAAM,EAAE,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;IAEtB,4CAA4C;IAC5C,MAAM,SAAS,GAAG,EAAE,CAAC,KAAK,CAAC,IAAI,MAAM,CAAC,GAAG,KAAK,gCAAgC,CAAC,CAAC,CAAC;IACjF,IAAI,SAAS,EAAE,CAAC;QACd,OAAO,SAAS,CAAC,CAAC,CAAC;aAChB,KAAK,CAAC,IAAI,CAAC;aACX,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;aAC5C,MAAM,CAAC,OAAO,CAAC,CAAC;IACrB,CAAC;IAED,uDAAuD;IACvD,MAAM,WAAW,GAAG,EAAE,CAAC,KAAK,CAAC,IAAI,MAAM,CAAC,GAAG,KAAK,2BAA2B,CAAC,CAAC,CAAC;IAC9E,IAAI,WAAW,EAAE,CAAC;QAChB,OAAO,WAAW,CAAC,CAAC,CAAC;aAClB,KAAK,CAAC,GAAG,CAAC;aACV,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,cAAc,EAAE,EAAE,CAAC,CAAC;aAChD,MAAM,CAAC,OAAO,CAAC,CAAC;IACrB,CAAC;IAED,OAAO,EAAE,CAAC;AACZ,CAAC;AAED,SAAS,gBAAgB,CAAC,OAAe;IACvC,MAAM,KAAK,GAAgB,IAAI,GAAG,EAAE,CAAC;IAErC,iCAAiC;IACjC,KAAK,MAAM,IAAI,IAAI,qBAAqB,CAAC,OAAO,EAAE,eAAe,CAAC,EAAE,CAAC;QACnE,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAClB,CAAC;IAED,8EAA8E;IAC9E,MAAM,IAAI,GAAG,OAAO,CAAC,OAAO,CAAC,qBAAqB,EAAE,EAAE,CAAC,CAAC;IACxD,KAAK,MAAM,KAAK,IAAI,YAAY,EAAE,CAAC;QACjC,KAAK,MAAM,MAAM,IAAI,KAAK,CAAC,QAAQ,EAAE,CAAC;YACpC,MAAM,KAAK,GAAG,IAAI,MAAM,CAAC,OAAO,MAAM,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC;YACzE,IAAI,KAAK,CAAC;YACV,OAAO,CAAC,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;gBAC3C,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;YACtB,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;AAC3B,CAAC;AAED,MAAM,UAAU,qBAAqB,CAAC,OAAe;IACnD,MAAM,SAAS,GAAG,gBAAgB,CAAC,OAAO,CAAC,CAAC;IAC5C,MAAM,IAAI,GAAoB,EAAE,CAAC;IAEjC,KAAK,MAAM,KAAK,IAAI,YAAY,EAAE,CAAC;QACjC,MAAM,OAAO,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CACxC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CACzD,CAAC;QACF,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;YAAE,SAAS;QAEnC,IAAI,CAAC,IAAI,CAAC;YACR,MAAM,EAAE,KAAK,CAAC,MAAM;YACpB,GAAG,EAAE,KAAK,CAAC,GAAG;YACd,SAAS,EAAE,KAAK,CAAC,SAAS;YAC1B,YAAY,EAAE,CAAC,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC;YACnC,aAAa,EAAE,kBAAkB,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,GAAG,CAAC;SAC3D,CAAC,CAAC;IACL,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,UAAU,uBAAuB,CAAC,OAAe;IACrD,MAAM,IAAI,GAAsB,EAAE,CAAC;IACnC,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAE/B,sCAAsC;IACtC,KAAK,MAAM,GAAG,IAAI,qBAAqB,CAAC,OAAO,EAAE,cAAc,CAAC,EAAE,CAAC;QACjE,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YACnB,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACd,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,GAAG,EAAE,MAAM,EAAE,aAAa,EAAE,CAAC,CAAC;QAClD,CAAC;IACH,CAAC;IAED,iCAAiC;IACjC,MAAM,IAAI,GAAG,OAAO,CAAC,OAAO,CAAC,qBAAqB,EAAE,EAAE,CAAC,CAAC;IAExD,sEAAsE;IACtE,MAAM,QAAQ,GAAG;QACf,sCAAsC;QACtC,iCAAiC;QACjC,0CAA0C;QAC1C,8CAA8C;KAC/C,CAAC;IAEF,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,IAAI,KAAK,CAAC;QACV,OAAO,CAAC,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;YAC7C,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC;YACpC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;gBACpB,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;gBACf,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,gBAAgB,EAAE,CAAC,CAAC;YAChD,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC"}
|
|
@@ -14,13 +14,25 @@ export async function scanSkills(root) {
|
|
|
14
14
|
if (!existsSync(root))
|
|
15
15
|
return skills;
|
|
16
16
|
// Layout 4: root IS the skill directory itself → {root}/SKILL.md
|
|
17
|
+
// Walk up from the skill dir to find the plugin name:
|
|
18
|
+
// .../plugins/{plugin}/skills/{skill}/ → plugin = grandparent of skill
|
|
19
|
+
// .../skills/{skill}/ → plugin = parent above "skills"
|
|
20
|
+
// Fallback to immediate parent if structure is unknown.
|
|
17
21
|
if (existsSync(join(root, "SKILL.md"))) {
|
|
18
22
|
const skillName = basename(root);
|
|
19
|
-
const
|
|
23
|
+
const parent = basename(dirname(root)); // e.g. "skills"
|
|
24
|
+
let pluginName;
|
|
25
|
+
if (parent === "skills") {
|
|
26
|
+
// Standard layout: .../plugins/{plugin}/skills/{skill}/ or .../{plugin}/skills/{skill}/
|
|
27
|
+
pluginName = basename(dirname(dirname(root))) || "default";
|
|
28
|
+
}
|
|
29
|
+
else {
|
|
30
|
+
pluginName = parent || "default";
|
|
31
|
+
}
|
|
20
32
|
const hasEvals = existsSync(join(root, "evals", "evals.json"));
|
|
21
33
|
const hasBenchmark = existsSync(join(root, "evals", "benchmark.json"));
|
|
22
34
|
skills.push({
|
|
23
|
-
plugin:
|
|
35
|
+
plugin: pluginName,
|
|
24
36
|
skill: skillName,
|
|
25
37
|
dir: root,
|
|
26
38
|
hasEvals,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"skill-scanner.js","sourceRoot":"","sources":["../../src/eval/skill-scanner.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,uCAAuC;AACvC,EAAE;AACF,qDAAqD;AACrD,yDAAyD;AACzD,iEAAiE;AACjE,0EAA0E;AAC1E,uEAAuE;AACvE,8EAA8E;AAE9E,OAAO,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AAClD,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAUpD,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,IAAY;IAC3C,MAAM,MAAM,GAAgB,EAAE,CAAC;IAE/B,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;QAAE,OAAO,MAAM,CAAC;IAErC,iEAAiE;IACjE,IAAI,UAAU,CAAC,IAAI,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC,EAAE,CAAC;QACvC,MAAM,SAAS,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC;QACjC,MAAM,UAAU,GAAG,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC;
|
|
1
|
+
{"version":3,"file":"skill-scanner.js","sourceRoot":"","sources":["../../src/eval/skill-scanner.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,uCAAuC;AACvC,EAAE;AACF,qDAAqD;AACrD,yDAAyD;AACzD,iEAAiE;AACjE,0EAA0E;AAC1E,uEAAuE;AACvE,8EAA8E;AAE9E,OAAO,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AAClD,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAUpD,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,IAAY;IAC3C,MAAM,MAAM,GAAgB,EAAE,CAAC;IAE/B,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;QAAE,OAAO,MAAM,CAAC;IAErC,iEAAiE;IACjE,sDAAsD;IACtD,2EAA2E;IAC3E,4EAA4E;IAC5E,wDAAwD;IACxD,IAAI,UAAU,CAAC,IAAI,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC,EAAE,CAAC;QACvC,MAAM,SAAS,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC;QACjC,MAAM,MAAM,GAAG,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,gBAAgB;QACxD,IAAI,UAAkB,CAAC;QACvB,IAAI,MAAM,KAAK,QAAQ,EAAE,CAAC;YACxB,wFAAwF;YACxF,UAAU,GAAG,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,SAAS,CAAC;QAC7D,CAAC;aAAM,CAAC;YACN,UAAU,GAAG,MAAM,IAAI,SAAS,CAAC;QACnC,CAAC;QACD,MAAM,QAAQ,GAAG,UAAU,CAAC,IAAI,CAAC,IAAI,EAAE,OAAO,EAAE,YAAY,CAAC,CAAC,CAAC;QAC/D,MAAM,YAAY,GAAG,UAAU,CAAC,IAAI,CAAC,IAAI,EAAE,OAAO,EAAE,gBAAgB,CAAC,CAAC,CAAC;QACvE,MAAM,CAAC,IAAI,CAAC;YACV,MAAM,EAAE,UAAU;YAClB,KAAK,EAAE,SAAS;YAChB,GAAG,EAAE,IAAI;YACT,QAAQ;YACR,YAAY;SACb,CAAC,CAAC;QACH,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,0EAA0E;IAC1E,aAAa,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,SAAS,EAAE,IAAI,CAAC,IAAI,EAAE,QAAQ,CAAC,EAAE,MAAM,CAAC,CAAC;IAEzE,4EAA4E;IAC5E,qFAAqF;IACrF,cAAc,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC,CAAC;IAEpD,kFAAkF;IAClF,oFAAoF;IACpF,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;IACzC,IAAI,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;QAC3B,cAAc,CAAC,UAAU,EAAE,MAAM,EAAE,CAAC,SAAS,CAAC,CAAC,CAAC;IAClD,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,cAAc,CAAC,GAAW,EAAE,MAAmB,EAAE,OAAiB;IACzE,IAAI,OAAiB,CAAC;IACtB,IAAI,CAAC;QACH,OAAO,GAAG,WAAW,CAAC,GAAG,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC;aAChD,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;aAC3D,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IACxB,CAAC;IAAC,MAAM,CAAC;QACP,OAAO;IACT,CAAC;IAED,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,aAAa,CAAC,MAAM,EAAE,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,CAAC,CAAC;IAC7D,CAAC;AACH,CAAC;AAED,SAAS,aAAa,CACpB,MAAc,EACd,SAAiB,EACjB,MAAmB;IAEnB,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC;QAAE,OAAO;IAEnC,IAAI,SAAmB,CAAC;IACxB,IAAI,CAAC;QACH,SAAS,GAAG,WAAW,CAAC,SAAS,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC;aACxD,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC;aAC9B,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IACxB,CAAC;IAAC,MAAM,CAAC;QACP,OAAO;IACT,CAAC;IAED,KAAK,MAAM,KAAK,IAAI,SAAS,EAAE,CAAC;QAC9B,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;QACxC,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;QAE3C,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC;YAAE,SAAS;QAEnC,MAAM,QAAQ,GAAG,UAAU,CAAC,IAAI,CAAC,QAAQ,EAAE,OAAO,EAAE,YAAY,CAAC,CAAC,CAAC;QACnE,MAAM,YAAY,GAAG,UAAU,CAC7B,IAAI,CAAC,QAAQ,EAAE,OAAO,EAAE,gBAAgB,CAAC,CAC1C,CAAC;QAEF,MAAM,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,EAAE,QAAQ,EAAE,QAAQ,EAAE,YAAY,EAAE,CAAC,CAAC;IACxE,CAAC;AACH,CAAC"}
|