vskill 0.2.61 → 0.2.63
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/__tests__/eval-serve.test.d.ts +1 -0
- package/dist/commands/__tests__/eval-serve.test.js +23 -0
- package/dist/commands/__tests__/eval-serve.test.js.map +1 -0
- package/dist/commands/eval/serve.d.ts +7 -1
- package/dist/commands/eval/serve.js +138 -6
- package/dist/commands/eval/serve.js.map +1 -1
- package/dist/commands/eval.js +1 -1
- package/dist/eval/__tests__/llm.test.js +28 -0
- package/dist/eval/__tests__/llm.test.js.map +1 -1
- package/dist/eval/llm.d.ts +6 -1
- package/dist/eval/llm.js +14 -12
- package/dist/eval/llm.js.map +1 -1
- package/dist/eval-server/api-routes.d.ts +1 -1
- package/dist/eval-server/api-routes.js +106 -9
- package/dist/eval-server/api-routes.js.map +1 -1
- package/dist/eval-server/eval-server.d.ts +1 -0
- package/dist/eval-server/eval-server.js +1 -1
- package/dist/eval-server/eval-server.js.map +1 -1
- package/dist/eval-ui/assets/index-BZGGjVxB.css +1 -0
- package/dist/eval-ui/assets/index-BrmwG3KC.js +70 -0
- package/dist/eval-ui/index.html +2 -2
- package/package.json +1 -1
- package/dist/eval-ui/assets/index-BYpLv_X1.css +0 -1
- package/dist/eval-ui/assets/index-C69Rpa8n.js +0 -70
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import { describe, it, expect } from "vitest";
|
|
2
|
+
import { projectPort } from "../eval/serve.js";
|
|
3
|
+
describe("projectPort", () => {
|
|
4
|
+
it("returns a port in range 3077-3177", () => {
|
|
5
|
+
const port = projectPort("/Users/alice/projects/my-skills");
|
|
6
|
+
expect(port).toBeGreaterThanOrEqual(3077);
|
|
7
|
+
expect(port).toBeLessThanOrEqual(3177);
|
|
8
|
+
});
|
|
9
|
+
it("returns the same port for the same path (deterministic)", () => {
|
|
10
|
+
const a = projectPort("/Users/alice/projects/my-skills");
|
|
11
|
+
const b = projectPort("/Users/alice/projects/my-skills");
|
|
12
|
+
expect(a).toBe(b);
|
|
13
|
+
});
|
|
14
|
+
it("returns different ports for different paths", () => {
|
|
15
|
+
const a = projectPort("/Users/alice/projects/repo-a");
|
|
16
|
+
const b = projectPort("/Users/alice/projects/repo-b");
|
|
17
|
+
// Not guaranteed by hash, but overwhelmingly likely for different inputs
|
|
18
|
+
// We just check both are valid
|
|
19
|
+
expect(a).toBeGreaterThanOrEqual(3077);
|
|
20
|
+
expect(b).toBeGreaterThanOrEqual(3077);
|
|
21
|
+
});
|
|
22
|
+
});
|
|
23
|
+
//# sourceMappingURL=eval-serve.test.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"eval-serve.test.js","sourceRoot":"","sources":["../../../src/commands/__tests__/eval-serve.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAC9C,OAAO,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAE/C,QAAQ,CAAC,aAAa,EAAE,GAAG,EAAE;IAC3B,EAAE,CAAC,mCAAmC,EAAE,GAAG,EAAE;QAC3C,MAAM,IAAI,GAAG,WAAW,CAAC,iCAAiC,CAAC,CAAC;QAC5D,MAAM,CAAC,IAAI,CAAC,CAAC,sBAAsB,CAAC,IAAI,CAAC,CAAC;QAC1C,MAAM,CAAC,IAAI,CAAC,CAAC,mBAAmB,CAAC,IAAI,CAAC,CAAC;IACzC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,yDAAyD,EAAE,GAAG,EAAE;QACjE,MAAM,CAAC,GAAG,WAAW,CAAC,iCAAiC,CAAC,CAAC;QACzD,MAAM,CAAC,GAAG,WAAW,CAAC,iCAAiC,CAAC,CAAC;QACzD,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACpB,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,6CAA6C,EAAE,GAAG,EAAE;QACrD,MAAM,CAAC,GAAG,WAAW,CAAC,8BAA8B,CAAC,CAAC;QACtD,MAAM,CAAC,GAAG,WAAW,CAAC,8BAA8B,CAAC,CAAC;QACtD,yEAAyE;QACzE,+BAA+B;QAC/B,MAAM,CAAC,CAAC,CAAC,CAAC,sBAAsB,CAAC,IAAI,CAAC,CAAC;QACvC,MAAM,CAAC,CAAC,CAAC,CAAC,sBAAsB,CAAC,IAAI,CAAC,CAAC;IACzC,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
|
@@ -1 +1,7 @@
|
|
|
1
|
-
|
|
1
|
+
/**
|
|
2
|
+
* Deterministic port for a project path.
|
|
3
|
+
* Maps the absolute path to a port in range 3077-3177 using a hash.
|
|
4
|
+
* Same project always gets the same port — bookmarkable, no collisions.
|
|
5
|
+
*/
|
|
6
|
+
export declare function projectPort(rootPath: string): number;
|
|
7
|
+
export declare function runEvalServe(root: string, port: number | null): Promise<void>;
|
|
@@ -1,14 +1,26 @@
|
|
|
1
1
|
// ---------------------------------------------------------------------------
|
|
2
2
|
// vskill eval serve -- start the eval UI web server
|
|
3
3
|
// ---------------------------------------------------------------------------
|
|
4
|
-
import { resolve } from "node:path";
|
|
4
|
+
import { resolve, basename } from "node:path";
|
|
5
5
|
import { existsSync } from "node:fs";
|
|
6
|
+
import { createHash } from "node:crypto";
|
|
7
|
+
import { execSync } from "node:child_process";
|
|
8
|
+
import * as net from "node:net";
|
|
6
9
|
import { homedir } from "node:os";
|
|
7
10
|
import { join } from "node:path";
|
|
8
11
|
import { startEvalServer } from "../../eval-server/eval-server.js";
|
|
9
|
-
import { yellow, dim } from "../../utils/output.js";
|
|
12
|
+
import { yellow, dim, red, cyan, bold } from "../../utils/output.js";
|
|
13
|
+
/**
|
|
14
|
+
* Deterministic port for a project path.
|
|
15
|
+
* Maps the absolute path to a port in range 3077-3177 using a hash.
|
|
16
|
+
* Same project always gets the same port — bookmarkable, no collisions.
|
|
17
|
+
*/
|
|
18
|
+
export function projectPort(rootPath) {
|
|
19
|
+
const hash = createHash("md5").update(rootPath).digest();
|
|
20
|
+
const offset = hash.readUInt16BE(0) % 101; // 0-100
|
|
21
|
+
return 3077 + offset;
|
|
22
|
+
}
|
|
10
23
|
function checkSkillCreator() {
|
|
11
|
-
// Check common skill-creator installation locations
|
|
12
24
|
const home = homedir();
|
|
13
25
|
const locations = [
|
|
14
26
|
join(home, ".claude", "plugins", "cache", "claude-plugins-official", "skill-creator"),
|
|
@@ -17,7 +29,7 @@ function checkSkillCreator() {
|
|
|
17
29
|
];
|
|
18
30
|
const found = locations.some((loc) => existsSync(loc));
|
|
19
31
|
if (!found) {
|
|
20
|
-
console.log(yellow("\n
|
|
32
|
+
console.log(yellow("\n Skill-Creator not detected.") +
|
|
21
33
|
"\n\n" +
|
|
22
34
|
dim(" The Skill-Creator skill provides the gold-standard evaluation\n") +
|
|
23
35
|
dim(" methodology (grading, blind A/B comparison, analysis).\n") +
|
|
@@ -34,15 +46,135 @@ function checkSkillCreator() {
|
|
|
34
46
|
"\n");
|
|
35
47
|
}
|
|
36
48
|
}
|
|
49
|
+
// ---------------------------------------------------------------------------
|
|
50
|
+
// Port conflict resolution
|
|
51
|
+
// ---------------------------------------------------------------------------
|
|
52
|
+
/** Quick TCP probe to check if a port is in use. */
|
|
53
|
+
function isPortInUse(port) {
|
|
54
|
+
return new Promise((resolve) => {
|
|
55
|
+
const server = net.createServer();
|
|
56
|
+
server.once("error", () => resolve(true));
|
|
57
|
+
server.once("listening", () => { server.close(() => resolve(false)); });
|
|
58
|
+
server.listen(port);
|
|
59
|
+
});
|
|
60
|
+
}
|
|
61
|
+
/** Probe a port to see if it's a vskill eval server. */
|
|
62
|
+
async function probeVskillServer(port) {
|
|
63
|
+
try {
|
|
64
|
+
const resp = await fetch(`http://localhost:${port}/api/config`, {
|
|
65
|
+
signal: AbortSignal.timeout(1500),
|
|
66
|
+
});
|
|
67
|
+
if (!resp.ok)
|
|
68
|
+
return null;
|
|
69
|
+
const data = await resp.json();
|
|
70
|
+
if (data.root) {
|
|
71
|
+
return { projectName: data.projectName || null, root: data.root, model: data.model || "unknown" };
|
|
72
|
+
}
|
|
73
|
+
return null;
|
|
74
|
+
}
|
|
75
|
+
catch {
|
|
76
|
+
return null;
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
/** Find the PID using a port. */
|
|
80
|
+
function findProcessOnPort(port) {
|
|
81
|
+
try {
|
|
82
|
+
const output = execSync(`lsof -ti:${port}`, { encoding: "utf-8", timeout: 3000 }).trim();
|
|
83
|
+
const pid = parseInt(output.split("\n")[0], 10);
|
|
84
|
+
if (!pid)
|
|
85
|
+
return null;
|
|
86
|
+
const cmd = execSync(`ps -p ${pid} -o command=`, { encoding: "utf-8", timeout: 3000 }).trim();
|
|
87
|
+
return { pid, command: cmd.slice(0, 120) };
|
|
88
|
+
}
|
|
89
|
+
catch {
|
|
90
|
+
return null;
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
/** Kill a process and wait for port release. */
|
|
94
|
+
async function killAndWait(pid) {
|
|
95
|
+
try {
|
|
96
|
+
process.kill(pid, "SIGTERM");
|
|
97
|
+
}
|
|
98
|
+
catch {
|
|
99
|
+
return;
|
|
100
|
+
}
|
|
101
|
+
const deadline = Date.now() + 3000;
|
|
102
|
+
while (Date.now() < deadline) {
|
|
103
|
+
try {
|
|
104
|
+
process.kill(pid, 0);
|
|
105
|
+
}
|
|
106
|
+
catch {
|
|
107
|
+
return;
|
|
108
|
+
} // gone
|
|
109
|
+
await new Promise((r) => setTimeout(r, 100));
|
|
110
|
+
}
|
|
111
|
+
try {
|
|
112
|
+
process.kill(pid, "SIGKILL");
|
|
113
|
+
}
|
|
114
|
+
catch { /* already dead */ }
|
|
115
|
+
await new Promise((r) => setTimeout(r, 200));
|
|
116
|
+
}
|
|
117
|
+
/**
|
|
118
|
+
* Handle port conflict:
|
|
119
|
+
* - Same project → auto-kill and restart
|
|
120
|
+
* - Different project or unknown → show diagnostics and exit
|
|
121
|
+
*/
|
|
122
|
+
async function handlePortConflict(port, resolvedRoot) {
|
|
123
|
+
const existing = await probeVskillServer(port);
|
|
124
|
+
if (existing) {
|
|
125
|
+
if (existing.root === resolvedRoot) {
|
|
126
|
+
// Same project — auto-restart
|
|
127
|
+
console.log(dim(`\n Port ${port} already running eval for this project. Restarting...\n`));
|
|
128
|
+
const proc = findProcessOnPort(port);
|
|
129
|
+
if (proc)
|
|
130
|
+
await killAndWait(proc.pid);
|
|
131
|
+
return;
|
|
132
|
+
}
|
|
133
|
+
// Different project
|
|
134
|
+
console.error(red(`\n Port ${port} is in use by another eval server:\n`) +
|
|
135
|
+
`\n ${bold("Project:")} ${cyan(existing.projectName || "unknown")}` +
|
|
136
|
+
`\n ${bold("Root:")} ${dim(existing.root)}` +
|
|
137
|
+
`\n ${bold("Model:")} ${dim(existing.model)}` +
|
|
138
|
+
`\n\n ${dim("Either stop it first, or use a different port:")}` +
|
|
139
|
+
`\n ${cyan(`vskill eval serve --port ${port + 1}`)}\n`);
|
|
140
|
+
process.exit(1);
|
|
141
|
+
}
|
|
142
|
+
// Not a vskill server
|
|
143
|
+
const proc = findProcessOnPort(port);
|
|
144
|
+
if (proc) {
|
|
145
|
+
console.error(red(`\n Port ${port} is in use by another process:\n`) +
|
|
146
|
+
`\n ${bold("PID:")} ${proc.pid}` +
|
|
147
|
+
`\n ${bold("Command:")} ${dim(proc.command)}` +
|
|
148
|
+
`\n\n ${dim("Either kill it, or use a different port:")}` +
|
|
149
|
+
`\n ${cyan(`kill ${proc.pid}`)} ${dim("or")} ${cyan(`vskill eval serve --port ${port + 1}`)}\n`);
|
|
150
|
+
}
|
|
151
|
+
else {
|
|
152
|
+
console.error(red(`\n Port ${port} is in use.\n`) +
|
|
153
|
+
` ${dim("Try:")} ${cyan(`vskill eval serve --port ${port + 1}`)}\n`);
|
|
154
|
+
}
|
|
155
|
+
process.exit(1);
|
|
156
|
+
}
|
|
157
|
+
// ---------------------------------------------------------------------------
|
|
158
|
+
// Main entry point
|
|
159
|
+
// ---------------------------------------------------------------------------
|
|
37
160
|
export async function runEvalServe(root, port) {
|
|
38
161
|
checkSkillCreator();
|
|
39
162
|
const resolvedRoot = resolve(root);
|
|
40
|
-
const
|
|
163
|
+
const effectivePort = port ?? projectPort(resolvedRoot);
|
|
164
|
+
const name = basename(resolvedRoot);
|
|
165
|
+
// Handle port conflicts gracefully
|
|
166
|
+
if (await isPortInUse(effectivePort)) {
|
|
167
|
+
await handlePortConflict(effectivePort, resolvedRoot);
|
|
168
|
+
}
|
|
169
|
+
const server = await startEvalServer({
|
|
170
|
+
port: effectivePort,
|
|
171
|
+
root: resolvedRoot,
|
|
172
|
+
projectName: name,
|
|
173
|
+
});
|
|
41
174
|
// Graceful shutdown
|
|
42
175
|
const shutdown = () => {
|
|
43
176
|
console.log("\nShutting down eval server...");
|
|
44
177
|
server.close(() => process.exit(0));
|
|
45
|
-
// Force exit after 5s
|
|
46
178
|
setTimeout(() => process.exit(0), 5000);
|
|
47
179
|
};
|
|
48
180
|
process.on("SIGINT", shutdown);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"serve.js","sourceRoot":"","sources":["../../../src/commands/eval/serve.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,oDAAoD;AACpD,8EAA8E;AAE9E,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;
|
|
1
|
+
{"version":3,"file":"serve.js","sourceRoot":"","sources":["../../../src/commands/eval/serve.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,oDAAoD;AACpD,8EAA8E;AAE9E,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAC9C,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AACrC,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAC9C,OAAO,KAAK,GAAG,MAAM,UAAU,CAAC;AAChC,OAAO,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AAClC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,eAAe,EAAE,MAAM,kCAAkC,CAAC;AACnE,OAAO,EAAE,MAAM,EAAE,GAAG,EAAE,GAAG,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,uBAAuB,CAAC;AAErE;;;;GAIG;AACH,MAAM,UAAU,WAAW,CAAC,QAAgB;IAC1C,MAAM,IAAI,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,MAAM,EAAE,CAAC;IACzD,MAAM,MAAM,GAAG,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,QAAQ;IACnD,OAAO,IAAI,GAAG,MAAM,CAAC;AACvB,CAAC;AAED,SAAS,iBAAiB;IACxB,MAAM,IAAI,GAAG,OAAO,EAAE,CAAC;IACvB,MAAM,SAAS,GAAG;QAChB,IAAI,CAAC,IAAI,EAAE,SAAS,EAAE,SAAS,EAAE,OAAO,EAAE,yBAAyB,EAAE,eAAe,CAAC;QACrF,IAAI,CAAC,IAAI,EAAE,SAAS,EAAE,QAAQ,EAAE,kBAAkB,CAAC;QACnD,IAAI,CAAC,IAAI,EAAE,SAAS,EAAE,SAAS,EAAE,OAAO,EAAE,WAAW,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,eAAe,CAAC;KACjG,CAAC;IAEF,MAAM,KAAK,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC;IAEvD,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,OAAO,CAAC,GAAG,CACT,MAAM,CAAC,iCAAiC,CAAC;YACvC,MAAM;YACN,GAAG,CAAC,mEAAmE,CAAC;YACxE,GAAG,CAAC,4DAA4D,CAAC;YACjE,GAAG,CAAC,kEAAkE,CAAC;YACvE,GAAG,CAAC,iDAAiD,CAAC;YACtD,6BAA6B;YAC7B,8BAA8B;YAC9B,IAAI;YACJ,8BAA8B;YAC9B,6DAA6D;YAC7D,IAAI;YACJ,8BAA8B;YAC9B,0CAA0C;YAC1C,IAAI,CACP,CAAC;IACJ,CAAC;AACH,CAAC;AAED,8EAA8E;AAC9E,2BAA2B;AAC3B,8EAA8E;AAE9E,oDAAoD;AACpD,SAAS,WAAW,CAAC,IAAY;IAC/B,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;QAC7B,MAAM,MAAM,GAAG,GAAG,CAAC,YAAY,EAAE,CAAC;QAClC,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC;QAC1C,MAAM,CAAC,IAAI,CAAC,WAAW,EAAE,GAAG,EAAE,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACxE,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACtB,CAAC,CAAC,CAAC;AACL,CAAC;AAED,wDAAwD;AACxD,KAAK,UAAU,iBAAiB,CAAC,IAAY;IAC3C,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,oBAAoB,IAAI,aAAa,EAAE;YAC9D,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,IAAI,CAAC;SAClC,CAAC,CAAC;QACH,IAAI,CAAC,IAAI,CAAC,EAAE;YAAE,OAAO,IAAI,CAAC;QAC1B,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,IAAI,EAA6D,CAAC;QAC1F,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;YACd,OAAO,EAAE,WAAW,EAAE,IAAI,CAAC,WAAW,IAAI,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,IAAI,SAAS,EAAE,CAAC;QACpG,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,iCAAiC;AACjC,SAAS,iBAAiB,CAAC,IAAY;IACrC,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,QAAQ,CAAC,YAAY,IAAI,EAAE,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QACzF,MAAM,GAAG,GAAG,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QAChD,IAAI,CAAC,GAAG;YAAE,OAAO,IAAI,CAAC;QACtB,MAAM,GAAG,GAAG,QAAQ,CAAC,SAAS,GAAG,cAAc,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QAC9F,OAAO,EAAE,GAAG,EAAE,OAAO,EAAE,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC;IAC7C,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,gDAAgD;AAChD,KAAK,UAAU,WAAW,CAAC,GAAW;IACpC,IAAI,CAAC;QAAC,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC;IAAC,CAAC;IAAC,MAAM,CAAC;QAAC,OAAO;IAAC,CAAC;IACvD,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC;IACnC,OAAO,IAAI,CAAC,GAAG,EAAE,GAAG,QAAQ,EAAE,CAAC;QAC7B,IAAI,CAAC;YAAC,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;QAAC,CAAC;QAAC,MAAM,CAAC;YAAC,OAAO;QAAC,CAAC,CAAC,OAAO;QACvD,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;IAC/C,CAAC;IACD,IAAI,CAAC;QAAC,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC;IAAC,CAAC;IAAC,MAAM,CAAC,CAAC,kBAAkB,CAAC,CAAC;IAClE,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;AAC/C,CAAC;AAED;;;;GAIG;AACH,KAAK,UAAU,kBAAkB,CAAC,IAAY,EAAE,YAAoB;IAClE,MAAM,QAAQ,GAAG,MAAM,iBAAiB,CAAC,IAAI,CAAC,CAAC;IAE/C,IAAI,QAAQ,EAAE,CAAC;QACb,IAAI,QAAQ,CAAC,IAAI,KAAK,YAAY,EAAE,CAAC;YACnC,8BAA8B;YAC9B,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,YAAY,IAAI,yDAAyD,CAAC,CAAC,CAAC;YAC5F,MAAM,IAAI,GAAG,iBAAiB,CAAC,IAAI,CAAC,CAAC;YACrC,IAAI,IAAI;gBAAE,MAAM,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACtC,OAAO;QACT,CAAC;QAED,oBAAoB;QACpB,OAAO,CAAC,KAAK,CACX,GAAG,CAAC,YAAY,IAAI,sCAAsC,CAAC;YAC3D,OAAO,IAAI,CAAC,UAAU,CAAC,KAAK,IAAI,CAAC,QAAQ,CAAC,WAAW,IAAI,SAAS,CAAC,EAAE;YACrE,OAAO,IAAI,CAAC,OAAO,CAAC,QAAQ,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE;YAChD,OAAO,IAAI,CAAC,QAAQ,CAAC,OAAO,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE;YACjD,SAAS,GAAG,CAAC,gDAAgD,CAAC,EAAE;YAChE,OAAO,IAAI,CAAC,4BAA4B,IAAI,GAAG,CAAC,EAAE,CAAC,IAAI,CACxD,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,sBAAsB;IACtB,MAAM,IAAI,GAAG,iBAAiB,CAAC,IAAI,CAAC,CAAC;IACrC,IAAI,IAAI,EAAE,CAAC;QACT,OAAO,CAAC,KAAK,CACX,GAAG,CAAC,YAAY,IAAI,kCAAkC,CAAC;YACvD,OAAO,IAAI,CAAC,MAAM,CAAC,SAAS,IAAI,CAAC,GAAG,EAAE;YACtC,OAAO,IAAI,CAAC,UAAU,CAAC,KAAK,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE;YAC/C,SAAS,GAAG,CAAC,0CAA0C,CAAC,EAAE;YAC1D,OAAO,IAAI,CAAC,QAAQ,IAAI,CAAC,GAAG,EAAE,CAAC,KAAK,GAAG,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,4BAA4B,IAAI,GAAG,CAAC,EAAE,CAAC,IAAI,CACnG,CAAC;IACJ,CAAC;SAAM,CAAC;QACN,OAAO,CAAC,KAAK,CACX,GAAG,CAAC,YAAY,IAAI,eAAe,CAAC;YACpC,KAAK,GAAG,CAAC,MAAM,CAAC,IAAI,IAAI,CAAC,4BAA4B,IAAI,GAAG,CAAC,EAAE,CAAC,IAAI,CACrE,CAAC;IACJ,CAAC;IACD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC;AAED,8EAA8E;AAC9E,mBAAmB;AACnB,8EAA8E;AAE9E,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,IAAY,EACZ,IAAmB;IAEnB,iBAAiB,EAAE,CAAC;IAEpB,MAAM,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACnC,MAAM,aAAa,GAAG,IAAI,IAAI,WAAW,CAAC,YAAY,CAAC,CAAC;IACxD,MAAM,IAAI,GAAG,QAAQ,CAAC,YAAY,CAAC,CAAC;IAEpC,mCAAmC;IACnC,IAAI,MAAM,WAAW,CAAC,aAAa,CAAC,EAAE,CAAC;QACrC,MAAM,kBAAkB,CAAC,aAAa,EAAE,YAAY,CAAC,CAAC;IACxD,CAAC;IAED,MAAM,MAAM,GAAG,MAAM,eAAe,CAAC;QACnC,IAAI,EAAE,aAAa;QACnB,IAAI,EAAE,YAAY;QAClB,WAAW,EAAE,IAAI;KAClB,CAAC,CAAC;IAEH,oBAAoB;IACpB,MAAM,QAAQ,GAAG,GAAG,EAAE;QACpB,OAAO,CAAC,GAAG,CAAC,gCAAgC,CAAC,CAAC;QAC9C,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;QACpC,UAAU,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;IAC1C,CAAC,CAAC;IAEF,OAAO,CAAC,EAAE,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;IAC/B,OAAO,CAAC,EAAE,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;AAClC,CAAC"}
|
package/dist/commands/eval.js
CHANGED
|
@@ -8,7 +8,7 @@ export async function evalCommand(subcommand, target, opts = {}) {
|
|
|
8
8
|
const root = opts.root ? resolve(opts.root) : resolve(".");
|
|
9
9
|
switch (subcommand) {
|
|
10
10
|
case "serve": {
|
|
11
|
-
const port = opts.port ? parseInt(opts.port, 10) :
|
|
11
|
+
const port = opts.port ? parseInt(opts.port, 10) : null;
|
|
12
12
|
const { runEvalServe } = await import("./eval/serve.js");
|
|
13
13
|
return runEvalServe(root, port);
|
|
14
14
|
}
|
|
@@ -86,6 +86,34 @@ describe("createLlmClient", () => {
|
|
|
86
86
|
expect(() => createLlmClient()).toThrow('Unknown VSKILL_EVAL_PROVIDER: "gpt-magic"');
|
|
87
87
|
});
|
|
88
88
|
// -------------------------------------------------------------------------
|
|
89
|
+
// Override params
|
|
90
|
+
// -------------------------------------------------------------------------
|
|
91
|
+
it("override provider takes precedence over env var", () => {
|
|
92
|
+
process.env.VSKILL_EVAL_PROVIDER = "ollama";
|
|
93
|
+
process.env.ANTHROPIC_API_KEY = "test-key";
|
|
94
|
+
const client = createLlmClient({ provider: "anthropic" });
|
|
95
|
+
expect(client.model).toBe("claude-sonnet-4-20250514");
|
|
96
|
+
});
|
|
97
|
+
it("override model takes precedence over env var", () => {
|
|
98
|
+
process.env.ANTHROPIC_API_KEY = "test-key";
|
|
99
|
+
const client = createLlmClient({ provider: "anthropic", model: "claude-opus-4-20250514" });
|
|
100
|
+
expect(client.model).toBe("claude-opus-4-20250514");
|
|
101
|
+
});
|
|
102
|
+
it("override provider=claude-cli with custom model", () => {
|
|
103
|
+
const client = createLlmClient({ provider: "claude-cli", model: "opus" });
|
|
104
|
+
expect(client.model).toBe("claude-opus");
|
|
105
|
+
});
|
|
106
|
+
it("override provider=ollama with custom model", () => {
|
|
107
|
+
const client = createLlmClient({ provider: "ollama", model: "qwen2.5:32b" });
|
|
108
|
+
expect(client.model).toBe("qwen2.5:32b");
|
|
109
|
+
});
|
|
110
|
+
it("override allows claude-cli even inside Claude Code when forced", () => {
|
|
111
|
+
process.env.CLAUDECODE = "1";
|
|
112
|
+
// override should bypass the CLAUDECODE guard
|
|
113
|
+
const client = createLlmClient({ provider: "claude-cli" });
|
|
114
|
+
expect(client.model).toBe("claude-sonnet");
|
|
115
|
+
});
|
|
116
|
+
// -------------------------------------------------------------------------
|
|
89
117
|
// Anthropic provider
|
|
90
118
|
// -------------------------------------------------------------------------
|
|
91
119
|
describe("anthropic provider", () => {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"llm.test.js","sourceRoot":"","sources":["../../../src/eval/__tests__/llm.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AAEzE,8EAA8E;AAC9E,QAAQ;AACR,8EAA8E;AAE9E,MAAM,UAAU,GAAG,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;AAC7C,MAAM,SAAS,GAAG,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;AAE5C,EAAE,CAAC,IAAI,CAAC,mBAAmB,EAAE,GAAG,EAAE,CAAC,CAAC;IAClC,OAAO,EAAE,MAAM,aAAa;QAC1B,QAAQ,GAAG,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC;KACnC;CACF,CAAC,CAAC,CAAC;AAEJ,EAAE,CAAC,IAAI,CAAC,oBAAoB,EAAE,GAAG,EAAE,CAAC,CAAC;IACnC,KAAK,EAAE,SAAS;CACjB,CAAC,CAAC,CAAC;AAEJ,8EAA8E;AAC9E,yDAAyD;AACzD,8EAA8E;AAC9E,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAE3C,SAAS,cAAc,CAAC,UAAkB,EAAE,QAAQ,GAAG,CAAC,EAAE,UAAU,GAAG,EAAE;IACvE,MAAM,IAAI,GAAG,IAAI,YAAY,EAAS,CAAC;IACvC,IAAI,CAAC,MAAM,GAAG,IAAI,YAAY,EAAE,CAAC;IACjC,IAAI,CAAC,MAAM,GAAG,IAAI,YAAY,EAAE,CAAC;IACjC,IAAI,CAAC,KAAK,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC;IAC9C,IAAI,CAAC,IAAI,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;IAEpB,4CAA4C;IAC5C,UAAU,CAAC,GAAG,EAAE;QACd,IAAI,UAAU;YAAE,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC;QAClE,IAAI,UAAU;YAAE,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC;QAClE,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;IAC/B,CAAC,EAAE,CAAC,CAAC,CAAC;IAEN,OAAO,IAAI,CAAC;AACd,CAAC;AAED,8EAA8E;AAC9E,uCAAuC;AACvC,8EAA8E;AAE9E,MAAM,EAAE,eAAe,EAAE,GAAG,MAAM,MAAM,CAAC,WAAW,CAAC,CAAC;AAEtD,8EAA8E;AAC9E,QAAQ;AACR,8EAA8E;AAE9E,QAAQ,CAAC,iBAAiB,EAAE,GAAG,EAAE;IAC/B,MAAM,OAAO,GAAG,EAAE,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC;IAEnC,UAAU,CAAC,GAAG,EAAE;QACd,EAAE,CAAC,aAAa,EAAE,CAAC;QACnB,OAAO,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC;QACxC,OAAO,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC;QACrC,OAAO,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC;QACrC,OAAO,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC;QACnC,OAAO,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC;IAChC,CAAC,CAAC,CAAC;IAEH,SAAS,CAAC,GAAG,EAAE;QACb,OAAO,CAAC,GAAG,GAAG,EAAE,GAAG,OAAO,EAAE,CAAC;IAC/B,CAAC,CAAC,CAAC;IAEH,4EAA4E;IAC5E,iBAAiB;IACjB,4EAA4E;IAE5E,EAAE,CAAC,8CAA8C,EAAE,GAAG,EAAE;QACtD,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;QACjC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;IAC7C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,gDAAgD,EAAE,GAAG,EAAE;QACxD,OAAO,CAAC,GAAG,CAAC,UAAU,GAAG,GAAG,CAAC;QAC7B,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;QACjC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;IAC3C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,sDAAsD,EAAE,GAAG,EAAE;QAC9D,OAAO,CAAC,GAAG,CAAC,iBAAiB,GAAG,UAAU,CAAC;QAC3C,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;QACjC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;IACxD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,qEAAqE,EAAE,GAAG,EAAE;QAC7E,OAAO,CAAC,GAAG,CAAC,UAAU,GAAG,GAAG,CAAC;QAC7B,OAAO,CAAC,GAAG,CAAC,iBAAiB,GAAG,UAAU,CAAC;QAC3C,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;QACjC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;IAC3C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,wDAAwD,EAAE,GAAG,EAAE;QAChE,OAAO,CAAC,GAAG,CAAC,oBAAoB,GAAG,QAAQ,CAAC;QAC5C,OAAO,CAAC,GAAG,CAAC,iBAAiB,GAAG,UAAU,CAAC;QAC3C,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;QACjC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;IAC3C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,4BAA4B,EAAE,GAAG,EAAE;QACpC,OAAO,CAAC,GAAG,CAAC,oBAAoB,GAAG,WAAW,CAAC;QAC/C,MAAM,CAAC,GAAG,EAAE,CAAC,eAAe,EAAE,CAAC,CAAC,OAAO,CAAC,2CAA2C,CAAC,CAAC;IACvF,CAAC,CAAC,CAAC;IAEH,4EAA4E;IAC5E,qBAAqB;IACrB,4EAA4E;IAE5E,QAAQ,CAAC,oBAAoB,EAAE,GAAG,EAAE;QAClC,UAAU,CAAC,GAAG,EAAE;YACd,OAAO,CAAC,GAAG,CAAC,oBAAoB,GAAG,WAAW,CAAC;YAC/C,OAAO,CAAC,GAAG,CAAC,iBAAiB,GAAG,UAAU,CAAC;QAC7C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,kDAAkD,EAAE,KAAK,IAAI,EAAE;YAChE,UAAU,CAAC,iBAAiB,CAAC;gBAC3B,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,oBAAoB,EAAE,CAAC;aACxD,CAAC,CAAC;YAEH,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;YACjC,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC,eAAe,EAAE,aAAa,CAAC,CAAC;YAErE,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;YAC1C,MAAM,CAAC,UAAU,CAAC,CAAC,oBAAoB,EAAE,CAAC;QAC5C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,6CAA6C,EAAE,KAAK,IAAI,EAAE;YAC3D,UAAU,CAAC,iBAAiB,CAAC;gBAC3B,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;aACxC,CAAC,CAAC;YAEH,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;YACjC,MAAM,MAAM,CAAC,QAAQ,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;YAEpC,MAAM,CAAC,UAAU,CAAC,CAAC,oBAAoB,CACrC,MAAM,CAAC,gBAAgB,CAAC,EAAE,KAAK,EAAE,0BAA0B,EAAE,CAAC,EAC9D,MAAM,CAAC,QAAQ,EAAE,CAClB,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,0CAA0C,EAAE,KAAK,IAAI,EAAE;YACxD,OAAO,CAAC,GAAG,CAAC,iBAAiB,GAAG,wBAAwB,CAAC;YACzD,UAAU,CAAC,iBAAiB,CAAC;gBAC3B,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;aACxC,CAAC,CAAC;YAEH,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;YACjC,MAAM,MAAM,CAAC,QAAQ,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;YAEpC,MAAM,CAAC,UAAU,CAAC,CAAC,oBAAoB,CACrC,MAAM,CAAC,gBAAgB,CAAC,EAAE,KAAK,EAAE,wBAAwB,EAAE,CAAC,EAC5D,MAAM,CAAC,QAAQ,EAAE,CAClB,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,mCAAmC,EAAE,KAAK,IAAI,EAAE;YACjD,UAAU,CAAC,iBAAiB,CAAC,IAAI,KAAK,CAAC,oBAAoB,CAAC,CAAC,CAAC;YAE9D,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;YACjC,MAAM,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CACzD,oBAAoB,CACrB,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,0CAA0C,EAAE,KAAK,IAAI,EAAE;YACxD,UAAU,CAAC,iBAAiB,CAAC;gBAC3B,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;aACxC,CAAC,CAAC;YAEH,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;YACjC,MAAM,MAAM,CAAC,QAAQ,CAAC,kBAAkB,EAAE,gBAAgB,CAAC,CAAC;YAE5D,MAAM,CAAC,UAAU,CAAC,CAAC,oBAAoB,CACrC,MAAM,CAAC,gBAAgB,CAAC;gBACtB,MAAM,EAAE,kBAAkB;gBAC1B,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,gBAAgB,EAAE,CAAC;gBACvD,UAAU,EAAE,IAAI;aACjB,CAAC,EACF,MAAM,CAAC,QAAQ,EAAE,CAClB,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,0CAA0C,EAAE,GAAG,EAAE;YAClD,OAAO,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC;YACrC,MAAM,CAAC,GAAG,EAAE,CAAC,eAAe,EAAE,CAAC,CAAC,OAAO,CAAC,8BAA8B,CAAC,CAAC;QAC1E,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,4EAA4E;IAC5E,sBAAsB;IACtB,4EAA4E;IAE5E,QAAQ,CAAC,qBAAqB,EAAE,GAAG,EAAE;QACnC,UAAU,CAAC,GAAG,EAAE;YACd,OAAO,CAAC,GAAG,CAAC,oBAAoB,GAAG,YAAY,CAAC;QAClD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,+DAA+D,EAAE,KAAK,IAAI,EAAE;YAC7E,SAAS,CAAC,eAAe,CAAC,cAAc,CAAC,gBAAgB,CAAC,CAAC,CAAC;YAE5D,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;YACjC,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC,eAAe,EAAE,aAAa,CAAC,CAAC;YAErE,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;YACpC,MAAM,CAAC,SAAS,CAAC,CAAC,oBAAoB,CACpC,QAAQ,EACR,CAAC,IAAI,EAAE,SAAS,EAAE,QAAQ,CAAC,EAC3B,MAAM,CAAC,gBAAgB,CAAC,EAAE,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC,CAC7D,CAAC;YACF,oCAAoC;YACpC,MAAM,IAAI,GAAG,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;YAC7C,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,oBAAoB,CAAC,8BAA8B,CAAC,CAAC;QAC9E,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,0BAA0B,EAAE,GAAG,EAAE;YAClC,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;YACjC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;QAC7C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,4CAA4C,EAAE,KAAK,IAAI,EAAE;YAC1D,OAAO,CAAC,GAAG,CAAC,iBAAiB,GAAG,MAAM,CAAC;YACvC,SAAS,CAAC,eAAe,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC;YAElD,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;YACjC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;YACzC,MAAM,MAAM,CAAC,QAAQ,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;YAEpC,MAAM,CAAC,SAAS,CAAC,CAAC,oBAAoB,CACpC,QAAQ,EACR,CAAC,IAAI,EAAE,SAAS,EAAE,MAAM,CAAC,EACzB,MAAM,CAAC,QAAQ,EAAE,CAClB,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,gDAAgD,EAAE,KAAK,IAAI,EAAE;YAC9D,MAAM,IAAI,GAAG,IAAI,YAAY,EAAS,CAAC;YACvC,IAAI,CAAC,MAAM,GAAG,IAAI,YAAY,EAAE,CAAC;YACjC,IAAI,CAAC,MAAM,GAAG,IAAI,YAAY,EAAE,CAAC;YACjC,IAAI,CAAC,KAAK,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC;YAC9C,IAAI,CAAC,IAAI,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;YACpB,SAAS,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC;YAEhC,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;YACjC,MAAM,OAAO,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;YAE9C,oBAAoB;YACpB,UAAU,CAAC,GAAG,EAAE;gBACd,MAAM,GAAG,GAAG,IAAI,KAAK,CAAC,qBAAqB,CAA0B,CAAC;gBACtE,GAAG,CAAC,IAAI,GAAG,QAAQ,CAAC;gBACpB,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;YAC1B,CAAC,EAAE,CAAC,CAAC,CAAC;YAEN,MAAM,MAAM,CAAC,OAAO,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,sBAAsB,CAAC,CAAC;QAChE,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,8CAA8C,EAAE,KAAK,IAAI,EAAE;YAC5D,SAAS,CAAC,eAAe,CAAC,cAAc,CAAC,EAAE,EAAE,CAAC,EAAE,sBAAsB,CAAC,CAAC,CAAC;YAEzE,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;YACjC,MAAM,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CACzD,qDAAqD,CACtD,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,4DAA4D,EAAE,GAAG,EAAE;YACpE,OAAO,CAAC,GAAG,CAAC,UAAU,GAAG,GAAG,CAAC;YAC7B,MAAM,CAAC,GAAG,EAAE,CAAC,eAAe,EAAE,CAAC,CAAC,OAAO,CACrC,6DAA6D,CAC9D,CAAC;QACJ,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,4EAA4E;IAC5E,kBAAkB;IAClB,4EAA4E;IAE5E,QAAQ,CAAC,iBAAiB,EAAE,GAAG,EAAE;QAC/B,UAAU,CAAC,GAAG,EAAE;YACd,OAAO,CAAC,GAAG,CAAC,oBAAoB,GAAG,QAAQ,CAAC;QAC9C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,gCAAgC,EAAE,GAAG,EAAE;YACxC,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;YACjC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QAC3C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,0CAA0C,EAAE,GAAG,EAAE;YAClD,OAAO,CAAC,GAAG,CAAC,iBAAiB,GAAG,aAAa,CAAC;YAC9C,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;YACjC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QAC3C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,4CAA4C,EAAE,KAAK,IAAI,EAAE;YAC1D,MAAM,SAAS,GAAG,EAAE,CAAC,KAAK,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC,iBAAiB,CAC/D,IAAI,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,QAAQ,EAAE,cAAc,EAAE,CAAC,EAAE,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,CAC5E,CAAC;YAEF,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;YACjC,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC,eAAe,EAAE,aAAa,CAAC,CAAC;YAErE,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;YACpC,MAAM,CAAC,SAAS,CAAC,CAAC,oBAAoB,CACpC,qCAAqC,EACrC,MAAM,CAAC,gBAAgB,CAAC;gBACtB,MAAM,EAAE,MAAM;gBACd,IAAI,EAAE,MAAM,CAAC,gBAAgB,CAAC,uBAAuB,CAAC;aACvD,CAAC,CACH,CAAC;YAEF,SAAS,CAAC,WAAW,EAAE,CAAC;QAC1B,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,2CAA2C,EAAE,KAAK,IAAI,EAAE;YACzD,OAAO,CAAC,GAAG,CAAC,eAAe,GAAG,yBAAyB,CAAC;YAExD,MAAM,SAAS,GAAG,EAAE,CAAC,KAAK,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC,iBAAiB,CAC/D,IAAI,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,EAAE,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,CAClE,CAAC;YAEF,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;YACjC,MAAM,MAAM,CAAC,QAAQ,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;YAEpC,MAAM,CAAC,SAAS,CAAC,CAAC,oBAAoB,CACpC,sCAAsC,EACtC,MAAM,CAAC,QAAQ,EAAE,CAClB,CAAC;YAEF,SAAS,CAAC,WAAW,EAAE,CAAC;QAC1B,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
|
1
|
+
{"version":3,"file":"llm.test.js","sourceRoot":"","sources":["../../../src/eval/__tests__/llm.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AAEzE,8EAA8E;AAC9E,QAAQ;AACR,8EAA8E;AAE9E,MAAM,UAAU,GAAG,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;AAC7C,MAAM,SAAS,GAAG,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;AAE5C,EAAE,CAAC,IAAI,CAAC,mBAAmB,EAAE,GAAG,EAAE,CAAC,CAAC;IAClC,OAAO,EAAE,MAAM,aAAa;QAC1B,QAAQ,GAAG,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC;KACnC;CACF,CAAC,CAAC,CAAC;AAEJ,EAAE,CAAC,IAAI,CAAC,oBAAoB,EAAE,GAAG,EAAE,CAAC,CAAC;IACnC,KAAK,EAAE,SAAS;CACjB,CAAC,CAAC,CAAC;AAEJ,8EAA8E;AAC9E,yDAAyD;AACzD,8EAA8E;AAC9E,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAE3C,SAAS,cAAc,CAAC,UAAkB,EAAE,QAAQ,GAAG,CAAC,EAAE,UAAU,GAAG,EAAE;IACvE,MAAM,IAAI,GAAG,IAAI,YAAY,EAAS,CAAC;IACvC,IAAI,CAAC,MAAM,GAAG,IAAI,YAAY,EAAE,CAAC;IACjC,IAAI,CAAC,MAAM,GAAG,IAAI,YAAY,EAAE,CAAC;IACjC,IAAI,CAAC,KAAK,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC;IAC9C,IAAI,CAAC,IAAI,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;IAEpB,4CAA4C;IAC5C,UAAU,CAAC,GAAG,EAAE;QACd,IAAI,UAAU;YAAE,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC;QAClE,IAAI,UAAU;YAAE,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC;QAClE,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;IAC/B,CAAC,EAAE,CAAC,CAAC,CAAC;IAEN,OAAO,IAAI,CAAC;AACd,CAAC;AAED,8EAA8E;AAC9E,uCAAuC;AACvC,8EAA8E;AAE9E,MAAM,EAAE,eAAe,EAAE,GAAG,MAAM,MAAM,CAAC,WAAW,CAAC,CAAC;AAEtD,8EAA8E;AAC9E,QAAQ;AACR,8EAA8E;AAE9E,QAAQ,CAAC,iBAAiB,EAAE,GAAG,EAAE;IAC/B,MAAM,OAAO,GAAG,EAAE,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC;IAEnC,UAAU,CAAC,GAAG,EAAE;QACd,EAAE,CAAC,aAAa,EAAE,CAAC;QACnB,OAAO,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC;QACxC,OAAO,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC;QACrC,OAAO,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC;QACrC,OAAO,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC;QACnC,OAAO,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC;IAChC,CAAC,CAAC,CAAC;IAEH,SAAS,CAAC,GAAG,EAAE;QACb,OAAO,CAAC,GAAG,GAAG,EAAE,GAAG,OAAO,EAAE,CAAC;IAC/B,CAAC,CAAC,CAAC;IAEH,4EAA4E;IAC5E,iBAAiB;IACjB,4EAA4E;IAE5E,EAAE,CAAC,8CAA8C,EAAE,GAAG,EAAE;QACtD,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;QACjC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;IAC7C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,gDAAgD,EAAE,GAAG,EAAE;QACxD,OAAO,CAAC,GAAG,CAAC,UAAU,GAAG,GAAG,CAAC;QAC7B,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;QACjC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;IAC3C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,sDAAsD,EAAE,GAAG,EAAE;QAC9D,OAAO,CAAC,GAAG,CAAC,iBAAiB,GAAG,UAAU,CAAC;QAC3C,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;QACjC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;IACxD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,qEAAqE,EAAE,GAAG,EAAE;QAC7E,OAAO,CAAC,GAAG,CAAC,UAAU,GAAG,GAAG,CAAC;QAC7B,OAAO,CAAC,GAAG,CAAC,iBAAiB,GAAG,UAAU,CAAC;QAC3C,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;QACjC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;IAC3C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,wDAAwD,EAAE,GAAG,EAAE;QAChE,OAAO,CAAC,GAAG,CAAC,oBAAoB,GAAG,QAAQ,CAAC;QAC5C,OAAO,CAAC,GAAG,CAAC,iBAAiB,GAAG,UAAU,CAAC;QAC3C,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;QACjC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;IAC3C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,4BAA4B,EAAE,GAAG,EAAE;QACpC,OAAO,CAAC,GAAG,CAAC,oBAAoB,GAAG,WAAW,CAAC;QAC/C,MAAM,CAAC,GAAG,EAAE,CAAC,eAAe,EAAE,CAAC,CAAC,OAAO,CAAC,2CAA2C,CAAC,CAAC;IACvF,CAAC,CAAC,CAAC;IAEH,4EAA4E;IAC5E,kBAAkB;IAClB,4EAA4E;IAE5E,EAAE,CAAC,iDAAiD,EAAE,GAAG,EAAE;QACzD,OAAO,CAAC,GAAG,CAAC,oBAAoB,GAAG,QAAQ,CAAC;QAC5C,OAAO,CAAC,GAAG,CAAC,iBAAiB,GAAG,UAAU,CAAC;QAC3C,MAAM,MAAM,GAAG,eAAe,CAAC,EAAE,QAAQ,EAAE,WAAW,EAAE,CAAC,CAAC;QAC1D,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;IACxD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,8CAA8C,EAAE,GAAG,EAAE;QACtD,OAAO,CAAC,GAAG,CAAC,iBAAiB,GAAG,UAAU,CAAC;QAC3C,MAAM,MAAM,GAAG,eAAe,CAAC,EAAE,QAAQ,EAAE,WAAW,EAAE,KAAK,EAAE,wBAAwB,EAAE,CAAC,CAAC;QAC3F,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAC;IACtD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,gDAAgD,EAAE,GAAG,EAAE;QACxD,MAAM,MAAM,GAAG,eAAe,CAAC,EAAE,QAAQ,EAAE,YAAY,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC;QAC1E,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;IAC3C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,4CAA4C,EAAE,GAAG,EAAE;QACpD,MAAM,MAAM,GAAG,eAAe,CAAC,EAAE,QAAQ,EAAE,QAAQ,EAAE,KAAK,EAAE,aAAa,EAAE,CAAC,CAAC;QAC7E,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;IAC3C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,gEAAgE,EAAE,GAAG,EAAE;QACxE,OAAO,CAAC,GAAG,CAAC,UAAU,GAAG,GAAG,CAAC;QAC7B,8CAA8C;QAC9C,MAAM,MAAM,GAAG,eAAe,CAAC,EAAE,QAAQ,EAAE,YAAY,EAAE,CAAC,CAAC;QAC3D,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;IAC7C,CAAC,CAAC,CAAC;IAEH,4EAA4E;IAC5E,qBAAqB;IACrB,4EAA4E;IAE5E,QAAQ,CAAC,oBAAoB,EAAE,GAAG,EAAE;QAClC,UAAU,CAAC,GAAG,EAAE;YACd,OAAO,CAAC,GAAG,CAAC,oBAAoB,GAAG,WAAW,CAAC;YAC/C,OAAO,CAAC,GAAG,CAAC,iBAAiB,GAAG,UAAU,CAAC;QAC7C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,kDAAkD,EAAE,KAAK,IAAI,EAAE;YAChE,UAAU,CAAC,iBAAiB,CAAC;gBAC3B,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,oBAAoB,EAAE,CAAC;aACxD,CAAC,CAAC;YAEH,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;YACjC,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC,eAAe,EAAE,aAAa,CAAC,CAAC;YAErE,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;YAC1C,MAAM,CAAC,UAAU,CAAC,CAAC,oBAAoB,EAAE,CAAC;QAC5C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,6CAA6C,EAAE,KAAK,IAAI,EAAE;YAC3D,UAAU,CAAC,iBAAiB,CAAC;gBAC3B,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;aACxC,CAAC,CAAC;YAEH,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;YACjC,MAAM,MAAM,CAAC,QAAQ,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;YAEpC,MAAM,CAAC,UAAU,CAAC,CAAC,oBAAoB,CACrC,MAAM,CAAC,gBAAgB,CAAC,EAAE,KAAK,EAAE,0BAA0B,EAAE,CAAC,EAC9D,MAAM,CAAC,QAAQ,EAAE,CAClB,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,0CAA0C,EAAE,KAAK,IAAI,EAAE;YACxD,OAAO,CAAC,GAAG,CAAC,iBAAiB,GAAG,wBAAwB,CAAC;YACzD,UAAU,CAAC,iBAAiB,CAAC;gBAC3B,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;aACxC,CAAC,CAAC;YAEH,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;YACjC,MAAM,MAAM,CAAC,QAAQ,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;YAEpC,MAAM,CAAC,UAAU,CAAC,CAAC,oBAAoB,CACrC,MAAM,CAAC,gBAAgB,CAAC,EAAE,KAAK,EAAE,wBAAwB,EAAE,CAAC,EAC5D,MAAM,CAAC,QAAQ,EAAE,CAClB,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,mCAAmC,EAAE,KAAK,IAAI,EAAE;YACjD,UAAU,CAAC,iBAAiB,CAAC,IAAI,KAAK,CAAC,oBAAoB,CAAC,CAAC,CAAC;YAE9D,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;YACjC,MAAM,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CACzD,oBAAoB,CACrB,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,0CAA0C,EAAE,KAAK,IAAI,EAAE;YACxD,UAAU,CAAC,iBAAiB,CAAC;gBAC3B,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;aACxC,CAAC,CAAC;YAEH,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;YACjC,MAAM,MAAM,CAAC,QAAQ,CAAC,kBAAkB,EAAE,gBAAgB,CAAC,CAAC;YAE5D,MAAM,CAAC,UAAU,CAAC,CAAC,oBAAoB,CACrC,MAAM,CAAC,gBAAgB,CAAC;gBACtB,MAAM,EAAE,kBAAkB;gBAC1B,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,gBAAgB,EAAE,CAAC;gBACvD,UAAU,EAAE,IAAI;aACjB,CAAC,EACF,MAAM,CAAC,QAAQ,EAAE,CAClB,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,0CAA0C,EAAE,GAAG,EAAE;YAClD,OAAO,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC;YACrC,MAAM,CAAC,GAAG,EAAE,CAAC,eAAe,EAAE,CAAC,CAAC,OAAO,CAAC,8BAA8B,CAAC,CAAC;QAC1E,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,4EAA4E;IAC5E,sBAAsB;IACtB,4EAA4E;IAE5E,QAAQ,CAAC,qBAAqB,EAAE,GAAG,EAAE;QACnC,UAAU,CAAC,GAAG,EAAE;YACd,OAAO,CAAC,GAAG,CAAC,oBAAoB,GAAG,YAAY,CAAC;QAClD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,+DAA+D,EAAE,KAAK,IAAI,EAAE;YAC7E,SAAS,CAAC,eAAe,CAAC,cAAc,CAAC,gBAAgB,CAAC,CAAC,CAAC;YAE5D,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;YACjC,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC,eAAe,EAAE,aAAa,CAAC,CAAC;YAErE,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;YACpC,MAAM,CAAC,SAAS,CAAC,CAAC,oBAAoB,CACpC,QAAQ,EACR,CAAC,IAAI,EAAE,SAAS,EAAE,QAAQ,CAAC,EAC3B,MAAM,CAAC,gBAAgB,CAAC,EAAE,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC,CAC7D,CAAC;YACF,oCAAoC;YACpC,MAAM,IAAI,GAAG,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;YAC7C,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,oBAAoB,CAAC,8BAA8B,CAAC,CAAC;QAC9E,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,0BAA0B,EAAE,GAAG,EAAE;YAClC,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;YACjC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;QAC7C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,4CAA4C,EAAE,KAAK,IAAI,EAAE;YAC1D,OAAO,CAAC,GAAG,CAAC,iBAAiB,GAAG,MAAM,CAAC;YACvC,SAAS,CAAC,eAAe,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC;YAElD,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;YACjC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;YACzC,MAAM,MAAM,CAAC,QAAQ,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;YAEpC,MAAM,CAAC,SAAS,CAAC,CAAC,oBAAoB,CACpC,QAAQ,EACR,CAAC,IAAI,EAAE,SAAS,EAAE,MAAM,CAAC,EACzB,MAAM,CAAC,QAAQ,EAAE,CAClB,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,gDAAgD,EAAE,KAAK,IAAI,EAAE;YAC9D,MAAM,IAAI,GAAG,IAAI,YAAY,EAAS,CAAC;YACvC,IAAI,CAAC,MAAM,GAAG,IAAI,YAAY,EAAE,CAAC;YACjC,IAAI,CAAC,MAAM,GAAG,IAAI,YAAY,EAAE,CAAC;YACjC,IAAI,CAAC,KAAK,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC;YAC9C,IAAI,CAAC,IAAI,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;YACpB,SAAS,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC;YAEhC,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;YACjC,MAAM,OAAO,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;YAE9C,oBAAoB;YACpB,UAAU,CAAC,GAAG,EAAE;gBACd,MAAM,GAAG,GAAG,IAAI,KAAK,CAAC,qBAAqB,CAA0B,CAAC;gBACtE,GAAG,CAAC,IAAI,GAAG,QAAQ,CAAC;gBACpB,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;YAC1B,CAAC,EAAE,CAAC,CAAC,CAAC;YAEN,MAAM,MAAM,CAAC,OAAO,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,sBAAsB,CAAC,CAAC;QAChE,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,8CAA8C,EAAE,KAAK,IAAI,EAAE;YAC5D,SAAS,CAAC,eAAe,CAAC,cAAc,CAAC,EAAE,EAAE,CAAC,EAAE,sBAAsB,CAAC,CAAC,CAAC;YAEzE,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;YACjC,MAAM,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CACzD,qDAAqD,CACtD,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,4DAA4D,EAAE,GAAG,EAAE;YACpE,OAAO,CAAC,GAAG,CAAC,UAAU,GAAG,GAAG,CAAC;YAC7B,MAAM,CAAC,GAAG,EAAE,CAAC,eAAe,EAAE,CAAC,CAAC,OAAO,CACrC,6DAA6D,CAC9D,CAAC;QACJ,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,4EAA4E;IAC5E,kBAAkB;IAClB,4EAA4E;IAE5E,QAAQ,CAAC,iBAAiB,EAAE,GAAG,EAAE;QAC/B,UAAU,CAAC,GAAG,EAAE;YACd,OAAO,CAAC,GAAG,CAAC,oBAAoB,GAAG,QAAQ,CAAC;QAC9C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,gCAAgC,EAAE,GAAG,EAAE;YACxC,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;YACjC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QAC3C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,0CAA0C,EAAE,GAAG,EAAE;YAClD,OAAO,CAAC,GAAG,CAAC,iBAAiB,GAAG,aAAa,CAAC;YAC9C,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;YACjC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QAC3C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,4CAA4C,EAAE,KAAK,IAAI,EAAE;YAC1D,MAAM,SAAS,GAAG,EAAE,CAAC,KAAK,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC,iBAAiB,CAC/D,IAAI,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,QAAQ,EAAE,cAAc,EAAE,CAAC,EAAE,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,CAC5E,CAAC;YAEF,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;YACjC,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC,eAAe,EAAE,aAAa,CAAC,CAAC;YAErE,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;YACpC,MAAM,CAAC,SAAS,CAAC,CAAC,oBAAoB,CACpC,qCAAqC,EACrC,MAAM,CAAC,gBAAgB,CAAC;gBACtB,MAAM,EAAE,MAAM;gBACd,IAAI,EAAE,MAAM,CAAC,gBAAgB,CAAC,uBAAuB,CAAC;aACvD,CAAC,CACH,CAAC;YAEF,SAAS,CAAC,WAAW,EAAE,CAAC;QAC1B,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,2CAA2C,EAAE,KAAK,IAAI,EAAE;YACzD,OAAO,CAAC,GAAG,CAAC,eAAe,GAAG,yBAAyB,CAAC;YAExD,MAAM,SAAS,GAAG,EAAE,CAAC,KAAK,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC,iBAAiB,CAC/D,IAAI,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,EAAE,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,CAClE,CAAC;YAEF,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;YACjC,MAAM,MAAM,CAAC,QAAQ,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;YAEpC,MAAM,CAAC,SAAS,CAAC,CAAC,oBAAoB,CACpC,sCAAsC,EACtC,MAAM,CAAC,QAAQ,EAAE,CAClB,CAAC;YAEF,SAAS,CAAC,WAAW,EAAE,CAAC;QAC1B,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
package/dist/eval/llm.d.ts
CHANGED
|
@@ -2,4 +2,9 @@ export interface LlmClient {
|
|
|
2
2
|
generate(systemPrompt: string, userPrompt: string): Promise<string>;
|
|
3
3
|
readonly model: string;
|
|
4
4
|
}
|
|
5
|
-
export
|
|
5
|
+
export type ProviderName = "anthropic" | "claude-cli" | "ollama";
|
|
6
|
+
export interface LlmOverrides {
|
|
7
|
+
provider?: ProviderName;
|
|
8
|
+
model?: string;
|
|
9
|
+
}
|
|
10
|
+
export declare function createLlmClient(overrides?: LlmOverrides): LlmClient;
|
package/dist/eval/llm.js
CHANGED
|
@@ -24,15 +24,17 @@ function detectProvider() {
|
|
|
24
24
|
return "anthropic";
|
|
25
25
|
return "claude-cli";
|
|
26
26
|
}
|
|
27
|
-
export function createLlmClient() {
|
|
28
|
-
const provider = (process.env.VSKILL_EVAL_PROVIDER || detectProvider());
|
|
27
|
+
export function createLlmClient(overrides) {
|
|
28
|
+
const provider = (overrides?.provider || process.env.VSKILL_EVAL_PROVIDER || detectProvider());
|
|
29
|
+
const modelOverride = overrides?.model;
|
|
30
|
+
const forced = !!overrides?.provider;
|
|
29
31
|
switch (provider) {
|
|
30
32
|
case "anthropic":
|
|
31
|
-
return createAnthropicClient();
|
|
33
|
+
return createAnthropicClient(modelOverride);
|
|
32
34
|
case "claude-cli":
|
|
33
|
-
return createClaudeCliClient();
|
|
35
|
+
return createClaudeCliClient(modelOverride, forced);
|
|
34
36
|
case "ollama":
|
|
35
|
-
return createOllamaClient();
|
|
37
|
+
return createOllamaClient(modelOverride);
|
|
36
38
|
default:
|
|
37
39
|
throw new Error(`Unknown VSKILL_EVAL_PROVIDER: "${provider}". Use "claude-cli", "anthropic", or "ollama".`);
|
|
38
40
|
}
|
|
@@ -40,13 +42,13 @@ export function createLlmClient() {
|
|
|
40
42
|
// ---------------------------------------------------------------------------
|
|
41
43
|
// Provider: Anthropic API
|
|
42
44
|
// ---------------------------------------------------------------------------
|
|
43
|
-
function createAnthropicClient() {
|
|
45
|
+
function createAnthropicClient(modelOverride) {
|
|
44
46
|
const DEFAULT_MODEL = "claude-sonnet-4-20250514";
|
|
45
47
|
const apiKey = process.env.ANTHROPIC_API_KEY;
|
|
46
48
|
if (!apiKey) {
|
|
47
49
|
throw new Error("ANTHROPIC_API_KEY is not set. Export it before running eval commands:\n export ANTHROPIC_API_KEY=sk-ant-...\n\nOr use a different provider:\n export VSKILL_EVAL_PROVIDER=claude-cli # uses your Claude Max plan\n export VSKILL_EVAL_PROVIDER=ollama # uses local Ollama");
|
|
48
50
|
}
|
|
49
|
-
const model = process.env.VSKILL_EVAL_MODEL || DEFAULT_MODEL;
|
|
51
|
+
const model = modelOverride || process.env.VSKILL_EVAL_MODEL || DEFAULT_MODEL;
|
|
50
52
|
let clientInstance = null;
|
|
51
53
|
return {
|
|
52
54
|
model,
|
|
@@ -81,11 +83,11 @@ function createAnthropicClient() {
|
|
|
81
83
|
// From a plain terminal: npx vskill eval run mobile/appstore
|
|
82
84
|
// Select model: VSKILL_EVAL_MODEL=opus npx vskill eval run mobile/appstore
|
|
83
85
|
// ---------------------------------------------------------------------------
|
|
84
|
-
function createClaudeCliClient() {
|
|
85
|
-
if (process.env.CLAUDECODE) {
|
|
86
|
+
function createClaudeCliClient(modelOverride, forced = false) {
|
|
87
|
+
if (process.env.CLAUDECODE && !forced) {
|
|
86
88
|
throw new Error("Cannot use claude-cli provider inside a Claude Code session.\nRun from a plain terminal, or use a different provider:\n export VSKILL_EVAL_PROVIDER=ollama");
|
|
87
89
|
}
|
|
88
|
-
const model = process.env.VSKILL_EVAL_MODEL || "sonnet";
|
|
90
|
+
const model = modelOverride || process.env.VSKILL_EVAL_MODEL || "sonnet";
|
|
89
91
|
return {
|
|
90
92
|
model: `claude-${model}`,
|
|
91
93
|
async generate(systemPrompt, userPrompt) {
|
|
@@ -130,9 +132,9 @@ function createClaudeCliClient() {
|
|
|
130
132
|
// ---------------------------------------------------------------------------
|
|
131
133
|
// Provider: Ollama (local models — free, no API key)
|
|
132
134
|
// ---------------------------------------------------------------------------
|
|
133
|
-
function createOllamaClient() {
|
|
135
|
+
function createOllamaClient(modelOverride) {
|
|
134
136
|
const baseUrl = process.env.OLLAMA_BASE_URL || "http://localhost:11434";
|
|
135
|
-
const model = process.env.VSKILL_EVAL_MODEL || "llama3.1:8b";
|
|
137
|
+
const model = modelOverride || process.env.VSKILL_EVAL_MODEL || "llama3.1:8b";
|
|
136
138
|
return {
|
|
137
139
|
model,
|
|
138
140
|
async generate(systemPrompt, userPrompt) {
|
package/dist/eval/llm.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"llm.js","sourceRoot":"","sources":["../../src/eval/llm.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,gFAAgF;AAChF,EAAE;AACF,uDAAuD;AACvD,wEAAwE;AACxE,8DAA8D;AAC9D,uEAAuE;AACvE,EAAE;AACF,uDAAuD;AACvD,4DAA4D;AAC5D,6CAA6C;AAC7C,2DAA2D;AAC3D,EAAE;AACF,iDAAiD;AACjD,+DAA+D;AAC/D,mEAAmE;AACnE,mDAAmD;AACnD,8EAA8E;AAE9E,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAS3C,SAAS,cAAc;IACrB,IAAI,OAAO,CAAC,GAAG,CAAC,UAAU;QAAE,OAAO,QAAQ,CAAC;IAC5C,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB;QAAE,OAAO,WAAW,CAAC;IACtD,OAAO,YAAY,CAAC;AACtB,CAAC;
|
|
1
|
+
{"version":3,"file":"llm.js","sourceRoot":"","sources":["../../src/eval/llm.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,gFAAgF;AAChF,EAAE;AACF,uDAAuD;AACvD,wEAAwE;AACxE,8DAA8D;AAC9D,uEAAuE;AACvE,EAAE;AACF,uDAAuD;AACvD,4DAA4D;AAC5D,6CAA6C;AAC7C,2DAA2D;AAC3D,EAAE;AACF,iDAAiD;AACjD,+DAA+D;AAC/D,mEAAmE;AACnE,mDAAmD;AACnD,8EAA8E;AAE9E,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAS3C,SAAS,cAAc;IACrB,IAAI,OAAO,CAAC,GAAG,CAAC,UAAU;QAAE,OAAO,QAAQ,CAAC;IAC5C,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB;QAAE,OAAO,WAAW,CAAC;IACtD,OAAO,YAAY,CAAC;AACtB,CAAC;AAOD,MAAM,UAAU,eAAe,CAAC,SAAwB;IACtD,MAAM,QAAQ,GAAG,CAAC,SAAS,EAAE,QAAQ,IAAI,OAAO,CAAC,GAAG,CAAC,oBAAoB,IAAI,cAAc,EAAE,CAAiB,CAAC;IAC/G,MAAM,aAAa,GAAG,SAAS,EAAE,KAAK,CAAC;IACvC,MAAM,MAAM,GAAG,CAAC,CAAC,SAAS,EAAE,QAAQ,CAAC;IAErC,QAAQ,QAAQ,EAAE,CAAC;QACjB,KAAK,WAAW;YACd,OAAO,qBAAqB,CAAC,aAAa,CAAC,CAAC;QAC9C,KAAK,YAAY;YACf,OAAO,qBAAqB,CAAC,aAAa,EAAE,MAAM,CAAC,CAAC;QACtD,KAAK,QAAQ;YACX,OAAO,kBAAkB,CAAC,aAAa,CAAC,CAAC;QAC3C;YACE,MAAM,IAAI,KAAK,CACb,kCAAkC,QAAQ,gDAAgD,CAC3F,CAAC;IACN,CAAC;AACH,CAAC;AAED,8EAA8E;AAC9E,0BAA0B;AAC1B,8EAA8E;AAC9E,SAAS,qBAAqB,CAAC,aAAsB;IACnD,MAAM,aAAa,GAAG,0BAA0B,CAAC;IAEjD,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC;IAC7C,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,MAAM,IAAI,KAAK,CACb,sRAAsR,CACvR,CAAC;IACJ,CAAC;IAED,MAAM,KAAK,GAAG,aAAa,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,aAAa,CAAC;IAC9E,IAAI,cAAc,GAAQ,IAAI,CAAC;IAE/B,OAAO;QACL,KAAK;QACL,KAAK,CAAC,QAAQ,CAAC,YAAoB,EAAE,UAAkB;YACrD,IAAI,CAAC,cAAc,EAAE,CAAC;gBACpB,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,GAAG,MAAM,MAAM,CAAC,mBAAmB,CAAC,CAAC;gBACjE,cAAc,GAAG,IAAI,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;YAC7C,CAAC;YAED,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;YACzC,MAAM,OAAO,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,OAAO,CAAC,CAAC;YAC9D,IAAI,CAAC;gBACH,MAAM,QAAQ,GAAG,MAAM,cAAc,CAAC,QAAQ,CAAC,MAAM,CACnD;oBACE,KAAK;oBACL,MAAM,EAAE,YAAY;oBACpB,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,CAAC;oBACjD,UAAU,EAAE,IAAI;iBACjB,EACD,EAAE,MAAM,EAAE,UAAU,CAAC,MAAM,EAAE,CAC9B,CAAC;gBAEF,MAAM,SAAS,GAAG,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC;gBACvE,OAAO,SAAS,IAAI,MAAM,IAAI,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;YAChE,CAAC;oBAAS,CAAC;gBACT,YAAY,CAAC,OAAO,CAAC,CAAC;YACxB,CAAC;QACH,CAAC;KACF,CAAC;AACJ,CAAC;AAED,8EAA8E;AAC9E,4EAA4E;AAC5E,EAAE;AACF,uEAAuE;AACvE,EAAE;AACF,6DAA6D;AAC7D,oFAAoF;AACpF,8EAA8E;AAC9E,SAAS,qBAAqB,CAAC,aAAsB,EAAE,MAAM,GAAG,KAAK;IACnE,IAAI,OAAO,CAAC,GAAG,CAAC,UAAU,IAAI,CAAC,MAAM,EAAE,CAAC;QACtC,MAAM,IAAI,KAAK,CACb,6JAA6J,CAC9J,CAAC;IACJ,CAAC;IAED,MAAM,KAAK,GAAG,aAAa,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,QAAQ,CAAC;IAEzE,OAAO;QACL,KAAK,EAAE,UAAU,KAAK,EAAE;QACxB,KAAK,CAAC,QAAQ,CAAC,YAAoB,EAAE,UAAkB;YACrD,MAAM,cAAc,GAAG,GAAG,YAAY,OAAO,UAAU,EAAE,CAAC;YAE1D,OAAO,IAAI,OAAO,CAAS,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;gBAC7C,MAAM,IAAI,GAAG,KAAK,CAAC,QAAQ,EAAE,CAAC,IAAI,EAAE,SAAS,EAAE,KAAK,CAAC,EAAE;oBACrD,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC;iBAChC,CAAC,CAAC;gBAEH,IAAI,MAAM,GAAG,EAAE,CAAC;gBAChB,IAAI,MAAM,GAAG,EAAE,CAAC;gBAChB,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,CAAS,EAAE,EAAE,GAAG,MAAM,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;gBACnE,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,CAAS,EAAE,EAAE,GAAG,MAAM,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;gBAEnE,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE;oBAC5B,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;oBACrB,MAAM,CAAC,IAAI,KAAK,CAAC,iCAAiC,CAAC,CAAC,CAAC;gBACvD,CAAC,EAAE,OAAO,CAAC,CAAC;gBAEZ,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAA0B,EAAE,EAAE;oBAC9C,YAAY,CAAC,KAAK,CAAC,CAAC;oBACpB,IAAI,GAAG,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;wBAC1B,MAAM,CAAC,IAAI,KAAK,CACd,qJAAqJ,CACtJ,CAAC,CAAC;oBACL,CAAC;yBAAM,CAAC;wBACN,MAAM,CAAC,IAAI,KAAK,CAAC,sBAAsB,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;oBACzD,CAAC;gBACH,CAAC,CAAC,CAAC;gBAEH,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,EAAE;oBACxB,YAAY,CAAC,KAAK,CAAC,CAAC;oBACpB,IAAI,IAAI,KAAK,CAAC,EAAE,CAAC;wBACf,OAAO,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;oBACzB,CAAC;yBAAM,CAAC;wBACN,MAAM,MAAM,GAAG,CAAC,MAAM,IAAI,MAAM,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;wBAChD,MAAM,CAAC,IAAI,KAAK,CAAC,+BAA+B,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;oBACzF,CAAC;gBACH,CAAC,CAAC,CAAC;gBAEH,yEAAyE;gBACzE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;YACjC,CAAC,CAAC,CAAC;QACL,CAAC;KACF,CAAC;AACJ,CAAC;AAED,8EAA8E;AAC9E,qDAAqD;AACrD,8EAA8E;AAC9E,SAAS,kBAAkB,CAAC,aAAsB;IAChD,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,eAAe,IAAI,wBAAwB,CAAC;IACxE,MAAM,KAAK,GAAG,aAAa,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,aAAa,CAAC;IAE9E,OAAO;QACL,KAAK;QACL,KAAK,CAAC,QAAQ,CAAC,YAAoB,EAAE,UAAkB;YACrD,MAAM,UAAU,GAAG,GAAG,YAAY,OAAO,UAAU,EAAE,CAAC;YAEtD,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,OAAO,eAAe,EAAE;gBACtD,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE;gBAC/C,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;oBACnB,KAAK;oBACL,MAAM,EAAE,UAAU;oBAClB,MAAM,EAAE,KAAK;oBACb,OAAO,EAAE;wBACP,WAAW,EAAE,IAAI;wBACjB,WAAW,EAAE,GAAG;qBACjB;iBACF,CAAC;gBACF,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,OAAO,CAAC;aACrC,CAAC,CAAC;YAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;gBACjB,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;gBACpC,IAAI,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;oBAC3D,MAAM,IAAI,KAAK,CACb,iBAAiB,KAAK,8CAA8C,KAAK,EAAE,CAC5E,CAAC;gBACJ,CAAC;gBACD,MAAM,IAAI,KAAK,CAAC,0BAA0B,KAAK,EAAE,CAAC,CAAC;YACrD,CAAC;YAED,MAAM,IAAI,GAAG,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAA0B,CAAC;YAC9D,OAAO,IAAI,CAAC,QAAQ,IAAI,EAAE,CAAC;QAC7B,CAAC;KACF,CAAC;AACJ,CAAC"}
|
|
@@ -1,2 +1,2 @@
|
|
|
1
1
|
import type { Router } from "./router.js";
|
|
2
|
-
export declare function registerRoutes(router: Router, root: string): void;
|
|
2
|
+
export declare function registerRoutes(router: Router, root: string, projectName?: string): void;
|
|
@@ -29,19 +29,116 @@ function resolveSkillDir(root, plugin, skill) {
|
|
|
29
29
|
return rootPath;
|
|
30
30
|
return directPath;
|
|
31
31
|
}
|
|
32
|
-
|
|
32
|
+
// ---------------------------------------------------------------------------
|
|
33
|
+
// In-memory config state — UI can change provider/model at runtime.
|
|
34
|
+
//
|
|
35
|
+
// Default: claude-cli (Sonnet). The eval server is always run from a separate
|
|
36
|
+
// terminal, so claude-cli is always safe — even if CLAUDECODE env is set
|
|
37
|
+
// (which only matters for the `vskill eval run` CLI command).
|
|
38
|
+
// ---------------------------------------------------------------------------
|
|
39
|
+
let currentOverrides = { provider: "claude-cli" };
|
|
40
|
+
function getClient() {
|
|
41
|
+
return createLlmClient(currentOverrides);
|
|
42
|
+
}
|
|
43
|
+
const PROVIDER_MODELS = {
|
|
44
|
+
"claude-cli": [
|
|
45
|
+
{ id: "sonnet", label: "Claude Sonnet" },
|
|
46
|
+
{ id: "opus", label: "Claude Opus" },
|
|
47
|
+
{ id: "haiku", label: "Claude Haiku" },
|
|
48
|
+
],
|
|
49
|
+
"anthropic": [
|
|
50
|
+
{ id: "claude-sonnet-4-20250514", label: "Claude Sonnet 4 (API)" },
|
|
51
|
+
{ id: "claude-opus-4-20250514", label: "Claude Opus 4 (API)" },
|
|
52
|
+
{ id: "claude-haiku-4-20250414", label: "Claude Haiku 4 (API)" },
|
|
53
|
+
],
|
|
54
|
+
"ollama": [
|
|
55
|
+
{ id: "llama3.1:8b", label: "Llama 3.1 8B" },
|
|
56
|
+
{ id: "qwen2.5:32b", label: "Qwen 2.5 32B" },
|
|
57
|
+
{ id: "gemma2:9b", label: "Gemma 2 9B" },
|
|
58
|
+
{ id: "mistral:7b", label: "Mistral 7B" },
|
|
59
|
+
],
|
|
60
|
+
};
|
|
61
|
+
async function detectAvailableProviders() {
|
|
62
|
+
const providers = [];
|
|
63
|
+
// Claude CLI — always available for the eval server (runs in a separate terminal)
|
|
64
|
+
providers.push({
|
|
65
|
+
id: "claude-cli",
|
|
66
|
+
label: "Claude (Max/Pro subscription)",
|
|
67
|
+
available: true,
|
|
68
|
+
models: PROVIDER_MODELS["claude-cli"],
|
|
69
|
+
});
|
|
70
|
+
// Anthropic API — available if ANTHROPIC_API_KEY is set
|
|
71
|
+
providers.push({
|
|
72
|
+
id: "anthropic",
|
|
73
|
+
label: "Anthropic API (requires key)",
|
|
74
|
+
available: !!process.env.ANTHROPIC_API_KEY,
|
|
75
|
+
models: PROVIDER_MODELS["anthropic"],
|
|
76
|
+
});
|
|
77
|
+
// Ollama — probe the server for actually installed models
|
|
78
|
+
let ollamaModels = PROVIDER_MODELS["ollama"];
|
|
79
|
+
let ollamaAvailable = false;
|
|
80
|
+
try {
|
|
81
|
+
const baseUrl = process.env.OLLAMA_BASE_URL || "http://localhost:11434";
|
|
82
|
+
const resp = await fetch(`${baseUrl}/api/tags`, { signal: AbortSignal.timeout(2000) });
|
|
83
|
+
if (resp.ok) {
|
|
84
|
+
ollamaAvailable = true;
|
|
85
|
+
const data = await resp.json();
|
|
86
|
+
if (data.models?.length) {
|
|
87
|
+
ollamaModels = data.models.map((m) => ({ id: m.name, label: m.name }));
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
catch { /* ollama not running */ }
|
|
92
|
+
providers.push({
|
|
93
|
+
id: "ollama",
|
|
94
|
+
label: "Ollama (local, free)",
|
|
95
|
+
available: ollamaAvailable,
|
|
96
|
+
models: ollamaModels,
|
|
97
|
+
});
|
|
98
|
+
return providers;
|
|
99
|
+
}
|
|
100
|
+
export function registerRoutes(router, root, projectName) {
|
|
33
101
|
// Health check
|
|
34
102
|
router.get("/api/health", (_req, res) => {
|
|
35
103
|
sendJson(res, { ok: true });
|
|
36
104
|
});
|
|
37
|
-
// Config — expose current provider/model
|
|
38
|
-
router.get("/api/config", (_req, res) => {
|
|
105
|
+
// Config — expose current provider/model + available providers + project
|
|
106
|
+
router.get("/api/config", async (_req, res) => {
|
|
107
|
+
try {
|
|
108
|
+
const client = getClient();
|
|
109
|
+
const providers = await detectAvailableProviders();
|
|
110
|
+
sendJson(res, {
|
|
111
|
+
provider: currentOverrides.provider || null,
|
|
112
|
+
model: client.model,
|
|
113
|
+
providers,
|
|
114
|
+
projectName: projectName || null,
|
|
115
|
+
root,
|
|
116
|
+
});
|
|
117
|
+
}
|
|
118
|
+
catch (err) {
|
|
119
|
+
const providers = await detectAvailableProviders();
|
|
120
|
+
sendJson(res, { provider: null, model: "unknown", error: err.message, providers, projectName: projectName || null, root });
|
|
121
|
+
}
|
|
122
|
+
});
|
|
123
|
+
// Update config — change provider/model at runtime
|
|
124
|
+
router.post("/api/config", async (req, res) => {
|
|
125
|
+
const body = (await readBody(req));
|
|
126
|
+
if (body.provider)
|
|
127
|
+
currentOverrides.provider = body.provider;
|
|
128
|
+
if (body.model)
|
|
129
|
+
currentOverrides.model = body.model;
|
|
130
|
+
// If provider changed but no model, clear model override so it uses the provider default
|
|
131
|
+
if (body.provider && !body.model)
|
|
132
|
+
delete currentOverrides.model;
|
|
39
133
|
try {
|
|
40
|
-
const client =
|
|
41
|
-
|
|
134
|
+
const client = getClient();
|
|
135
|
+
const providers = await detectAvailableProviders();
|
|
136
|
+
sendJson(res, { provider: currentOverrides.provider || null, model: client.model, providers });
|
|
42
137
|
}
|
|
43
138
|
catch (err) {
|
|
44
|
-
|
|
139
|
+
// Revert on error
|
|
140
|
+
currentOverrides = {};
|
|
141
|
+
sendJson(res, { error: err.message }, 400, req);
|
|
45
142
|
}
|
|
46
143
|
});
|
|
47
144
|
// List all skills
|
|
@@ -139,7 +236,7 @@ export function registerRoutes(router, root) {
|
|
|
139
236
|
const evals = loadAndValidateEvals(skillDir);
|
|
140
237
|
const skillMdPath = join(skillDir, "SKILL.md");
|
|
141
238
|
const skillContent = existsSync(skillMdPath) ? readFileSync(skillMdPath, "utf-8") : "";
|
|
142
|
-
const client =
|
|
239
|
+
const client = getClient();
|
|
143
240
|
const systemPrompt = skillContent
|
|
144
241
|
? `You are an AI assistant enhanced with the following skill:\n\n${skillContent}`
|
|
145
242
|
: "You are a helpful AI assistant.";
|
|
@@ -238,7 +335,7 @@ export function registerRoutes(router, root) {
|
|
|
238
335
|
const evals = loadAndValidateEvals(skillDir);
|
|
239
336
|
const skillMdPath = join(skillDir, "SKILL.md");
|
|
240
337
|
const skillContent = existsSync(skillMdPath) ? readFileSync(skillMdPath, "utf-8") : "";
|
|
241
|
-
const client =
|
|
338
|
+
const client = getClient();
|
|
242
339
|
const comparisonResults = [];
|
|
243
340
|
for (const evalCase of evals.evals) {
|
|
244
341
|
if (aborted)
|
|
@@ -379,7 +476,7 @@ export function registerRoutes(router, root) {
|
|
|
379
476
|
// Extract description from frontmatter
|
|
380
477
|
const descMatch = skillContent.match(/^---[\s\S]*?description:\s*"([^"]+)"[\s\S]*?---/);
|
|
381
478
|
const description = descMatch ? descMatch[1] : skillContent.slice(0, 500);
|
|
382
|
-
const client =
|
|
479
|
+
const client = getClient();
|
|
383
480
|
const summary = await testActivation(description, body.prompts, client, (result) => {
|
|
384
481
|
if (!aborted) {
|
|
385
482
|
sendSSE(res, "prompt_result", result);
|