vskill 0.2.26 → 0.2.28

package/dist/commands/eval.d.ts

@@ -0,0 +1,4 @@
+export declare function evalCommand(subcommand: string, target?: string, opts?: {
+    force?: boolean;
+    root?: string;
+}): Promise<void>;

package/dist/commands/eval.js

@@ -0,0 +1,48 @@
+// ---------------------------------------------------------------------------
+// vskill eval -- subcommand router
+// ---------------------------------------------------------------------------
+import { join, resolve } from "node:path";
+import { red, dim } from "../utils/output.js";
+export async function evalCommand(subcommand, target, opts = {}) {
+    const root = opts.root ? resolve(opts.root) : resolve("plugins");
+    switch (subcommand) {
+        case "init": {
+            if (!target) {
+                console.error(red("Usage: vskill eval init <plugin>/<skill>"));
+                process.exit(1);
+            }
+            const skillDir = resolveSkillDir(root, target);
+            const { runEvalInit } = await import("./eval/init.js");
+            return runEvalInit(skillDir, !!opts.force);
+        }
+        case "run": {
+            if (!target) {
+                console.error(red("Usage: vskill eval run <plugin>/<skill>"));
+                process.exit(1);
+            }
+            const skillDir = resolveSkillDir(root, target);
+            const { runEvalRun } = await import("./eval/run.js");
+            return runEvalRun(skillDir);
+        }
+        case "coverage": {
+            const { runEvalCoverage } = await import("./eval/coverage.js");
+            return runEvalCoverage(root);
+        }
+        case "generate-all": {
+            const { runEvalGenerateAll } = await import("./eval/generate-all.js");
+            return runEvalGenerateAll(root, !!opts.force);
+        }
+        default:
+            console.error(red(`Unknown subcommand: "${subcommand}"\n`) +
+                dim("Available: init, run, coverage, generate-all"));
+    }
+}
+function resolveSkillDir(root, target) {
+    const parts = target.split("/");
+    if (parts.length !== 2) {
+        console.error(red(`Invalid target "${target}". Expected format: <plugin>/<skill>`));
+        process.exit(1);
+    }
+    return join(root, parts[0], "skills", parts[1]);
+}
+//# sourceMappingURL=eval.js.map

package/dist/commands/eval.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval.js","sourceRoot":"","sources":["../../src/commands/eval.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,mCAAmC;AACnC,8EAA8E;AAE9E,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,GAAG,EAAE,GAAG,EAAE,MAAM,oBAAoB,CAAC;AAE9C,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,UAAkB,EAClB,MAAe,EACf,OAA2C,EAAE;IAE7C,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;IAEjE,QAAQ,UAAU,EAAE,CAAC;QACnB,KAAK,MAAM,CAAC,CAAC,CAAC;YACZ,IAAI,CAAC,MAAM,EAAE,CAAC;gBACZ,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,0CAA0C,CAAC,CAAC,CAAC;gBAC/D,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAClB,CAAC;YACD,MAAM,QAAQ,GAAG,eAAe,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;YAC/C,MAAM,EAAE,WAAW,EAAE,GAAG,MAAM,MAAM,CAAC,gBAAgB,CAAC,CAAC;YACvD,OAAO,WAAW,CAAC,QAAQ,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC7C,CAAC;QAED,KAAK,KAAK,CAAC,CAAC,CAAC;YACX,IAAI,CAAC,MAAM,EAAE,CAAC;gBACZ,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,yCAAyC,CAAC,CAAC,CAAC;gBAC9D,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAClB,CAAC;YACD,MAAM,QAAQ,GAAG,eAAe,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;YAC/C,MAAM,EAAE,UAAU,EAAE,GAAG,MAAM,MAAM,CAAC,eAAe,CAAC,CAAC;YACrD,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC;QAC9B,CAAC;QAED,KAAK,UAAU,CAAC,CAAC,CAAC;YAChB,MAAM,EAAE,eAAe,EAAE,GAAG,MAAM,MAAM,CAAC,oBAAoB,CAAC,CAAC;YAC/D,OAAO,eAAe,CAAC,IAAI,CAAC,CAAC;QAC/B,CAAC;QAED,KAAK,cAAc,CAAC,CAAC,CAAC;YACpB,MAAM,EAAE,kBAAkB,EAAE,GAAG,MAAM,MAAM,CAAC,wBAAwB,CAAC,CAAC;YACtE,OAAO,kBAAkB,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAChD,CAAC;QAED;YACE,OAAO,CAAC,KAAK,CACX,GAAG,CAAC,wBAAwB,UAAU,KAAK,CAAC;gBAC1C,GAAG,CAAC,8CAA8C,CAAC,CACtD,CAAC;IACN,CAAC;AACH,CAAC;AAED,SAAS,eAAe,CAAC,IAAY,EAAE,MAAc;IACnD,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAChC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACvB,OAAO,CAAC,KAAK,CACX,GAAG,CAAC,mBAAmB,MAAM,sCAAsC,CAAC,CACrE,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IACD,OAAO,IAAI,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;AAClD,CAAC"}

package/dist/eval/__tests__/benchmark.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/eval/__tests__/benchmark.test.js

@@ -0,0 +1,65 @@
+import { describe, it, expect, beforeEach, afterEach } from "vitest";
+import { mkdirSync, writeFileSync, rmSync } from "node:fs";
+import { join } from "node:path";
+import { tmpdir } from "node:os";
+import { writeBenchmark, readBenchmark } from "../benchmark.js";
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+let testDir;
+const SAMPLE_BENCHMARK = {
+    timestamp: "2026-03-01T00:00:00.000Z",
+    model: "claude-sonnet-4-20250514",
+    skill_name: "test-skill",
+    cases: [
+        {
+            eval_id: 1,
+            eval_name: "Basic test",
+            status: "pass",
+            error_message: null,
+            pass_rate: 1.0,
+            assertions: [
+                {
+                    id: "a1",
+                    text: "Check result",
+                    pass: true,
+                    reasoning: "Looks good",
+                },
+            ],
+        },
+    ],
+};
+// ---------------------------------------------------------------------------
+// Tests
+// ---------------------------------------------------------------------------
+describe("benchmark", () => {
+    beforeEach(() => {
+        testDir = join(tmpdir(), `vskill-bench-${Date.now()}`);
+        mkdirSync(join(testDir, "evals"), { recursive: true });
+    });
+    afterEach(() => {
+        rmSync(testDir, { recursive: true, force: true });
+    });
+    it("writes benchmark.json with all required fields", async () => {
+        await writeBenchmark(testDir, SAMPLE_BENCHMARK);
+        const result = await readBenchmark(testDir);
+        expect(result).not.toBeNull();
+        expect(result.timestamp).toBe("2026-03-01T00:00:00.000Z");
+        expect(result.model).toBe("claude-sonnet-4-20250514");
+        expect(result.skill_name).toBe("test-skill");
+        expect(result.cases).toHaveLength(1);
+        expect(result.cases[0].assertions).toHaveLength(1);
+    });
+    it("reads benchmark.json and returns typed result", async () => {
+        writeFileSync(join(testDir, "evals", "benchmark.json"), JSON.stringify(SAMPLE_BENCHMARK));
+        const result = await readBenchmark(testDir);
+        expect(result.skill_name).toBe("test-skill");
+        expect(result.cases[0].pass_rate).toBe(1.0);
+    });
+    it("returns null for missing benchmark.json", async () => {
+        rmSync(join(testDir, "evals"), { recursive: true, force: true });
+        const result = await readBenchmark(testDir);
+        expect(result).toBeNull();
+    });
+});
+//# sourceMappingURL=benchmark.test.js.map

package/dist/eval/__tests__/benchmark.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"benchmark.test.js","sourceRoot":"","sources":["../../../src/eval/__tests__/benchmark.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AACrE,OAAO,EAAE,SAAS,EAAE,aAAa,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AAC3D,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AACjC,OAAO,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAGhE,8EAA8E;AAC9E,UAAU;AACV,8EAA8E;AAE9E,IAAI,OAAe,CAAC;AAEpB,MAAM,gBAAgB,GAAoB;IACxC,SAAS,EAAE,0BAA0B;IACrC,KAAK,EAAE,0BAA0B;IACjC,UAAU,EAAE,YAAY;IACxB,KAAK,EAAE;QACL;YACE,OAAO,EAAE,CAAC;YACV,SAAS,EAAE,YAAY;YACvB,MAAM,EAAE,MAAM;YACd,aAAa,EAAE,IAAI;YACnB,SAAS,EAAE,GAAG;YACd,UAAU,EAAE;gBACV;oBACE,EAAE,EAAE,IAAI;oBACR,IAAI,EAAE,cAAc;oBACpB,IAAI,EAAE,IAAI;oBACV,SAAS,EAAE,YAAY;iBACxB;aACF;SACF;KACF;CACF,CAAC;AAEF,8EAA8E;AAC9E,QAAQ;AACR,8EAA8E;AAE9E,QAAQ,CAAC,WAAW,EAAE,GAAG,EAAE;IACzB,UAAU,CAAC,GAAG,EAAE;QACd,OAAO,GAAG,IAAI,CAAC,MAAM,EAAE,EAAE,gBAAgB,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;QACvD,SAAS,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACzD,CAAC,CAAC,CAAC;IAEH,SAAS,CAAC,GAAG,EAAE;QACb,MAAM,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IACpD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,gDAAgD,EAAE,KAAK,IAAI,EAAE;QAC9D,MAAM,cAAc,CAAC,OAAO,EAAE,gBAAgB,CAAC,CAAC;QAEhD,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,OAAO,CAAC,CAAC;QAC5C,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC;QAC9B,MAAM,CAAC,MAAO,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;QAC3D,MAAM,CAAC,MAAO,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;QACvD,MAAM,CAAC,MAAO,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;QAC9C,MAAM,CAAC,MAAO,CAAC,KAAK,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QACtC,MAAM,CAAC,MAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;IACtD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,+CAA+C,EAAE,KAAK,IAAI,EAAE;QAC7D,aAAa,CACX,IAAI,CAAC,OAAO,EAAE,OAAO,EAAE,gBAAgB,CAAC,EACxC,IAAI,CAAC,SAAS,CAAC,gBAAgB,CAAC,CACjC,CAAC;QAEF,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,OAAO,CAAC,CAAC;QAC5C,MAAM,CAAC,MAAO,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;QAC9C,MAAM,CAAC,MAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC/C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,yCAAyC,EAAE,KAAK,IAAI,EAAE;QACvD,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QAEjE,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,OAAO,CAAC,CAAC;QAC5C,MAAM,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,CAAC;IAC5B,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

package/dist/eval/__tests__/judge.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/eval/__tests__/judge.test.js

@@ -0,0 +1,45 @@
+import { describe, it, expect, vi } from "vitest";
+import { judgeAssertion } from "../judge.js";
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+function mockClient(response) {
+    return { generate: vi.fn().mockResolvedValue(response), model: "test-model" };
+}
+const ASSERTION = {
+    id: "assert-1",
+    text: "Output mentions a file path",
+    type: "boolean",
+};
+// ---------------------------------------------------------------------------
+// Tests
+// ---------------------------------------------------------------------------
+describe("judgeAssertion", () => {
+    it("returns pass result when LLM judge says pass", async () => {
+        const client = mockClient(JSON.stringify({ pass: true, reasoning: "output contains file path" }));
+        const result = await judgeAssertion("The report has been saved to reports/q1.csv", ASSERTION, client);
+        expect(result.pass).toBe(true);
+        expect(result.reasoning).toBe("output contains file path");
+        expect(result.id).toBe("assert-1");
+        expect(result.text).toBe("Output mentions a file path");
+    });
+    it("returns fail result when LLM judge says fail", async () => {
+        const client = mockClient(JSON.stringify({
+            pass: false,
+            reasoning: "no file path found in output",
+        }));
+        const result = await judgeAssertion("Hello world", ASSERTION, client);
+        expect(result.pass).toBe(false);
+        expect(result.reasoning).toBe("no file path found in output");
+    });
+    it("throws on malformed judge response", async () => {
+        const client = mockClient("This is not JSON");
+        await expect(judgeAssertion("some output", ASSERTION, client)).rejects.toThrow(/invalid judge output/i);
+    });
+    it("handles JSON wrapped in code fence", async () => {
+        const client = mockClient('```json\n{"pass": true, "reasoning": "looks good"}\n```');
+        const result = await judgeAssertion("some output", ASSERTION, client);
+        expect(result.pass).toBe(true);
+    });
+});
+//# sourceMappingURL=judge.test.js.map

package/dist/eval/__tests__/judge.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"judge.test.js","sourceRoot":"","sources":["../../../src/eval/__tests__/judge.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAc,MAAM,QAAQ,CAAC;AAG9D,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAE7C,8EAA8E;AAC9E,UAAU;AACV,8EAA8E;AAE9E,SAAS,UAAU,CAAC,QAAgB;IAClC,OAAO,EAAE,QAAQ,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,QAAQ,CAAC,EAAE,KAAK,EAAE,YAAY,EAAE,CAAC;AAChF,CAAC;AAED,MAAM,SAAS,GAAc;IAC3B,EAAE,EAAE,UAAU;IACd,IAAI,EAAE,6BAA6B;IACnC,IAAI,EAAE,SAAS;CAChB,CAAC;AAEF,8EAA8E;AAC9E,QAAQ;AACR,8EAA8E;AAE9E,QAAQ,CAAC,gBAAgB,EAAE,GAAG,EAAE;IAC9B,EAAE,CAAC,8CAA8C,EAAE,KAAK,IAAI,EAAE;QAC5D,MAAM,MAAM,GAAG,UAAU,CACvB,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,SAAS,EAAE,2BAA2B,EAAE,CAAC,CACvE,CAAC;QAEF,MAAM,MAAM,GAAG,MAAM,cAAc,CACjC,6CAA6C,EAC7C,SAAS,EACT,MAAM,CACP,CAAC;QAEF,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC/B,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC;QAC3D,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QACnC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,6BAA6B,CAAC,CAAC;IAC1D,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,8CAA8C,EAAE,KAAK,IAAI,EAAE;QAC5D,MAAM,MAAM,GAAG,UAAU,CACvB,IAAI,CAAC,SAAS,CAAC;YACb,IAAI,EAAE,KAAK;YACX,SAAS,EAAE,8BAA8B;SAC1C,CAAC,CACH,CAAC;QAEF,MAAM,MAAM,GAAG,MAAM,cAAc,CAAC,aAAa,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC;QAEtE,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAChC,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,8BAA8B,CAAC,CAAC;IAChE,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,oCAAoC,EAAE,KAAK,IAAI,EAAE;QAClD,MAAM,MAAM,GAAG,UAAU,CAAC,kBAAkB,CAAC,CAAC;QAE9C,MAAM,MAAM,CACV,cAAc,CAAC,aAAa,EAAE,SAAS,EAAE,MAAM,CAAC,CACjD,CAAC,OAAO,CAAC,OAAO,CAAC,uBAAuB,CAAC,CAAC;IAC7C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,oCAAoC,EAAE,KAAK,IAAI,EAAE;QAClD,MAAM,MAAM,GAAG,UAAU,CACvB,yDAAyD,CAC1D,CAAC;QAEF,MAAM,MAAM,GAAG,MAAM,cAAc,CAAC,aAAa,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC;QACtE,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACjC,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

package/dist/eval/__tests__/llm.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/eval/__tests__/llm.test.js

@@ -0,0 +1,85 @@
+import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
+// ---------------------------------------------------------------------------
+// Mocks
+// ---------------------------------------------------------------------------
+const mockCreate = vi.hoisted(() => vi.fn());
+vi.mock("@anthropic-ai/sdk", () => ({
+    default: class MockAnthropic {
+        messages = { create: mockCreate };
+    },
+}));
+// ---------------------------------------------------------------------------
+// Import module under test AFTER mocks
+// ---------------------------------------------------------------------------
+const { createLlmClient } = await import("../llm.js");
+// ---------------------------------------------------------------------------
+// Tests
+// ---------------------------------------------------------------------------
+describe("createLlmClient", () => {
+    const origEnv = { ...process.env };
+    beforeEach(() => {
+        vi.resetAllMocks();
+        process.env.ANTHROPIC_API_KEY = "test-key";
+        delete process.env.VSKILL_EVAL_MODEL;
+    });
+    afterEach(() => {
+        process.env = { ...origEnv };
+    });
+    it("returns text content on successful generate call", async () => {
+        mockCreate.mockResolvedValue({
+            content: [{ type: "text", text: "Generated response" }],
+        });
+        const client = createLlmClient();
+        const result = await client.generate("system prompt", "user prompt");
+        expect(result).toBe("Generated response");
+        expect(mockCreate).toHaveBeenCalledOnce();
+    });
+    it("uses default model claude-sonnet-4-20250514 when env not set", async () => {
+        mockCreate.mockResolvedValue({
+            content: [{ type: "text", text: "ok" }],
+        });
+        const client = createLlmClient();
+        await client.generate("sys", "usr");
+        expect(mockCreate).toHaveBeenCalledWith(expect.objectContaining({ model: "claude-sonnet-4-20250514" }), expect.anything());
+    });
+    it("uses custom model from VSKILL_EVAL_MODEL env var", async () => {
+        process.env.VSKILL_EVAL_MODEL = "claude-opus-4-20250514";
+        mockCreate.mockResolvedValue({
+            content: [{ type: "text", text: "ok" }],
+        });
+        const client = createLlmClient();
+        await client.generate("sys", "usr");
+        expect(mockCreate).toHaveBeenCalledWith(expect.objectContaining({ model: "claude-opus-4-20250514" }), expect.anything());
+    });
+    it("propagates network error from SDK", async () => {
+        mockCreate.mockRejectedValue(new Error("Connection timeout"));
+        const client = createLlmClient();
+        await expect(client.generate("sys", "usr")).rejects.toThrow("Connection timeout");
+    });
+    it("passes system and user prompts correctly", async () => {
+        mockCreate.mockResolvedValue({
+            content: [{ type: "text", text: "ok" }],
+        });
+        const client = createLlmClient();
+        await client.generate("my system prompt", "my user prompt");
+        expect(mockCreate).toHaveBeenCalledWith(expect.objectContaining({
+            system: "my system prompt",
+            messages: [{ role: "user", content: "my user prompt" }],
+            max_tokens: 4096,
+        }), expect.anything());
+    });
+    it("throws when ANTHROPIC_API_KEY is not set", () => {
+        delete process.env.ANTHROPIC_API_KEY;
+        expect(() => createLlmClient()).toThrow("ANTHROPIC_API_KEY is not set");
+    });
+    it("exposes model name on the client", () => {
+        const client = createLlmClient();
+        expect(client.model).toBe("claude-sonnet-4-20250514");
+    });
+    it("exposes custom model name when VSKILL_EVAL_MODEL is set", () => {
+        process.env.VSKILL_EVAL_MODEL = "claude-opus-4-20250514";
+        const client = createLlmClient();
+        expect(client.model).toBe("claude-opus-4-20250514");
+    });
+});
+//# sourceMappingURL=llm.test.js.map

package/dist/eval/__tests__/llm.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"llm.test.js","sourceRoot":"","sources":["../../../src/eval/__tests__/llm.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AAEzE,8EAA8E;AAC9E,QAAQ;AACR,8EAA8E;AAE9E,MAAM,UAAU,GAAG,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;AAE7C,EAAE,CAAC,IAAI,CAAC,mBAAmB,EAAE,GAAG,EAAE,CAAC,CAAC;IAClC,OAAO,EAAE,MAAM,aAAa;QAC1B,QAAQ,GAAG,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC;KACnC;CACF,CAAC,CAAC,CAAC;AAEJ,8EAA8E;AAC9E,uCAAuC;AACvC,8EAA8E;AAE9E,MAAM,EAAE,eAAe,EAAE,GAAG,MAAM,MAAM,CAAC,WAAW,CAAC,CAAC;AAEtD,8EAA8E;AAC9E,QAAQ;AACR,8EAA8E;AAE9E,QAAQ,CAAC,iBAAiB,EAAE,GAAG,EAAE;IAC/B,MAAM,OAAO,GAAG,EAAE,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC;IAEnC,UAAU,CAAC,GAAG,EAAE;QACd,EAAE,CAAC,aAAa,EAAE,CAAC;QACnB,OAAO,CAAC,GAAG,CAAC,iBAAiB,GAAG,UAAU,CAAC;QAC3C,OAAO,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC;IACvC,CAAC,CAAC,CAAC;IAEH,SAAS,CAAC,GAAG,EAAE;QACb,OAAO,CAAC,GAAG,GAAG,EAAE,GAAG,OAAO,EAAE,CAAC;IAC/B,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,kDAAkD,EAAE,KAAK,IAAI,EAAE;QAChE,UAAU,CAAC,iBAAiB,CAAC;YAC3B,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,oBAAoB,EAAE,CAAC;SACxD,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;QACjC,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC,eAAe,EAAE,aAAa,CAAC,CAAC;QAErE,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;QAC1C,MAAM,CAAC,UAAU,CAAC,CAAC,oBAAoB,EAAE,CAAC;IAC5C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,8DAA8D,EAAE,KAAK,IAAI,EAAE;QAC5E,UAAU,CAAC,iBAAiB,CAAC;YAC3B,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;SACxC,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;QACjC,MAAM,MAAM,CAAC,QAAQ,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;QAEpC,MAAM,CAAC,UAAU,CAAC,CAAC,oBAAoB,CACrC,MAAM,CAAC,gBAAgB,CAAC,EAAE,KAAK,EAAE,0BAA0B,EAAE,CAAC,EAC9D,MAAM,CAAC,QAAQ,EAAE,CAClB,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,kDAAkD,EAAE,KAAK,IAAI,EAAE;QAChE,OAAO,CAAC,GAAG,CAAC,iBAAiB,GAAG,wBAAwB,CAAC;QACzD,UAAU,CAAC,iBAAiB,CAAC;YAC3B,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;SACxC,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;QACjC,MAAM,MAAM,CAAC,QAAQ,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;QAEpC,MAAM,CAAC,UAAU,CAAC,CAAC,oBAAoB,CACrC,MAAM,CAAC,gBAAgB,CAAC,EAAE,KAAK,EAAE,wBAAwB,EAAE,CAAC,EAC5D,MAAM,CAAC,QAAQ,EAAE,CAClB,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,mCAAmC,EAAE,KAAK,IAAI,EAAE;QACjD,UAAU,CAAC,iBAAiB,CAAC,IAAI,KAAK,CAAC,oBAAoB,CAAC,CAAC,CAAC;QAE9D,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;QACjC,MAAM,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CACzD,oBAAoB,CACrB,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,0CAA0C,EAAE,KAAK,IAAI,EAAE;QACxD,UAAU,CAAC,iBAAiB,CAAC;YAC3B,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;SACxC,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;QACjC,MAAM,MAAM,CAAC,QAAQ,CAAC,kBAAkB,EAAE,gBAAgB,CAAC,CAAC;QAE5D,MAAM,CAAC,UAAU,CAAC,CAAC,oBAAoB,CACrC,MAAM,CAAC,gBAAgB,CAAC;YACtB,MAAM,EAAE,kBAAkB;YAC1B,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,gBAAgB,EAAE,CAAC;YACvD,UAAU,EAAE,IAAI;SACjB,CAAC,EACF,MAAM,CAAC,QAAQ,EAAE,CAClB,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,0CAA0C,EAAE,GAAG,EAAE;QAClD,OAAO,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC;QACrC,MAAM,CAAC,GAAG,EAAE,CAAC,eAAe,EAAE,CAAC,CAAC,OAAO,CAAC,8BAA8B,CAAC,CAAC;IAC1E,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,kCAAkC,EAAE,GAAG,EAAE;QAC1C,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;QACjC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;IACxD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,yDAAyD,EAAE,GAAG,EAAE;QACjE,OAAO,CAAC,GAAG,CAAC,iBAAiB,GAAG,wBAAwB,CAAC;QACzD,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;QACjC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAC;IACtD,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

package/dist/eval/__tests__/prompt-builder.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/eval/__tests__/prompt-builder.test.js

@@ -0,0 +1,72 @@
+import { describe, it, expect } from "vitest";
+import { buildEvalInitPrompt, parseGeneratedEvals, } from "../prompt-builder.js";
+// ---------------------------------------------------------------------------
+// buildEvalInitPrompt
+// ---------------------------------------------------------------------------
+describe("buildEvalInitPrompt", () => {
+    const skillContent = "# My Skill\nThis skill does amazing things.";
+    it("includes skill content in the prompt", () => {
+        const prompt = buildEvalInitPrompt(skillContent);
+        expect(prompt).toContain(skillContent);
+    });
+    it("includes schema reference fields", () => {
+        const prompt = buildEvalInitPrompt(skillContent);
+        expect(prompt).toContain("skill_name");
+        expect(prompt).toContain("assertions");
+        expect(prompt).toContain("expected_output");
+    });
+    it("includes social-media-posting example", () => {
+        const prompt = buildEvalInitPrompt(skillContent);
+        expect(prompt).toContain("social-media-posting");
+    });
+    it("includes best practices section", () => {
+        const prompt = buildEvalInitPrompt(skillContent);
+        expect(prompt).toContain("Best Practices");
+        expect(prompt).toContain("objectively verifiable");
+    });
+});
+// ---------------------------------------------------------------------------
+// parseGeneratedEvals
+// ---------------------------------------------------------------------------
+describe("parseGeneratedEvals", () => {
+    it("extracts JSON from markdown code fence", () => {
+        const raw = `Here is the evals.json:
+
+\`\`\`json
+{
+  "skill_name": "test-skill",
+  "evals": [
+    {
+      "id": 1,
+      "name": "Basic test",
+      "prompt": "Test prompt",
+      "expected_output": "Expected output",
+      "files": [],
+      "assertions": [
+        { "id": "a1", "text": "Check result", "type": "boolean" }
+      ]
+    }
+  ]
+}
+\`\`\`
+
+That's the evals file.`;
+        const result = parseGeneratedEvals(raw);
+        expect(result.skill_name).toBe("test-skill");
+        expect(result.evals).toHaveLength(1);
+        expect(result.evals[0].assertions).toHaveLength(1);
+    });
+    it("throws when no code block is found", () => {
+        const raw = "Just some text without any JSON code block.";
+        expect(() => parseGeneratedEvals(raw)).toThrow(/code block/i);
+    });
+    it("throws when JSON inside fence is invalid", () => {
+        const raw = "```json\n{ invalid json }\n```";
+        expect(() => parseGeneratedEvals(raw)).toThrow();
+    });
+    it("validates extracted JSON against schema", () => {
+        const raw = '```json\n{ "skill_name": "test" }\n```';
+        expect(() => parseGeneratedEvals(raw)).toThrow(); // missing evals array
+    });
+});
+//# sourceMappingURL=prompt-builder.test.js.map

package/dist/eval/__tests__/prompt-builder.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"prompt-builder.test.js","sourceRoot":"","sources":["../../../src/eval/__tests__/prompt-builder.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAC9C,OAAO,EACL,mBAAmB,EACnB,mBAAmB,GACpB,MAAM,sBAAsB,CAAC;AAE9B,8EAA8E;AAC9E,sBAAsB;AACtB,8EAA8E;AAE9E,QAAQ,CAAC,qBAAqB,EAAE,GAAG,EAAE;IACnC,MAAM,YAAY,GAAG,6CAA6C,CAAC;IAEnE,EAAE,CAAC,sCAAsC,EAAE,GAAG,EAAE;QAC9C,MAAM,MAAM,GAAG,mBAAmB,CAAC,YAAY,CAAC,CAAC;QACjD,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,YAAY,CAAC,CAAC;IACzC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,kCAAkC,EAAE,GAAG,EAAE;QAC1C,MAAM,MAAM,GAAG,mBAAmB,CAAC,YAAY,CAAC,CAAC;QACjD,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,YAAY,CAAC,CAAC;QACvC,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,YAAY,CAAC,CAAC;QACvC,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,iBAAiB,CAAC,CAAC;IAC9C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,uCAAuC,EAAE,GAAG,EAAE;QAC/C,MAAM,MAAM,GAAG,mBAAmB,CAAC,YAAY,CAAC,CAAC;QACjD,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,sBAAsB,CAAC,CAAC;IACnD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,iCAAiC,EAAE,GAAG,EAAE;QACzC,MAAM,MAAM,GAAG,mBAAmB,CAAC,YAAY,CAAC,CAAC;QACjD,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,gBAAgB,CAAC,CAAC;QAC3C,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,wBAAwB,CAAC,CAAC;IACrD,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,8EAA8E;AAC9E,sBAAsB;AACtB,8EAA8E;AAE9E,QAAQ,CAAC,qBAAqB,EAAE,GAAG,EAAE;IACnC,EAAE,CAAC,wCAAwC,EAAE,GAAG,EAAE;QAChD,MAAM,GAAG,GAAG;;;;;;;;;;;;;;;;;;;;uBAoBO,CAAC;QAEpB,MAAM,MAAM,GAAG,mBAAmB,CAAC,GAAG,CAAC,CAAC;QACxC,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;QAC7C,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QACrC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;IACrD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,oCAAoC,EAAE,GAAG,EAAE;QAC5C,MAAM,GAAG,GAAG,6CAA6C,CAAC;QAC1D,MAAM,CAAC,GAAG,EAAE,CAAC,mBAAmB,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC;IAChE,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,0CAA0C,EAAE,GAAG,EAAE;QAClD,MAAM,GAAG,GAAG,gCAAgC,CAAC;QAC7C,MAAM,CAAC,GAAG,EAAE,CAAC,mBAAmB,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC;IACnD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,yCAAyC,EAAE,GAAG,EAAE;QACjD,MAAM,GAAG,GAAG,wCAAwC,CAAC;QACrD,MAAM,CAAC,GAAG,EAAE,CAAC,mBAAmB,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,sBAAsB;IAC1E,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

package/dist/eval/__tests__/schema.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/eval/__tests__/schema.test.js

@@ -0,0 +1,209 @@
+import { describe, it, expect, vi, beforeEach } from "vitest";
+// ---------------------------------------------------------------------------
+// Mocks
+// ---------------------------------------------------------------------------
+const mocks = vi.hoisted(() => ({
+    readFileSync: vi.fn(),
+    existsSync: vi.fn(),
+}));
+vi.mock("node:fs", () => ({
+    readFileSync: mocks.readFileSync,
+    existsSync: mocks.existsSync,
+}));
+// ---------------------------------------------------------------------------
+// Import module under test AFTER mocks
+// ---------------------------------------------------------------------------
+const { loadAndValidateEvals, EvalValidationError } = await import("../schema.js");
+// ---------------------------------------------------------------------------
+// Fixtures
+// ---------------------------------------------------------------------------
+const VALID_EVALS = {
+    skill_name: "social-media-posting",
+    evals: [
+        {
+            id: 1,
+            name: "Basic post generation",
+            prompt: "Create a social media post about AI",
+            expected_output: "A well-crafted social media post",
+            files: [],
+            assertions: [
+                { id: "assert-1", text: "Output mentions AI", type: "boolean" },
+                { id: "assert-2", text: "Output is under 280 chars", type: "boolean" },
+            ],
+        },
+    ],
+};
+// ---------------------------------------------------------------------------
+// Tests
+// ---------------------------------------------------------------------------
+describe("loadAndValidateEvals", () => {
+    beforeEach(() => {
+        vi.resetAllMocks();
+    });
+    it("accepts valid evals.json and returns typed EvalsFile", () => {
+        mocks.existsSync.mockReturnValue(true);
+        mocks.readFileSync.mockReturnValue(JSON.stringify(VALID_EVALS));
+        const result = loadAndValidateEvals("/skills/my-skill");
+        expect(result.skill_name).toBe("social-media-posting");
+        expect(result.evals).toHaveLength(1);
+        expect(result.evals[0].assertions).toHaveLength(2);
+    });
+    it("throws for missing skill_name", () => {
+        const invalid = { ...VALID_EVALS, skill_name: undefined };
+        mocks.existsSync.mockReturnValue(true);
+        mocks.readFileSync.mockReturnValue(JSON.stringify(invalid));
+        expect(() => loadAndValidateEvals("/skills/my-skill")).toThrow(EvalValidationError);
+        try {
+            loadAndValidateEvals("/skills/my-skill");
+        }
+        catch (e) {
+            expect(e.errors.some((err) => err.path === "skill_name")).toBe(true);
+        }
+    });
+    it("throws for missing evals array", () => {
+        const invalid = { skill_name: "test" };
+        mocks.existsSync.mockReturnValue(true);
+        mocks.readFileSync.mockReturnValue(JSON.stringify(invalid));
+        expect(() => loadAndValidateEvals("/skills/my-skill")).toThrow(EvalValidationError);
+        try {
+            loadAndValidateEvals("/skills/my-skill");
+        }
+        catch (e) {
+            expect(e.errors.some((err) => err.path === "evals")).toBe(true);
+        }
+    });
+    it("throws for missing prompt in eval case", () => {
+        const invalid = {
+            skill_name: "test",
+            evals: [
+                {
+                    id: 1,
+                    name: "Test",
+                    expected_output: "output",
+                    assertions: [{ id: "a1", text: "ok", type: "boolean" }],
+                },
+            ],
+        };
+        mocks.existsSync.mockReturnValue(true);
+        mocks.readFileSync.mockReturnValue(JSON.stringify(invalid));
+        expect(() => loadAndValidateEvals("/skills/my-skill")).toThrow(EvalValidationError);
+        try {
+            loadAndValidateEvals("/skills/my-skill");
+        }
+        catch (e) {
+            expect(e.errors.some((err) => err.path === "evals[0].prompt")).toBe(true);
+        }
+    });
+    it("throws for missing assertions", () => {
+        const invalid = {
+            skill_name: "test",
+            evals: [
+                {
+                    id: 1,
+                    name: "Test",
+                    prompt: "do something",
+                    expected_output: "output",
+                },
+            ],
+        };
+        mocks.existsSync.mockReturnValue(true);
+        mocks.readFileSync.mockReturnValue(JSON.stringify(invalid));
+        expect(() => loadAndValidateEvals("/skills/my-skill")).toThrow(EvalValidationError);
+        try {
+            loadAndValidateEvals("/skills/my-skill");
+        }
+        catch (e) {
+            expect(e.errors.some((err) => err.path === "evals[0].assertions")).toBe(true);
+        }
+    });
+    it("throws for duplicate assertion IDs within a case", () => {
+        const invalid = {
+            skill_name: "test",
+            evals: [
+                {
+                    id: 1,
+                    name: "Test",
+                    prompt: "do something",
+                    expected_output: "output",
+                    assertions: [
+                        { id: "assert-1", text: "first", type: "boolean" },
+                        { id: "assert-1", text: "duplicate", type: "boolean" },
+                    ],
+                },
+            ],
+        };
+        mocks.existsSync.mockReturnValue(true);
+        mocks.readFileSync.mockReturnValue(JSON.stringify(invalid));
+        expect(() => loadAndValidateEvals("/skills/my-skill")).toThrow(EvalValidationError);
+        try {
+            loadAndValidateEvals("/skills/my-skill");
+        }
+        catch (e) {
+            expect(e.errors.some((err) => err.message.includes("assert-1"))).toBe(true);
+        }
+    });
+    it("throws for invalid JSON with file path info", () => {
+        mocks.existsSync.mockReturnValue(true);
+        mocks.readFileSync.mockReturnValue("{ broken json");
+        expect(() => loadAndValidateEvals("/skills/my-skill")).toThrow(EvalValidationError);
+        try {
+            loadAndValidateEvals("/skills/my-skill");
+        }
+        catch (e) {
+            expect(e.errors[0].path).toContain("evals.json");
+            expect(e.errors[0].message).toMatch(/parse|JSON/i);
+        }
+    });
+    it("throws for empty assertions array", () => {
+        const invalid = {
+            skill_name: "test",
+            evals: [
+                {
+                    id: 1,
+                    name: "Test",
+                    prompt: "do something",
+                    expected_output: "output",
+                    assertions: [],
+                },
+            ],
+        };
+        mocks.existsSync.mockReturnValue(true);
+        mocks.readFileSync.mockReturnValue(JSON.stringify(invalid));
+        expect(() => loadAndValidateEvals("/skills/my-skill")).toThrow(EvalValidationError);
+        try {
+            loadAndValidateEvals("/skills/my-skill");
+        }
+        catch (e) {
+            expect(e.errors.some((err) => err.message.includes("at least 1"))).toBe(true);
+        }
+    });
+    it("throws when evals.json does not exist", () => {
+        mocks.existsSync.mockReturnValue(false);
+        expect(() => loadAndValidateEvals("/skills/my-skill")).toThrow(EvalValidationError);
+        try {
+            loadAndValidateEvals("/skills/my-skill");
+        }
+        catch (e) {
+            expect(e.errors[0].message).toMatch(/not found|No evals\.json/i);
+        }
+    });
+    it("accepts evals with optional files field defaulting to empty array", () => {
+        const valid = {
+            skill_name: "test",
+            evals: [
+                {
+                    id: 1,
+                    name: "Test",
+                    prompt: "do something",
+                    expected_output: "output",
+                    assertions: [{ id: "a1", text: "check", type: "boolean" }],
+                },
+            ],
+        };
+        mocks.existsSync.mockReturnValue(true);
+        mocks.readFileSync.mockReturnValue(JSON.stringify(valid));
+        const result = loadAndValidateEvals("/skills/my-skill");
+        expect(result.evals[0].files).toEqual([]);
+    });
+});
+//# sourceMappingURL=schema.test.js.map

package/dist/eval/__tests__/schema.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"schema.test.js","sourceRoot":"","sources":["../../../src/eval/__tests__/schema.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AAE9D,8EAA8E;AAC9E,QAAQ;AACR,8EAA8E;AAE9E,MAAM,KAAK,GAAG,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;IAC9B,YAAY,EAAE,EAAE,CAAC,EAAE,EAAE;IACrB,UAAU,EAAE,EAAE,CAAC,EAAE,EAAE;CACpB,CAAC,CAAC,CAAC;AAEJ,EAAE,CAAC,IAAI,CAAC,SAAS,EAAE,GAAG,EAAE,CAAC,CAAC;IACxB,YAAY,EAAE,KAAK,CAAC,YAAY;IAChC,UAAU,EAAE,KAAK,CAAC,UAAU;CAC7B,CAAC,CAAC,CAAC;AAEJ,8EAA8E;AAC9E,uCAAuC;AACvC,8EAA8E;AAE9E,MAAM,EAAE,oBAAoB,EAAE,mBAAmB,EAAE,GAAG,MAAM,MAAM,CAChE,cAAc,CACf,CAAC;AAEF,8EAA8E;AAC9E,WAAW;AACX,8EAA8E;AAE9E,MAAM,WAAW,GAAG;IAClB,UAAU,EAAE,sBAAsB;IAClC,KAAK,EAAE;QACL;YACE,EAAE,EAAE,CAAC;YACL,IAAI,EAAE,uBAAuB;YAC7B,MAAM,EAAE,qCAAqC;YAC7C,eAAe,EAAE,kCAAkC;YACnD,KAAK,EAAE,EAAE;YACT,UAAU,EAAE;gBACV,EAAE,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,oBAAoB,EAAE,IAAI,EAAE,SAAS,EAAE;gBAC/D,EAAE,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,2BAA2B,EAAE,IAAI,EAAE,SAAS,EAAE;aACvE;SACF;KACF;CACF,CAAC;AAEF,8EAA8E;AAC9E,QAAQ;AACR,8EAA8E;AAE9E,QAAQ,CAAC,sBAAsB,EAAE,GAAG,EAAE;IACpC,UAAU,CAAC,GAAG,EAAE;QACd,EAAE,CAAC,aAAa,EAAE,CAAC;IACrB,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,sDAAsD,EAAE,GAAG,EAAE;QAC9D,KAAK,CAAC,UAAU,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC;QACvC,KAAK,CAAC,YAAY,CAAC,eAAe,CAAC,IAAI,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC,CAAC;QAEhE,MAAM,MAAM,GAAG,oBAAoB,CAAC,kBAAkB,CAAC,CAAC;QAExD,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,sBAAsB,CAAC,CAAC;QACvD,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QACrC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;IACrD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,+BAA+B,EAAE,GAAG,EAAE;QACvC,MAAM,OAAO,GAAG,EAAE,GAAG,WAAW,EAAE,UAAU,EAAE,SAAS,EAAE,CAAC;QAC1D,KAAK,CAAC,UAAU,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC;QACvC,KAAK,CAAC,YAAY,CAAC,eAAe,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC;QAE5D,MAAM,CAAC,GAAG,EAAE,CAAC,oBAAoB,CAAC,kBAAkB,CAAC,CAAC,CAAC,OAAO,CAC5D,mBAAmB,CACpB,CAAC;QACF,IAAI,CAAC;YACH,oBAAoB,CAAC,kBAAkB,CAAC,CAAC;QAC3C,CAAC;QAAC,OAAO,CAAM,EAAE,CAAC;YAChB,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,GAAQ,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,KAAK,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC5E,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,gCAAgC,EAAE,GAAG,EAAE;QACxC,MAAM,OAAO,GAAG,EAAE,UAAU,EAAE,MAAM,EAAE,CAAC;QACvC,KAAK,CAAC,UAAU,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC;QACvC,KAAK,CAAC,YAAY,CAAC,eAAe,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC;QAE5D,MAAM,CAAC,GAAG,EAAE,CAAC,oBAAoB,CAAC,kBAAkB,CAAC,CAAC,CAAC,OAAO,CAC5D,mBAAmB,CACpB,CAAC;QACF,IAAI,CAAC;YACH,oBAAoB,CAAC,kBAAkB,CAAC,CAAC;QAC3C,CAAC;QAAC,OAAO,CAAM,EAAE,CAAC;YAChB,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,GAAQ,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,KAAK,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACvE,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,wCAAwC,EAAE,GAAG,EAAE;QAChD,MAAM,OAAO,GAAG;YACd,UAAU,EAAE,MAAM;YAClB,KAAK,EAAE;gBACL;oBACE,EAAE,EAAE,CAAC;oBACL,IAAI,EAAE,MAAM;oBACZ,eAAe,EAAE,QAAQ;oBACzB,UAAU,EAAE,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC;iBACxD;aACF;SACF,CAAC;QACF,KAAK,CAAC,UAAU,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC;QACvC,KAAK,CAAC,YAAY,CAAC,eAAe,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC;QAE5D,MAAM,CAAC,GAAG,EAAE,CAAC,oBAAoB,CAAC,kBAAkB,CAAC,CAAC,CAAC,OAAO,CAC5D,mBAAmB,CACpB,CAAC;QACF,IAAI,CAAC;YACH,oBAAoB,CAAC,kBAAkB,CAAC,CAAC;QAC3C,CAAC;QAAC,OAAO,CAAM,EAAE,CAAC;YAChB,MAAM,CACJ,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,GAAQ,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,KAAK,iBAAiB,CAAC,CAC5D,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACf,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,+BAA+B,EAAE,GAAG,EAAE;QACvC,MAAM,OAAO,GAAG;YACd,UAAU,EAAE,MAAM;YAClB,KAAK,EAAE;gBACL;oBACE,EAAE,EAAE,CAAC;oBACL,IAAI,EAAE,MAAM;oBACZ,MAAM,EAAE,cAAc;oBACtB,eAAe,EAAE,QAAQ;iBAC1B;aACF;SACF,CAAC;QACF,KAAK,CAAC,UAAU,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC;QACvC,KAAK,CAAC,YAAY,CAAC,eAAe,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC;QAE5D,MAAM,CAAC,GAAG,EAAE,CAAC,oBAAoB,CAAC,kBAAkB,CAAC,CAAC,CAAC,OAAO,CAC5D,mBAAmB,CACpB,CAAC;QACF,IAAI,CAAC;YACH,oBAAoB,CAAC,kBAAkB,CAAC,CAAC;QAC3C,CAAC;QAAC,OAAO,CAAM,EAAE,CAAC;YAChB,MAAM,CACJ,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,GAAQ,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,KAAK,qBAAqB,CAAC,CAChE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACf,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,kDAAkD,EAAE,GAAG,EAAE;QAC1D,MAAM,OAAO,GAAG;YACd,UAAU,EAAE,MAAM;YAClB,KAAK,EAAE;gBACL;oBACE,EAAE,EAAE,CAAC;oBACL,IAAI,EAAE,MAAM;oBACZ,MAAM,EAAE,cAAc;oBACtB,eAAe,EAAE,QAAQ;oBACzB,UAAU,EAAE;wBACV,EAAE,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE;wBAClD,EAAE,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,SAAS,EAAE;qBACvD;iBACF;aACF;SACF,CAAC;QACF,KAAK,CAAC,UAAU,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC;QACvC,KAAK,CAAC,YAAY,CAAC,eAAe,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC;QAE5D,MAAM,CAAC,GAAG,EAAE,CAAC,oBAAoB,CAAC,kBAAkB,CAAC,CAAC,CAAC,OAAO,CAC5D,mBAAmB,CACpB,CAAC;QACF,IAAI,CAAC;YACH,oBAAoB,CAAC,kBAAkB,CAAC,CAAC;QAC3C,CAAC;QAAC,OAAO,CAAM,EAAE,CAAC;YAChB,MAAM,CACJ,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,GAAQ,EAAE,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,CAC9D,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACf,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,6CAA6C,EAAE,GAAG,EAAE;QACrD,KAAK,CAAC,UAAU,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC;QACvC,KAAK,CAAC,YAAY,CAAC,eAAe,CAAC,eAAe,CAAC,CAAC;QAEpD,MAAM,CAAC,GAAG,EAAE,CAAC,oBAAoB,CAAC,kBAAkB,CAAC,CAAC,CAAC,OAAO,CAC5D,mBAAmB,CACpB,CAAC;QACF,IAAI,CAAC;YACH,oBAAoB,CAAC,kBAAkB,CAAC,CAAC;QAC3C,CAAC;QAAC,OAAO,CAAM,EAAE,CAAC;YAChB,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,SAAS,CAAC,YAAY,CAAC,CAAC;YACjD,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC;QACrD,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,mCAAmC,EAAE,GAAG,EAAE;QAC3C,MAAM,OAAO,GAAG;YACd,UAAU,EAAE,MAAM;YAClB,KAAK,EAAE;gBACL;oBACE,EAAE,EAAE,CAAC;oBACL,IAAI,EAAE,MAAM;oBACZ,MAAM,EAAE,cAAc;oBACtB,eAAe,EAAE,QAAQ;oBACzB,UAAU,EAAE,EAAE;iBACf;aACF;SACF,CAAC;QACF,KAAK,CAAC,UAAU,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC;QACvC,KAAK,CAAC,YAAY,CAAC,eAAe,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC;QAE5D,MAAM,CAAC,GAAG,EAAE,CAAC,oBAAoB,CAAC,kBAAkB,CAAC,CAAC,CAAC,OAAO,CAC5D,mBAAmB,CACpB,CAAC;QACF,IAAI,CAAC;YACH,oBAAoB,CAAC,kBAAkB,CAAC,CAAC;QAC3C,CAAC;QAAC,OAAO,CAAM,EAAE,CAAC;YAChB,MAAM,CACJ,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,GAAQ,EAAE,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC,CAChE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACf,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,uCAAuC,EAAE,GAAG,EAAE;QAC/C,KAAK,CAAC,UAAU,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;QAExC,MAAM,CAAC,GAAG,EAAE,CAAC,oBAAoB,CAAC,kBAAkB,CAAC,CAAC,CAAC,OAAO,CAC5D,mBAAmB,CACpB,CAAC;QACF,IAAI,CAAC;YACH,oBAAoB,CAAC,kBAAkB,CAAC,CAAC;QAC3C,CAAC;QAAC,OAAO,CAAM,EAAE,CAAC;YAChB,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,OAAO,CAAC,2BAA2B,CAAC,CAAC;QACnE,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,mEAAmE,EAAE,GAAG,EAAE;QAC3E,MAAM,KAAK,GAAG;YACZ,UAAU,EAAE,MAAM;YAClB,KAAK,EAAE;gBACL;oBACE,EAAE,EAAE,CAAC;oBACL,IAAI,EAAE,MAAM;oBACZ,MAAM,EAAE,cAAc;oBACtB,eAAe,EAAE,QAAQ;oBACzB,UAAU,EAAE,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC;iBAC3D;aACF;SACF,CAAC;QACF,KAAK,CAAC,UAAU,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC;QACvC,KAAK,CAAC,YAAY,CAAC,eAAe,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC;QAE1D,MAAM,MAAM,GAAG,oBAAoB,CAAC,kBAAkB,CAAC,CAAC;QACxD,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IAC5C,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

package/dist/eval/__tests__/skill-scanner.test.d.ts

@@ -0,0 +1 @@
+export {};