vskill 0.2.55 → 0.2.56

package/dist/eval/__tests__/comparator.test.js

@@ -0,0 +1,136 @@
+import { describe, it, expect, vi } from "vitest";
+import { generateComparisonOutputs, scoreComparison, runComparison, } from "../comparator.js";
+function mockClient(responses) {
+    let callIndex = 0;
+    return {
+        model: "test-model",
+        generate: vi.fn(async () => responses[callIndex++] ?? ""),
+    };
+}
+describe("generateComparisonOutputs", () => {
+    it("generates skill and baseline outputs sequentially", async () => {
+        const client = mockClient(["skill response", "baseline response"]);
+        const result = await generateComparisonOutputs("test prompt", "# Skill Content", client);
+        expect(result.skillOutput).toBe("skill response");
+        expect(result.baselineOutput).toBe("baseline response");
+        expect(result.skillDurationMs).toBeGreaterThanOrEqual(0);
+        expect(result.baselineDurationMs).toBeGreaterThanOrEqual(0);
+        expect(client.generate).toHaveBeenCalledTimes(2);
+        // First call should include skill content
+        const firstCall = client.generate.mock.calls[0];
+        expect(firstCall[0]).toContain("Skill Content");
+        // Second call should be generic
+        const secondCall = client.generate.mock.calls[1];
+        expect(secondCall[0]).toContain("helpful AI assistant");
+    });
+});
+describe("scoreComparison", () => {
+    it("parses JSON scores from LLM response", async () => {
+        const client = mockClient([
+            JSON.stringify({
+                content_score_a: 4,
+                structure_score_a: 3,
+                content_score_b: 5,
+                structure_score_b: 4,
+                winner: "second",
+                reasoning: "B is better",
+            }),
+        ]);
+        const result = await scoreComparison("output A", "output B", "prompt", client);
+        expect(result.contentScoreA).toBe(4);
+        expect(result.structureScoreA).toBe(3);
+        expect(result.contentScoreB).toBe(5);
+        expect(result.structureScoreB).toBe(4);
+        expect(result.winner).toBe("second");
+    });
+    it("parses JSON from code fence", async () => {
+        const client = mockClient([
+            '```json\n{"content_score_a": 3, "structure_score_a": 3, "content_score_b": 3, "structure_score_b": 3, "winner": "tie"}\n```',
+        ]);
+        const result = await scoreComparison("A", "B", "p", client);
+        expect(result.winner).toBe("tie");
+        expect(result.contentScoreA).toBe(3);
+    });
+    it("clamps scores to 1-5 range", async () => {
+        const client = mockClient([
+            JSON.stringify({
+                content_score_a: 0,
+                structure_score_a: 10,
+                content_score_b: -1,
+                structure_score_b: 6,
+                winner: "first",
+            }),
+        ]);
+        const result = await scoreComparison("A", "B", "p", client);
+        expect(result.contentScoreA).toBe(1);
+        expect(result.structureScoreA).toBe(5);
+        expect(result.contentScoreB).toBe(1);
+        expect(result.structureScoreB).toBe(5);
+    });
+    it("defaults invalid winner to tie", async () => {
+        const client = mockClient([
+            JSON.stringify({
+                content_score_a: 3,
+                structure_score_a: 3,
+                content_score_b: 3,
+                structure_score_b: 3,
+                winner: "invalid",
+            }),
+        ]);
+        const result = await scoreComparison("A", "B", "p", client);
+        expect(result.winner).toBe("tie");
+    });
+});
+describe("runComparison", () => {
+    it("maps scores back to skill/baseline correctly", async () => {
+        // Mock: first two calls = skill + baseline outputs, third = scoring
+        const client = mockClient([
+            "skill output here",
+            "baseline output here",
+            JSON.stringify({
+                content_score_a: 4,
+                structure_score_a: 5,
+                content_score_b: 2,
+                structure_score_b: 3,
+                winner: "first",
+                reasoning: "A is better",
+            }),
+        ]);
+        // Fix randomness for deterministic test
+        vi.spyOn(Math, "random").mockReturnValue(0.3); // < 0.5 → skill is A
+        const result = await runComparison("test prompt", "skill content", client);
+        expect(result.prompt).toBe("test prompt");
+        expect(result.skillOutput).toBe("skill output here");
+        expect(result.baselineOutput).toBe("baseline output here");
+        // skill is A, so scores map directly
+        expect(result.skillContentScore).toBe(4);
+        expect(result.skillStructureScore).toBe(5);
+        expect(result.baselineContentScore).toBe(2);
+        expect(result.baselineStructureScore).toBe(3);
+        expect(result.winner).toBe("skill");
+        vi.restoreAllMocks();
+    });
+    it("maps scores correctly when baseline is A", async () => {
+        const client = mockClient([
+            "skill out",
+            "baseline out",
+            JSON.stringify({
+                content_score_a: 2,
+                structure_score_a: 2,
+                content_score_b: 4,
+                structure_score_b: 4,
+                winner: "second",
+            }),
+        ]);
+        // > 0.5 → skill is B
+        vi.spyOn(Math, "random").mockReturnValue(0.7);
+        const result = await runComparison("p", "s", client);
+        // skill is B → scores.contentScoreB is skill
+        expect(result.skillContentScore).toBe(4);
+        expect(result.baselineContentScore).toBe(2);
+        // winner "second" = B = skill
+        expect(result.winner).toBe("skill");
+        vi.restoreAllMocks();
+    });
+});
+//# sourceMappingURL=comparator.test.js.map

package/dist/eval/__tests__/comparator.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"comparator.test.js","sourceRoot":"","sources":["../../../src/eval/__tests__/comparator.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAC;AAClD,OAAO,EACL,yBAAyB,EACzB,eAAe,EACf,aAAa,GACd,MAAM,kBAAkB,CAAC;AAG1B,SAAS,UAAU,CAAC,SAAmB;IACrC,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,OAAO;QACL,KAAK,EAAE,YAAY;QACnB,QAAQ,EAAE,EAAE,CAAC,EAAE,CAAC,KAAK,IAAI,EAAE,CAAC,SAAS,CAAC,SAAS,EAAE,CAAC,IAAI,EAAE,CAAC;KAC1D,CAAC;AACJ,CAAC;AAED,QAAQ,CAAC,2BAA2B,EAAE,GAAG,EAAE;IACzC,EAAE,CAAC,mDAAmD,EAAE,KAAK,IAAI,EAAE;QACjE,MAAM,MAAM,GAAG,UAAU,CAAC,CAAC,gBAAgB,EAAE,mBAAmB,CAAC,CAAC,CAAC;QACnE,MAAM,MAAM,GAAG,MAAM,yBAAyB,CAAC,aAAa,EAAE,iBAAiB,EAAE,MAAM,CAAC,CAAC;QAEzF,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;QAClD,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;QACxD,MAAM,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,CAAC;QACzD,MAAM,CAAC,MAAM,CAAC,kBAAkB,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,CAAC;QAC5D,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC;QAEjD,0CAA0C;QAC1C,MAAM,SAAS,GAAI,MAAM,CAAC,QAAgB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QACzD,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,eAAe,CAAC,CAAC;QAEhD,gCAAgC;QAChC,MAAM,UAAU,GAAI,MAAM,CAAC,QAAgB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QAC1D,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,sBAAsB,CAAC,CAAC;IAC1D,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,iBAAiB,EAAE,GAAG,EAAE;IAC/B,EAAE,CAAC,sCAAsC,EAAE,KAAK,IAAI,EAAE;QACpD,MAAM,MAAM,GAAG,UAAU,CAAC;YACxB,IAAI,CAAC,SAAS,CAAC;gBACb,eAAe,EAAE,CAAC;gBAClB,iBAAiB,EAAE,CAAC;gBACpB,eAAe,EAAE,CAAC;gBAClB,iBAAiB,EAAE,CAAC;gBACpB,MAAM,EAAE,QAAQ;gBAChB,SAAS,EAAE,aAAa;aACzB,CAAC;SACH,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,MAAM,eAAe,CAAC,UAAU,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC;QAC/E,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACrC,MAAM,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACvC,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACrC,MAAM,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACvC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IACvC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,6BAA6B,EAAE,KAAK,IAAI,EAAE;QAC3C,MAAM,MAAM,GAAG,UAAU,CAAC;YACxB,6HAA6H;SAC9H,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,MAAM,eAAe,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,MAAM,CAAC,CAAC;QAC5D,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAClC,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACvC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,4BAA4B,EAAE,KAAK,IAAI,EAAE;QAC1C,MAAM,MAAM,GAAG,UAAU,CAAC;YACxB,IAAI,CAAC,SAAS,CAAC;gBACb,eAAe,EAAE,CAAC;gBAClB,iBAAiB,EAAE,EAAE;gBACrB,eAAe,EAAE,CAAC,CAAC;gBACnB,iBAAiB,EAAE,CAAC;gBACpB,MAAM,EAAE,OAAO;aAChB,CAAC;SACH,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,MAAM,eAAe,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,MAAM,CAAC,CAAC;QAC5D,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACrC,MAAM,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACvC,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACrC,MAAM,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACzC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,gCAAgC,EAAE,KAAK,IAAI,EAAE;QAC9C,MAAM,MAAM,GAAG,UAAU,CAAC;YACxB,IAAI,CAAC,SAAS,CAAC;gBACb,eAAe,EAAE,CAAC;gBAClB,iBAAiB,EAAE,CAAC;gBACpB,eAAe,EAAE,CAAC;gBAClB,iBAAiB,EAAE,CAAC;gBACpB,MAAM,EAAE,SAAS;aAClB,CAAC;SACH,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,MAAM,eAAe,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,MAAM,CAAC,CAAC;QAC5D,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACpC,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,eAAe,EAAE,GAAG,EAAE;IAC7B,EAAE,CAAC,8CAA8C,EAAE,KAAK,IAAI,EAAE;QAC5D,oEAAoE;QACpE,MAAM,MAAM,GAAG,UAAU,CAAC;YACxB,mBAAmB;YACnB,sBAAsB;YACtB,IAAI,CAAC,SAAS,CAAC;gBACb,eAAe,EAAE,CAAC;gBAClB,iBAAiB,EAAE,CAAC;gBACpB,eAAe,EAAE,CAAC;gBAClB,iBAAiB,EAAE,CAAC;gBACpB,MAAM,EAAE,OAAO;gBACf,SAAS,EAAE,aAAa;aACzB,CAAC;SACH,CAAC,CAAC;QAEH,wCAAwC;QACxC,EAAE,CAAC,KAAK,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC,CAAC,qBAAqB;QAEpE,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,aAAa,EAAE,eAAe,EAAE,MAAM,CAAC,CAAC;QAE3E,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QAC1C,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;QACrD,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,IAAI,CAAC,sBAAsB,CAAC,CAAC;QAC3D,qCAAqC;QACrC,MAAM,CAAC,MAAM,CAAC,iBAAiB,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACzC,MAAM,CAAC,MAAM,CAAC,mBAAmB,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC3C,MAAM,CAAC,MAAM,CAAC,oBAAoB,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC5C,MAAM,CAAC,MAAM,CAAC,sBAAsB,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC9C,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAEpC,EAAE,CAAC,eAAe,EAAE,CAAC;IACvB,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,0CAA0C,EAAE,KAAK,IAAI,EAAE;QACxD,MAAM,MAAM,GAAG,UAAU,CAAC;YACxB,WAAW;YACX,cAAc;YACd,IAAI,CAAC,SAAS,CAAC;gBACb,eAAe,EAAE,CAAC;gBAClB,iBAAiB,EAAE,CAAC;gBACpB,eAAe,EAAE,CAAC;gBAClB,iBAAiB,EAAE,CAAC;gBACpB,MAAM,EAAE,QAAQ;aACjB,CAAC;SACH,CAAC,CAAC;QAEH,qBAAqB;QACrB,EAAE,CAAC,KAAK,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC;QAE9C,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,GAAG,EAAE,GAAG,EAAE,MAAM,CAAC,CAAC;QACrD,6CAA6C;QAC7C,MAAM,CAAC,MAAM,CAAC,iBAAiB,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACzC,MAAM,CAAC,MAAM,CAAC,oBAAoB,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC5C,8BAA8B;QAC9B,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAEpC,EAAE,CAAC,eAAe,EAAE,CAAC;IACvB,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

package/dist/eval/__tests__/llm.test.js

@@ -3,11 +3,18 @@ import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
 // Mocks
 // ---------------------------------------------------------------------------
 const mockCreate = vi.hoisted(() => vi.fn());
+const mockExecFile = vi.hoisted(() => vi.fn());
 vi.mock("@anthropic-ai/sdk", () => ({
     default: class MockAnthropic {
         messages = { create: mockCreate };
     },
 }));
+vi.mock("node:child_process", () => ({
+    execFile: mockExecFile,
+}));
+vi.mock("node:util", () => ({
+    promisify: (fn) => fn,
+}));
 // ---------------------------------------------------------------------------
 // Import module under test AFTER mocks
 // ---------------------------------------------------------------------------
@@ -19,67 +26,177 @@ describe("createLlmClient", () => {
     const origEnv = { ...process.env };
     beforeEach(() => {
         vi.resetAllMocks();
-        process.env.ANTHROPIC_API_KEY = "test-key";
+        delete process.env.VSKILL_EVAL_PROVIDER;
         delete process.env.VSKILL_EVAL_MODEL;
+        delete process.env.ANTHROPIC_API_KEY;
+        delete process.env.OLLAMA_BASE_URL;
+        delete process.env.CLAUDECODE;
     });
     afterEach(() => {
         process.env = { ...origEnv };
     });
-    it("returns text content on successful generate call", async () => {
-        mockCreate.mockResolvedValue({
-            content: [{ type: "text", text: "Generated response" }],
-        });
+    // -------------------------------------------------------------------------
+    // Auto-detection
+    // -------------------------------------------------------------------------
+    it("defaults to claude-cli from a plain terminal", () => {
         const client = createLlmClient();
-        const result = await client.generate("system prompt", "user prompt");
-        expect(result).toBe("Generated response");
-        expect(mockCreate).toHaveBeenCalledOnce();
+        expect(client.model).toBe("claude-sonnet");
     });
-    it("uses default model claude-sonnet-4-20250514 when env not set", async () => {
-        mockCreate.mockResolvedValue({
-            content: [{ type: "text", text: "ok" }],
-        });
+    it("auto-detects ollama inside Claude Code session", () => {
+        process.env.CLAUDECODE = "1";
         const client = createLlmClient();
-        await client.generate("sys", "usr");
-        expect(mockCreate).toHaveBeenCalledWith(expect.objectContaining({ model: "claude-sonnet-4-20250514" }), expect.anything());
+        expect(client.model).toBe("llama3.1:8b");
     });
-    it("uses custom model from VSKILL_EVAL_MODEL env var", async () => {
-        process.env.VSKILL_EVAL_MODEL = "claude-opus-4-20250514";
-        mockCreate.mockResolvedValue({
-            content: [{ type: "text", text: "ok" }],
-        });
+    it("auto-detects anthropic when ANTHROPIC_API_KEY is set", () => {
+        process.env.ANTHROPIC_API_KEY = "test-key";
         const client = createLlmClient();
-        await client.generate("sys", "usr");
-        expect(mockCreate).toHaveBeenCalledWith(expect.objectContaining({ model: "claude-opus-4-20250514" }), expect.anything());
+        expect(client.model).toBe("claude-sonnet-4-20250514");
     });
-    it("propagates network error from SDK", async () => {
-        mockCreate.mockRejectedValue(new Error("Connection timeout"));
+    it("CLAUDECODE takes priority over ANTHROPIC_API_KEY for auto-detection", () => {
+        process.env.CLAUDECODE = "1";
+        process.env.ANTHROPIC_API_KEY = "test-key";
         const client = createLlmClient();
-        await expect(client.generate("sys", "usr")).rejects.toThrow("Connection timeout");
+        expect(client.model).toBe("llama3.1:8b");
     });
-    it("passes system and user prompts correctly", async () => {
-        mockCreate.mockResolvedValue({
-            content: [{ type: "text", text: "ok" }],
-        });
+    it("explicit VSKILL_EVAL_PROVIDER overrides auto-detection", () => {
+        process.env.VSKILL_EVAL_PROVIDER = "ollama";
+        process.env.ANTHROPIC_API_KEY = "test-key";
         const client = createLlmClient();
-        await client.generate("my system prompt", "my user prompt");
-        expect(mockCreate).toHaveBeenCalledWith(expect.objectContaining({
-            system: "my system prompt",
-            messages: [{ role: "user", content: "my user prompt" }],
-            max_tokens: 4096,
-        }), expect.anything());
+        expect(client.model).toBe("llama3.1:8b");
     });
-    it("throws when ANTHROPIC_API_KEY is not set", () => {
-        delete process.env.ANTHROPIC_API_KEY;
-        expect(() => createLlmClient()).toThrow("ANTHROPIC_API_KEY is not set");
+    it("throws on unknown provider", () => {
+        process.env.VSKILL_EVAL_PROVIDER = "gpt-magic";
+        expect(() => createLlmClient()).toThrow('Unknown VSKILL_EVAL_PROVIDER: "gpt-magic"');
     });
-    it("exposes model name on the client", () => {
-        const client = createLlmClient();
-        expect(client.model).toBe("claude-sonnet-4-20250514");
+    // -------------------------------------------------------------------------
+    // Anthropic provider
+    // -------------------------------------------------------------------------
+    describe("anthropic provider", () => {
+        beforeEach(() => {
+            process.env.VSKILL_EVAL_PROVIDER = "anthropic";
+            process.env.ANTHROPIC_API_KEY = "test-key";
+        });
+        it("returns text content on successful generate call", async () => {
+            mockCreate.mockResolvedValue({
+                content: [{ type: "text", text: "Generated response" }],
+            });
+            const client = createLlmClient();
+            const result = await client.generate("system prompt", "user prompt");
+            expect(result).toBe("Generated response");
+            expect(mockCreate).toHaveBeenCalledOnce();
+        });
+        it("uses default model claude-sonnet-4-20250514", async () => {
+            mockCreate.mockResolvedValue({
+                content: [{ type: "text", text: "ok" }],
+            });
+            const client = createLlmClient();
+            await client.generate("sys", "usr");
+            expect(mockCreate).toHaveBeenCalledWith(expect.objectContaining({ model: "claude-sonnet-4-20250514" }), expect.anything());
+        });
+        it("uses custom model from VSKILL_EVAL_MODEL", async () => {
+            process.env.VSKILL_EVAL_MODEL = "claude-opus-4-20250514";
+            mockCreate.mockResolvedValue({
+                content: [{ type: "text", text: "ok" }],
+            });
+            const client = createLlmClient();
+            await client.generate("sys", "usr");
+            expect(mockCreate).toHaveBeenCalledWith(expect.objectContaining({ model: "claude-opus-4-20250514" }), expect.anything());
+        });
+        it("propagates network error from SDK", async () => {
+            mockCreate.mockRejectedValue(new Error("Connection timeout"));
+            const client = createLlmClient();
+            await expect(client.generate("sys", "usr")).rejects.toThrow("Connection timeout");
+        });
+        it("passes system and user prompts correctly", async () => {
+            mockCreate.mockResolvedValue({
+                content: [{ type: "text", text: "ok" }],
+            });
+            const client = createLlmClient();
+            await client.generate("my system prompt", "my user prompt");
+            expect(mockCreate).toHaveBeenCalledWith(expect.objectContaining({
+                system: "my system prompt",
+                messages: [{ role: "user", content: "my user prompt" }],
+                max_tokens: 4096,
+            }), expect.anything());
+        });
+        it("throws when ANTHROPIC_API_KEY is not set", () => {
+            delete process.env.ANTHROPIC_API_KEY;
+            expect(() => createLlmClient()).toThrow("ANTHROPIC_API_KEY is not set");
+        });
     });
-    it("exposes custom model name when VSKILL_EVAL_MODEL is set", () => {
-        process.env.VSKILL_EVAL_MODEL = "claude-opus-4-20250514";
-        const client = createLlmClient();
-        expect(client.model).toBe("claude-opus-4-20250514");
+    // -------------------------------------------------------------------------
+    // Claude CLI provider
+    // -------------------------------------------------------------------------
+    describe("claude-cli provider", () => {
+        beforeEach(() => {
+            process.env.VSKILL_EVAL_PROVIDER = "claude-cli";
+        });
+        it("calls claude CLI with --model flag", async () => {
+            mockExecFile.mockResolvedValue({ stdout: "CLI response\n" });
+            const client = createLlmClient();
+            const result = await client.generate("system prompt", "user prompt");
+            expect(result).toBe("CLI response");
+            expect(mockExecFile).toHaveBeenCalledWith("claude", ["-p", "system prompt\n\nuser prompt", "--model", "sonnet", "--no-input"], expect.objectContaining({ timeout: 120_000 }));
+        });
+        it("defaults to sonnet model", () => {
+            const client = createLlmClient();
+            expect(client.model).toBe("claude-sonnet");
+        });
+        it("passes custom model from VSKILL_EVAL_MODEL", async () => {
+            process.env.VSKILL_EVAL_MODEL = "opus";
+            mockExecFile.mockResolvedValue({ stdout: "ok\n" });
+            const client = createLlmClient();
+            expect(client.model).toBe("claude-opus");
+            await client.generate("sys", "usr");
+            expect(mockExecFile).toHaveBeenCalledWith("claude", expect.arrayContaining(["--model", "opus"]), expect.anything());
+        });
+        it("throws helpful error when claude CLI not found", async () => {
+            const err = new Error("ENOENT");
+            err.code = "ENOENT";
+            mockExecFile.mockRejectedValue(err);
+            const client = createLlmClient();
+            await expect(client.generate("sys", "usr")).rejects.toThrow("Claude CLI not found");
+        });
+        it("throws when explicitly selected inside Claude Code session", () => {
+            process.env.CLAUDECODE = "1";
+            expect(() => createLlmClient()).toThrow("Cannot use claude-cli provider inside a Claude Code session");
+        });
+    });
+    // -------------------------------------------------------------------------
+    // Ollama provider
+    // -------------------------------------------------------------------------
+    describe("ollama provider", () => {
+        beforeEach(() => {
+            process.env.VSKILL_EVAL_PROVIDER = "ollama";
+        });
+        it("uses default model llama3.1:8b", () => {
+            const client = createLlmClient();
+            expect(client.model).toBe("llama3.1:8b");
+        });
+        it("uses custom model from VSKILL_EVAL_MODEL", () => {
+            process.env.VSKILL_EVAL_MODEL = "qwen2.5:32b";
+            const client = createLlmClient();
+            expect(client.model).toBe("qwen2.5:32b");
+        });
+        it("calls Ollama HTTP API with correct payload", async () => {
+            const mockFetch = vi.spyOn(globalThis, "fetch").mockResolvedValue(new Response(JSON.stringify({ response: "Ollama reply" }), { status: 200 }));
+            const client = createLlmClient();
+            const result = await client.generate("system prompt", "user prompt");
+            expect(result).toBe("Ollama reply");
+            expect(mockFetch).toHaveBeenCalledWith("http://localhost:11434/api/generate", expect.objectContaining({
+                method: "POST",
+                body: expect.stringContaining('"model":"llama3.1:8b"'),
+            }));
+            mockFetch.mockRestore();
+        });
+        it("uses custom base URL from OLLAMA_BASE_URL", async () => {
+            process.env.OLLAMA_BASE_URL = "http://gpu-server:11434";
+            const mockFetch = vi.spyOn(globalThis, "fetch").mockResolvedValue(new Response(JSON.stringify({ response: "ok" }), { status: 200 }));
+            const client = createLlmClient();
+            await client.generate("sys", "usr");
+            expect(mockFetch).toHaveBeenCalledWith("http://gpu-server:11434/api/generate", expect.anything());
+            mockFetch.mockRestore();
+        });
     });
 });
 //# sourceMappingURL=llm.test.js.map

package/dist/eval/__tests__/llm.test.js.map

@@ -1 +1 @@
-{"version":3,"file":"llm.test.js","sourceRoot":"","sources":["../../../src/eval/__tests__/llm.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AAEzE,8EAA8E;AAC9E,QAAQ;AACR,8EAA8E;AAE9E,MAAM,UAAU,GAAG,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;AAE7C,EAAE,CAAC,IAAI,CAAC,mBAAmB,EAAE,GAAG,EAAE,CAAC,CAAC;IAClC,OAAO,EAAE,MAAM,aAAa;QAC1B,QAAQ,GAAG,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC;KACnC;CACF,CAAC,CAAC,CAAC;AAEJ,8EAA8E;AAC9E,uCAAuC;AACvC,8EAA8E;AAE9E,MAAM,EAAE,eAAe,EAAE,GAAG,MAAM,MAAM,CAAC,WAAW,CAAC,CAAC;AAEtD,8EAA8E;AAC9E,QAAQ;AACR,8EAA8E;AAE9E,QAAQ,CAAC,iBAAiB,EAAE,GAAG,EAAE;IAC/B,MAAM,OAAO,GAAG,EAAE,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC;IAEnC,UAAU,CAAC,GAAG,EAAE;QACd,EAAE,CAAC,aAAa,EAAE,CAAC;QACnB,OAAO,CAAC,GAAG,CAAC,iBAAiB,GAAG,UAAU,CAAC;QAC3C,OAAO,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC;IACvC,CAAC,CAAC,CAAC;IAEH,SAAS,CAAC,GAAG,EAAE;QACb,OAAO,CAAC,GAAG,GAAG,EAAE,GAAG,OAAO,EAAE,CAAC;IAC/B,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,kDAAkD,EAAE,KAAK,IAAI,EAAE;QAChE,UAAU,CAAC,iBAAiB,CAAC;YAC3B,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,oBAAoB,EAAE,CAAC;SACxD,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;QACjC,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC,eAAe,EAAE,aAAa,CAAC,CAAC;QAErE,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;QAC1C,MAAM,CAAC,UAAU,CAAC,CAAC,oBAAoB,EAAE,CAAC;IAC5C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,8DAA8D,EAAE,KAAK,IAAI,EAAE;QAC5E,UAAU,CAAC,iBAAiB,CAAC;YAC3B,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;SACxC,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;QACjC,MAAM,MAAM,CAAC,QAAQ,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;QAEpC,MAAM,CAAC,UAAU,CAAC,CAAC,oBAAoB,CACrC,MAAM,CAAC,gBAAgB,CAAC,EAAE,KAAK,EAAE,0BAA0B,EAAE,CAAC,EAC9D,MAAM,CAAC,QAAQ,EAAE,CAClB,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,kDAAkD,EAAE,KAAK,IAAI,EAAE;QAChE,OAAO,CAAC,GAAG,CAAC,iBAAiB,GAAG,wBAAwB,CAAC;QACzD,UAAU,CAAC,iBAAiB,CAAC;YAC3B,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;SACxC,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;QACjC,MAAM,MAAM,CAAC,QAAQ,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;QAEpC,MAAM,CAAC,UAAU,CAAC,CAAC,oBAAoB,CACrC,MAAM,CAAC,gBAAgB,CAAC,EAAE,KAAK,EAAE,wBAAwB,EAAE,CAAC,EAC5D,MAAM,CAAC,QAAQ,EAAE,CAClB,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,mCAAmC,EAAE,KAAK,IAAI,EAAE;QACjD,UAAU,CAAC,iBAAiB,CAAC,IAAI,KAAK,CAAC,oBAAoB,CAAC,CAAC,CAAC;QAE9D,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;QACjC,MAAM,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CACzD,oBAAoB,CACrB,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,0CAA0C,EAAE,KAAK,IAAI,EAAE;QACxD,UAAU,CAAC,iBAAiB,CAAC;YAC3B,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;SACxC,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;QACjC,MAAM,MAAM,CAAC,QAAQ,CAAC,kBAAkB,EAAE,gBAAgB,CAAC,CAAC;QAE5D,MAAM,CAAC,UAAU,CAAC,CAAC,oBAAoB,CACrC,MAAM,CAAC,gBAAgB,CAAC;YACtB,MAAM,EAAE,kBAAkB;YAC1B,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,gBAAgB,EAAE,CAAC;YACvD,UAAU,EAAE,IAAI;SACjB,CAAC,EACF,MAAM,CAAC,QAAQ,EAAE,CAClB,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,0CAA0C,EAAE,GAAG,EAAE;QAClD,OAAO,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC;QACrC,MAAM,CAAC,GAAG,EAAE,CAAC,eAAe,EAAE,CAAC,CAAC,OAAO,CAAC,8BAA8B,CAAC,CAAC;IAC1E,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,kCAAkC,EAAE,GAAG,EAAE;QAC1C,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;QACjC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;IACxD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,yDAAyD,EAAE,GAAG,EAAE;QACjE,OAAO,CAAC,GAAG,CAAC,iBAAiB,GAAG,wBAAwB,CAAC;QACzD,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;QACjC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAC;IACtD,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
+{"version":3,"file":"llm.test.js","sourceRoot":"","sources":["../../../src/eval/__tests__/llm.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AAEzE,8EAA8E;AAC9E,QAAQ;AACR,8EAA8E;AAE9E,MAAM,UAAU,GAAG,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;AAC7C,MAAM,YAAY,GAAG,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;AAE/C,EAAE,CAAC,IAAI,CAAC,mBAAmB,EAAE,GAAG,EAAE,CAAC,CAAC;IAClC,OAAO,EAAE,MAAM,aAAa;QAC1B,QAAQ,GAAG,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC;KACnC;CACF,CAAC,CAAC,CAAC;AAEJ,EAAE,CAAC,IAAI,CAAC,oBAAoB,EAAE,GAAG,EAAE,CAAC,CAAC;IACnC,QAAQ,EAAE,YAAY;CACvB,CAAC,CAAC,CAAC;AAEJ,EAAE,CAAC,IAAI,CAAC,WAAW,EAAE,GAAG,EAAE,CAAC,CAAC;IAC1B,SAAS,EAAE,CAAC,EAAO,EAAE,EAAE,CAAC,EAAE;CAC3B,CAAC,CAAC,CAAC;AAEJ,8EAA8E;AAC9E,uCAAuC;AACvC,8EAA8E;AAE9E,MAAM,EAAE,eAAe,EAAE,GAAG,MAAM,MAAM,CAAC,WAAW,CAAC,CAAC;AAEtD,8EAA8E;AAC9E,QAAQ;AACR,8EAA8E;AAE9E,QAAQ,CAAC,iBAAiB,EAAE,GAAG,EAAE;IAC/B,MAAM,OAAO,GAAG,EAAE,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC;IAEnC,UAAU,CAAC,GAAG,EAAE;QACd,EAAE,CAAC,aAAa,EAAE,CAAC;QACnB,OAAO,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC;QACxC,OAAO,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC;QACrC,OAAO,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC;QACrC,OAAO,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC;QACnC,OAAO,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC;IAChC,CAAC,CAAC,CAAC;IAEH,SAAS,CAAC,GAAG,EAAE;QACb,OAAO,CAAC,GAAG,GAAG,EAAE,GAAG,OAAO,EAAE,CAAC;IAC/B,CAAC,CAAC,CAAC;IAEH,4EAA4E;IAC5E,iBAAiB;IACjB,4EAA4E;IAE5E,EAAE,CAAC,8CAA8C,EAAE,GAAG,EAAE;QACtD,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;QACjC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;IAC7C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,gDAAgD,EAAE,GAAG,EAAE;QACxD,OAAO,CAAC,GAAG,CAAC,UAAU,GAAG,GAAG,CAAC;QAC7B,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;QACjC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;IAC3C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,sDAAsD,EAAE,GAAG,EAAE;QAC9D,OAAO,CAAC,GAAG,CAAC,iBAAiB,GAAG,UAAU,CAAC;QAC3C,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;QACjC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;IACxD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,qEAAqE,EAAE,GAAG,EAAE;QAC7E,OAAO,CAAC,GAAG,CAAC,UAAU,GAAG,GAAG,CAAC;QAC7B,OAAO,CAAC,GAAG,CAAC,iBAAiB,GAAG,UAAU,CAAC;QAC3C,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;QACjC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;IAC3C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,wDAAwD,EAAE,GAAG,EAAE;QAChE,OAAO,CAAC,GAAG,CAAC,oBAAoB,GAAG,QAAQ,CAAC;QAC5C,OAAO,CAAC,GAAG,CAAC,iBAAiB,GAAG,UAAU,CAAC;QAC3C,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;QACjC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;IAC3C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,4BAA4B,EAAE,GAAG,EAAE;QACpC,OAAO,CAAC,GAAG,CAAC,oBAAoB,GAAG,WAAW,CAAC;QAC/C,MAAM,CAAC,GAAG,EAAE,CAAC,eAAe,EAAE,CAAC,CAAC,OAAO,CAAC,2CAA2C,CAAC,CAAC;IACvF,CAAC,CAAC,CAAC;IAEH,4EAA4E;IAC5E,qBAAqB;IACrB,4EAA4E;IAE5E,QAAQ,CAAC,oBAAoB,EAAE,GAAG,EAAE;QAClC,UAAU,CAAC,GAAG,EAAE;YACd,OAAO,CAAC,GAAG,CAAC,oBAAoB,GAAG,WAAW,CAAC;YAC/C,OAAO,CAAC,GAAG,CAAC,iBAAiB,GAAG,UAAU,CAAC;QAC7C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,kDAAkD,EAAE,KAAK,IAAI,EAAE;YAChE,UAAU,CAAC,iBAAiB,CAAC;gBAC3B,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,oBAAoB,EAAE,CAAC;aACxD,CAAC,CAAC;YAEH,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;YACjC,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC,eAAe,EAAE,aAAa,CAAC,CAAC;YAErE,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;YAC1C,MAAM,CAAC,UAAU,CAAC,CAAC,oBAAoB,EAAE,CAAC;QAC5C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,6CAA6C,EAAE,KAAK,IAAI,EAAE;YAC3D,UAAU,CAAC,iBAAiB,CAAC;gBAC3B,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;aACxC,CAAC,CAAC;YAEH,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;YACjC,MAAM,MAAM,CAAC,QAAQ,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;YAEpC,MAAM,CAAC,UAAU,CAAC,CAAC,oBAAoB,CACrC,MAAM,CAAC,gBAAgB,CAAC,EAAE,KAAK,EAAE,0BAA0B,EAAE,CAAC,EAC9D,MAAM,CAAC,QAAQ,EAAE,CAClB,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,0CAA0C,EAAE,KAAK,IAAI,EAAE;YACxD,OAAO,CAAC,GAAG,CAAC,iBAAiB,GAAG,wBAAwB,CAAC;YACzD,UAAU,CAAC,iBAAiB,CAAC;gBAC3B,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;aACxC,CAAC,CAAC;YAEH,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;YACjC,MAAM,MAAM,CAAC,QAAQ,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;YAEpC,MAAM,CAAC,UAAU,CAAC,CAAC,oBAAoB,CACrC,MAAM,CAAC,gBAAgB,CAAC,EAAE,KAAK,EAAE,wBAAwB,EAAE,CAAC,EAC5D,MAAM,CAAC,QAAQ,EAAE,CAClB,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,mCAAmC,EAAE,KAAK,IAAI,EAAE;YACjD,UAAU,CAAC,iBAAiB,CAAC,IAAI,KAAK,CAAC,oBAAoB,CAAC,CAAC,CAAC;YAE9D,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;YACjC,MAAM,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CACzD,oBAAoB,CACrB,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,0CAA0C,EAAE,KAAK,IAAI,EAAE;YACxD,UAAU,CAAC,iBAAiB,CAAC;gBAC3B,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;aACxC,CAAC,CAAC;YAEH,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;YACjC,MAAM,MAAM,CAAC,QAAQ,CAAC,kBAAkB,EAAE,gBAAgB,CAAC,CAAC;YAE5D,MAAM,CAAC,UAAU,CAAC,CAAC,oBAAoB,CACrC,MAAM,CAAC,gBAAgB,CAAC;gBACtB,MAAM,EAAE,kBAAkB;gBAC1B,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,gBAAgB,EAAE,CAAC;gBACvD,UAAU,EAAE,IAAI;aACjB,CAAC,EACF,MAAM,CAAC,QAAQ,EAAE,CAClB,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,0CAA0C,EAAE,GAAG,EAAE;YAClD,OAAO,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC;YACrC,MAAM,CAAC,GAAG,EAAE,CAAC,eAAe,EAAE,CAAC,CAAC,OAAO,CAAC,8BAA8B,CAAC,CAAC;QAC1E,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,4EAA4E;IAC5E,sBAAsB;IACtB,4EAA4E;IAE5E,QAAQ,CAAC,qBAAqB,EAAE,GAAG,EAAE;QACnC,UAAU,CAAC,GAAG,EAAE;YACd,OAAO,CAAC,GAAG,CAAC,oBAAoB,GAAG,YAAY,CAAC;QAClD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,oCAAoC,EAAE,KAAK,IAAI,EAAE;YAClD,YAAY,CAAC,iBAAiB,CAAC,EAAE,MAAM,EAAE,gBAAgB,EAAE,CAAC,CAAC;YAE7D,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;YACjC,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC,eAAe,EAAE,aAAa,CAAC,CAAC;YAErE,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;YACpC,MAAM,CAAC,YAAY,CAAC,CAAC,oBAAoB,CACvC,QAAQ,EACR,CAAC,IAAI,EAAE,8BAA8B,EAAE,SAAS,EAAE,QAAQ,EAAE,YAAY,CAAC,EACzE,MAAM,CAAC,gBAAgB,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAC9C,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,0BAA0B,EAAE,GAAG,EAAE;YAClC,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;YACjC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;QAC7C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,4CAA4C,EAAE,KAAK,IAAI,EAAE;YAC1D,OAAO,CAAC,GAAG,CAAC,iBAAiB,GAAG,MAAM,CAAC;YACvC,YAAY,CAAC,iBAAiB,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;YAEnD,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;YACjC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;YACzC,MAAM,MAAM,CAAC,QAAQ,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;YAEpC,MAAM,CAAC,YAAY,CAAC,CAAC,oBAAoB,CACvC,QAAQ,EACR,MAAM,CAAC,eAAe,CAAC,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC,EAC3C,MAAM,CAAC,QAAQ,EAAE,CAClB,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,gDAAgD,EAAE,KAAK,IAAI,EAAE;YAC9D,MAAM,GAAG,GAAG,IAAI,KAAK,CAAC,QAAQ,CAAQ,CAAC;YACvC,GAAG,CAAC,IAAI,GAAG,QAAQ,CAAC;YACpB,YAAY,CAAC,iBAAiB,CAAC,GAAG,CAAC,CAAC;YAEpC,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;YACjC,MAAM,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CACzD,sBAAsB,CACvB,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,4DAA4D,EAAE,GAAG,EAAE;YACpE,OAAO,CAAC,GAAG,CAAC,UAAU,GAAG,GAAG,CAAC;YAC7B,MAAM,CAAC,GAAG,EAAE,CAAC,eAAe,EAAE,CAAC,CAAC,OAAO,CACrC,6DAA6D,CAC9D,CAAC;QACJ,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,4EAA4E;IAC5E,kBAAkB;IAClB,4EAA4E;IAE5E,QAAQ,CAAC,iBAAiB,EAAE,GAAG,EAAE;QAC/B,UAAU,CAAC,GAAG,EAAE;YACd,OAAO,CAAC,GAAG,CAAC,oBAAoB,GAAG,QAAQ,CAAC;QAC9C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,gCAAgC,EAAE,GAAG,EAAE;YACxC,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;YACjC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QAC3C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,0CAA0C,EAAE,GAAG,EAAE;YAClD,OAAO,CAAC,GAAG,CAAC,iBAAiB,GAAG,aAAa,CAAC;YAC9C,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;YACjC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QAC3C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,4CAA4C,EAAE,KAAK,IAAI,EAAE;YAC1D,MAAM,SAAS,GAAG,EAAE,CAAC,KAAK,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC,iBAAiB,CAC/D,IAAI,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,QAAQ,EAAE,cAAc,EAAE,CAAC,EAAE,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,CAC5E,CAAC;YAEF,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;YACjC,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC,eAAe,EAAE,aAAa,CAAC,CAAC;YAErE,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;YACpC,MAAM,CAAC,SAAS,CAAC,CAAC,oBAAoB,CACpC,qCAAqC,EACrC,MAAM,CAAC,gBAAgB,CAAC;gBACtB,MAAM,EAAE,MAAM;gBACd,IAAI,EAAE,MAAM,CAAC,gBAAgB,CAAC,uBAAuB,CAAC;aACvD,CAAC,CACH,CAAC;YAEF,SAAS,CAAC,WAAW,EAAE,CAAC;QAC1B,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,2CAA2C,EAAE,KAAK,IAAI,EAAE;YACzD,OAAO,CAAC,GAAG,CAAC,eAAe,GAAG,yBAAyB,CAAC;YAExD,MAAM,SAAS,GAAG,EAAE,CAAC,KAAK,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC,iBAAiB,CAC/D,IAAI,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,EAAE,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,CAClE,CAAC;YAEF,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;YACjC,MAAM,MAAM,CAAC,QAAQ,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;YAEpC,MAAM,CAAC,SAAS,CAAC,CAAC,oBAAoB,CACpC,sCAAsC,EACtC,MAAM,CAAC,QAAQ,EAAE,CAClB,CAAC;YAEF,SAAS,CAAC,WAAW,EAAE,CAAC;QAC1B,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

package/dist/eval/__tests__/verdict.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/eval/__tests__/verdict.test.js

@@ -0,0 +1,47 @@
+import { describe, it, expect } from "vitest";
+import { computeVerdict, verdictColor } from "../verdict.js";
+describe("computeVerdict", () => {
+    it("returns EFFECTIVE when passRate >= 0.8 and skill rubric > baseline + 1", () => {
+        expect(computeVerdict(0.85, 4.5, 3.0)).toBe("EFFECTIVE");
+        expect(computeVerdict(0.80, 4.0, 2.5)).toBe("EFFECTIVE");
+        expect(computeVerdict(1.0, 5.0, 1.0)).toBe("EFFECTIVE");
+    });
+    it("returns MARGINAL when passRate >= 0.6 and skill rubric > baseline (but not EFFECTIVE)", () => {
+        expect(computeVerdict(0.70, 3.5, 3.0)).toBe("MARGINAL");
+        expect(computeVerdict(0.60, 2.5, 2.0)).toBe("MARGINAL");
+        // High pass rate but rubric only slightly better → MARGINAL
+        expect(computeVerdict(0.85, 3.5, 3.0)).toBe("MARGINAL");
+    });
+    it("returns INEFFECTIVE when passRate >= 0.4 (but not MARGINAL)", () => {
+        expect(computeVerdict(0.50, 2.5, 3.0)).toBe("INEFFECTIVE");
+        expect(computeVerdict(0.45, 3.0, 3.0)).toBe("INEFFECTIVE");
+        expect(computeVerdict(0.40, 1.0, 5.0)).toBe("INEFFECTIVE");
+    });
+    it("returns DEGRADING when passRate < 0.4", () => {
+        expect(computeVerdict(0.30, 2.0, 3.0)).toBe("DEGRADING");
+        expect(computeVerdict(0.10, 1.0, 1.0)).toBe("DEGRADING");
+        expect(computeVerdict(0.0, 0.0, 0.0)).toBe("DEGRADING");
+        expect(computeVerdict(0.39, 5.0, 1.0)).toBe("DEGRADING");
+    });
+    it("handles boundary values correctly", () => {
+        // Exactly 0.8 pass rate, exactly +1 rubric → EFFECTIVE
+        expect(computeVerdict(0.8, 4.0, 2.9)).toBe("EFFECTIVE");
+        // 0.8 pass rate but rubric diff exactly 1 → NOT EFFECTIVE (needs >1)
+        expect(computeVerdict(0.8, 4.0, 3.0)).toBe("MARGINAL");
+        // Exactly 0.6 pass rate, skill > baseline → MARGINAL
+        expect(computeVerdict(0.6, 3.1, 3.0)).toBe("MARGINAL");
+        // Exactly 0.6 pass rate, skill = baseline → INEFFECTIVE
+        expect(computeVerdict(0.6, 3.0, 3.0)).toBe("INEFFECTIVE");
+        // Exactly 0.4 pass rate → INEFFECTIVE
+        expect(computeVerdict(0.4, 3.0, 3.0)).toBe("INEFFECTIVE");
+    });
+});
+describe("verdictColor", () => {
+    it("returns correct colors for each verdict", () => {
+        expect(verdictColor("EFFECTIVE")).toBe("green");
+        expect(verdictColor("MARGINAL")).toBe("yellow");
+        expect(verdictColor("INEFFECTIVE")).toBe("orange");
+        expect(verdictColor("DEGRADING")).toBe("red");
+    });
+});
+//# sourceMappingURL=verdict.test.js.map

package/dist/eval/__tests__/verdict.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"verdict.test.js","sourceRoot":"","sources":["../../../src/eval/__tests__/verdict.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAC9C,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAE7D,QAAQ,CAAC,gBAAgB,EAAE,GAAG,EAAE;IAC9B,EAAE,CAAC,wEAAwE,EAAE,GAAG,EAAE;QAChF,MAAM,CAAC,cAAc,CAAC,IAAI,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QACzD,MAAM,CAAC,cAAc,CAAC,IAAI,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QACzD,MAAM,CAAC,cAAc,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IAC1D,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,uFAAuF,EAAE,GAAG,EAAE;QAC/F,MAAM,CAAC,cAAc,CAAC,IAAI,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QACxD,MAAM,CAAC,cAAc,CAAC,IAAI,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QACxD,4DAA4D;QAC5D,MAAM,CAAC,cAAc,CAAC,IAAI,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IAC1D,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,6DAA6D,EAAE,GAAG,EAAE;QACrE,MAAM,CAAC,cAAc,CAAC,IAAI,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QAC3D,MAAM,CAAC,cAAc,CAAC,IAAI,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QAC3D,MAAM,CAAC,cAAc,CAAC,IAAI,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;IAC7D,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,uCAAuC,EAAE,GAAG,EAAE;QAC/C,MAAM,CAAC,cAAc,CAAC,IAAI,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QACzD,MAAM,CAAC,cAAc,CAAC,IAAI,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QACzD,MAAM,CAAC,cAAc,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QACxD,MAAM,CAAC,cAAc,CAAC,IAAI,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IAC3D,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,mCAAmC,EAAE,GAAG,EAAE;QAC3C,uDAAuD;QACvD,MAAM,CAAC,cAAc,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QACxD,qEAAqE;QACrE,MAAM,CAAC,cAAc,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QACvD,qDAAqD;QACrD,MAAM,CAAC,cAAc,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QACvD,wDAAwD;QACxD,MAAM,CAAC,cAAc,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QAC1D,sCAAsC;QACtC,MAAM,CAAC,cAAc,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;IAC5D,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,cAAc,EAAE,GAAG,EAAE;IAC5B,EAAE,CAAC,yCAAyC,EAAE,GAAG,EAAE;QACjD,MAAM,CAAC,YAAY,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAChD,MAAM,CAAC,YAAY,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAChD,MAAM,CAAC,YAAY,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACnD,MAAM,CAAC,YAAY,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAChD,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

package/dist/eval/activation-tester.d.ts

@@ -0,0 +1,25 @@
+import type { LlmClient } from "./llm.js";
+export interface ActivationPrompt {
+    prompt: string;
+    expected: "should_activate" | "should_not_activate";
+}
+export interface ActivationResult {
+    prompt: string;
+    expected: "should_activate" | "should_not_activate";
+    activate: boolean;
+    confidence: "high" | "medium" | "low";
+    reasoning: string;
+    classification: "TP" | "TN" | "FP" | "FN";
+}
+export interface ActivationSummary {
+    results: ActivationResult[];
+    precision: number;
+    recall: number;
+    reliability: number;
+    total: number;
+    tp: number;
+    tn: number;
+    fp: number;
+    fn: number;
+}
+export declare function testActivation(skillDescription: string, prompts: ActivationPrompt[], client: LlmClient, onResult?: (result: ActivationResult) => void): Promise<ActivationSummary>;

package/dist/eval/activation-tester.js

@@ -0,0 +1,89 @@
+// ---------------------------------------------------------------------------
+// activation-tester.ts -- test SKILL.md description auto-activation quality
+// ---------------------------------------------------------------------------
+const ACTIVATION_SYSTEM_PROMPT = `You are evaluating whether a user prompt would trigger an AI skill based on its description.
+
+Given the skill description and a user prompt, determine:
+1. Would this prompt trigger this skill? (yes/no)
+2. How confident are you? (high/medium/low)
+3. Brief reasoning
+
+Respond with ONLY valid JSON:
+{
+  "activate": true/false,
+  "confidence": "high" | "medium" | "low",
+  "reasoning": "brief explanation"
+}`;
+export async function testActivation(skillDescription, prompts, client, onResult) {
+    const results = [];
+    for (const p of prompts) {
+        const userPrompt = `## Skill Description
+${skillDescription}
+
+## User Prompt
+${p.prompt}
+
+Would this user prompt trigger this skill?`;
+        try {
+            const response = await client.generate(ACTIVATION_SYSTEM_PROMPT, userPrompt);
+            const jsonMatch = response.match(/```(?:json)?\s*([\s\S]*?)```/) || [null, response];
+            const json = JSON.parse(jsonMatch[1].trim());
+            const activate = !!json.activate;
+            const confidence = ["high", "medium", "low"].includes(json.confidence)
+                ? json.confidence
+                : "low";
+            const classification = classifyResult(p.expected, activate);
+            const result = {
+                prompt: p.prompt,
+                expected: p.expected,
+                activate,
+                confidence,
+                reasoning: String(json.reasoning || ""),
+                classification,
+            };
+            results.push(result);
+            onResult?.(result);
+        }
+        catch (err) {
+            const result = {
+                prompt: p.prompt,
+                expected: p.expected,
+                activate: false,
+                confidence: "low",
+                reasoning: `Error: ${err instanceof Error ? err.message : String(err)}`,
+                classification: p.expected === "should_activate" ? "FN" : "TN",
+            };
+            results.push(result);
+            onResult?.(result);
+        }
+    }
+    return computeSummary(results);
+}
+function classifyResult(expected, actual) {
+    if (expected === "should_activate" && actual)
+        return "TP";
+    if (expected === "should_activate" && !actual)
+        return "FN";
+    if (expected === "should_not_activate" && !actual)
+        return "TN";
+    return "FP";
+}
+function computeSummary(results) {
+    const tp = results.filter((r) => r.classification === "TP").length;
+    const tn = results.filter((r) => r.classification === "TN").length;
+    const fp = results.filter((r) => r.classification === "FP").length;
+    const fn = results.filter((r) => r.classification === "FN").length;
+    const total = results.length;
+    return {
+        results,
+        precision: tp + fp > 0 ? tp / (tp + fp) : 0,
+        recall: tp + fn > 0 ? tp / (tp + fn) : 0,
+        reliability: total > 0 ? (tp + tn) / total : 0,
+        total,
+        tp,
+        tn,
+        fp,
+        fn,
+    };
+}
+//# sourceMappingURL=activation-tester.js.map

package/dist/eval/activation-tester.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"activation-tester.js","sourceRoot":"","sources":["../../src/eval/activation-tester.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,4EAA4E;AAC5E,8EAA8E;AA8B9E,MAAM,wBAAwB,GAAG;;;;;;;;;;;;EAY/B,CAAC;AAEH,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,gBAAwB,EACxB,OAA2B,EAC3B,MAAiB,EACjB,QAA6C;IAE7C,MAAM,OAAO,GAAuB,EAAE,CAAC;IAEvC,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACxB,MAAM,UAAU,GAAG;EACrB,gBAAgB;;;EAGhB,CAAC,CAAC,MAAM;;2CAEiC,CAAC;QAExC,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC,wBAAwB,EAAE,UAAU,CAAC,CAAC;YAC7E,MAAM,SAAS,GAAG,QAAQ,CAAC,KAAK,CAAC,8BAA8B,CAAC,IAAI,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;YACrF,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAE,CAAC,IAAI,EAAE,CAAC,CAAC;YAE9C,MAAM,QAAQ,GAAG,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC;YACjC,MAAM,UAAU,GAAG,CAAC,MAAM,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,UAAU,CAAC;gBACpE,CAAC,CAAE,IAAI,CAAC,UAAwC;gBAChD,CAAC,CAAC,KAAK,CAAC;YAEV,MAAM,cAAc,GAAG,cAAc,CAAC,CAAC,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;YAE5D,MAAM,MAAM,GAAqB;gBAC/B,MAAM,EAAE,CAAC,CAAC,MAAM;gBAChB,QAAQ,EAAE,CAAC,CAAC,QAAQ;gBACpB,QAAQ;gBACR,UAAU;gBACV,SAAS,EAAE,MAAM,CAAC,IAAI,CAAC,SAAS,IAAI,EAAE,CAAC;gBACvC,cAAc;aACf,CAAC;YACF,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACrB,QAAQ,EAAE,CAAC,MAAM,CAAC,CAAC;QACrB,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,MAAM,GAAqB;gBAC/B,MAAM,EAAE,CAAC,CAAC,MAAM;gBAChB,QAAQ,EAAE,CAAC,CAAC,QAAQ;gBACpB,QAAQ,EAAE,KAAK;gBACf,UAAU,EAAE,KAAK;gBACjB,SAAS,EAAE,UAAU,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE;gBACvE,cAAc,EAAE,CAAC,CAAC,QAAQ,KAAK,iBAAiB,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI;aAC/D,CAAC;YACF,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACrB,QAAQ,EAAE,CAAC,MAAM,CAAC,CAAC;QACrB,CAAC;IACH,CAAC;IAED,OAAO,cAAc,CAAC,OAAO,CAAC,CAAC;AACjC,CAAC;AAED,SAAS,cAAc,CACrB,QAAmD,EACnD,MAAe;IAEf,IAAI,QAAQ,KAAK,iBAAiB,IAAI,MAAM;QAAE,OAAO,IAAI,CAAC;IAC1D,IAAI,QAAQ,KAAK,iBAAiB,IAAI,CAAC,MAAM;QAAE,OAAO,IAAI,CAAC;IAC3D,IAAI,QAAQ,KAAK,qBAAqB,IAAI,CAAC,MAAM;QAAE,OAAO,IAAI,CAAC;IAC/D,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,cAAc,CAAC,OAA2B;IACjD,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,cAAc,KAAK,IAAI,CAAC,CAAC,MAAM,CAAC;IACnE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,cAAc,KAAK,IAAI,CAAC,CAAC,MAAM,CAAC;IACnE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,cAAc,KAAK,IAAI,CAAC,CAAC,MAAM,CAAC;IACnE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,cAAc,KAAK,IAAI,CAAC,CAAC,MAAM,CAAC;IACnE,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC;IAE7B,OAAO;QACL,OAAO;QACP,SAAS,EAAE,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QAC3C,MAAM,EAAE,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QACxC,WAAW,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;QAC9C,KAAK;QACL,EAAE;QACF,EAAE;QACF,EAAE;QACF,EAAE;KACH,CAAC;AACJ,CAAC"}

package/dist/eval/benchmark-history.d.ts

@@ -0,0 +1,23 @@
+import type { BenchmarkResult } from "./benchmark.js";
+export interface HistorySummary {
+    timestamp: string;
+    filename: string;
+    model: string;
+    skillName: string;
+    passRate: number;
+    type: "benchmark" | "comparison";
+}
+export interface RegressionEntry {
+    assertionId: string;
+    evalId: number;
+    evalName: string;
+    previousStatus: boolean;
+    currentStatus: boolean;
+    change: "regression" | "improvement";
+}
+export declare function writeHistoryEntry(skillDir: string, result: BenchmarkResult & {
+    type?: "benchmark" | "comparison";
+}): Promise<string>;
+export declare function listHistory(skillDir: string): Promise<HistorySummary[]>;
+export declare function readHistoryEntry(skillDir: string, timestamp: string): Promise<BenchmarkResult | null>;
+export declare function computeRegressions(current: BenchmarkResult, previous: BenchmarkResult): RegressionEntry[];