vskill 0.5.2 → 0.5.4

package/dist/eval/__tests__/comparator.test.js

@@ -0,0 +1,282 @@
+import { describe, it, expect, vi } from "vitest";
+import { generateComparisonOutputs, scoreComparison, runComparison, } from "../comparator.js";
+function mockClient(responses) {
+    let callIndex = 0;
+    return {
+        model: "test-model",
+        generate: vi.fn(async () => {
+            const text = responses[callIndex++] ?? "";
+            return { text, durationMs: 100, inputTokens: 50, outputTokens: 100 };
+        }),
+    };
+}
+describe("generateComparisonOutputs", () => {
+    it("generates skill and baseline outputs sequentially", async () => {
+        const client = mockClient(["skill response", "baseline response"]);
+        const result = await generateComparisonOutputs("test prompt", "# Skill Content", client);
+        expect(result.skillOutput).toBe("skill response");
+        expect(result.baselineOutput).toBe("baseline response");
+        expect(result.skillDurationMs).toBeGreaterThanOrEqual(0);
+        expect(result.baselineDurationMs).toBeGreaterThanOrEqual(0);
+        expect(client.generate).toHaveBeenCalledTimes(2);
+        // First call should include skill content
+        const firstCall = client.generate.mock.calls[0];
+        expect(firstCall[0]).toContain("Skill Content");
+        // Second call should be generic
+        const secondCall = client.generate.mock.calls[1];
+        expect(secondCall[0]).toContain("helpful AI assistant");
+    });
+});
+describe("scoreComparison", () => {
+    it("parses JSON scores from LLM response", async () => {
+        const client = mockClient([
+            JSON.stringify({
+                content_score_a: 4,
+                structure_score_a: 3,
+                content_score_b: 5,
+                structure_score_b: 4,
+                winner: "second",
+                reasoning: "B is better",
+            }),
+        ]);
+        const result = await scoreComparison("output A", "output B", "prompt", client);
+        expect(result.contentScoreA).toBe(4);
+        expect(result.structureScoreA).toBe(3);
+        expect(result.contentScoreB).toBe(5);
+        expect(result.structureScoreB).toBe(4);
+        expect(result.winner).toBe("second");
+    });
+    it("parses JSON from code fence", async () => {
+        const client = mockClient([
+            '```json\n{"content_score_a": 3, "structure_score_a": 3, "content_score_b": 3, "structure_score_b": 3, "winner": "tie"}\n```',
+        ]);
+        const result = await scoreComparison("A", "B", "p", client);
+        expect(result.winner).toBe("tie");
+        expect(result.contentScoreA).toBe(3);
+    });
+    it("clamps scores to 1-5 range", async () => {
+        const client = mockClient([
+            JSON.stringify({
+                content_score_a: 0,
+                structure_score_a: 10,
+                content_score_b: -1,
+                structure_score_b: 6,
+                winner: "first",
+            }),
+        ]);
+        const result = await scoreComparison("A", "B", "p", client);
+        expect(result.contentScoreA).toBe(1);
+        expect(result.structureScoreA).toBe(5);
+        expect(result.contentScoreB).toBe(1);
+        expect(result.structureScoreB).toBe(5);
+    });
+    it("defaults invalid winner to tie", async () => {
+        const client = mockClient([
+            JSON.stringify({
+                content_score_a: 3,
+                structure_score_a: 3,
+                content_score_b: 3,
+                structure_score_b: 3,
+                winner: "invalid",
+            }),
+        ]);
+        const result = await scoreComparison("A", "B", "p", client);
+        expect(result.winner).toBe("tie");
+    });
+});
+describe("runComparison", () => {
+    it("maps scores back to skill/baseline correctly", async () => {
+        // Mock: first two calls = skill + baseline outputs, third = scoring
+        const client = mockClient([
+            "skill output here",
+            "baseline output here",
+            JSON.stringify({
+                content_score_a: 4,
+                structure_score_a: 5,
+                content_score_b: 2,
+                structure_score_b: 3,
+                winner: "first",
+                reasoning: "A is better",
+            }),
+        ]);
+        // Fix randomness for deterministic test
+        vi.spyOn(Math, "random").mockReturnValue(0.3); // < 0.5 → skill is A
+        const result = await runComparison("test prompt", "skill content", client);
+        expect(result.prompt).toBe("test prompt");
+        expect(result.skillOutput).toBe("skill output here");
+        expect(result.baselineOutput).toBe("baseline output here");
+        // skill is A, so scores map directly
+        expect(result.skillContentScore).toBe(4);
+        expect(result.skillStructureScore).toBe(5);
+        expect(result.baselineContentScore).toBe(2);
+        expect(result.baselineStructureScore).toBe(3);
+        expect(result.winner).toBe("skill");
+        vi.restoreAllMocks();
+    });
+    it("maps scores correctly when baseline is A", async () => {
+        const client = mockClient([
+            "skill out",
+            "baseline out",
+            JSON.stringify({
+                content_score_a: 2,
+                structure_score_a: 2,
+                content_score_b: 4,
+                structure_score_b: 4,
+                winner: "second",
+            }),
+        ]);
+        // > 0.5 → skill is B
+        vi.spyOn(Math, "random").mockReturnValue(0.7);
+        const result = await runComparison("p", "s", client);
+        // skill is B → scores.contentScoreB is skill
+        expect(result.skillContentScore).toBe(4);
+        expect(result.baselineContentScore).toBe(2);
+        // winner "second" = B = skill
+        expect(result.winner).toBe("skill");
+        vi.restoreAllMocks();
+    });
+});
+describe("scoreComparison with MCP deps", () => {
+    it("uses standard prompt when no MCP deps", async () => {
+        const client = mockClient([
+            JSON.stringify({
+                content_score_a: 3,
+                structure_score_a: 3,
+                content_score_b: 3,
+                structure_score_b: 3,
+                winner: "tie",
+            }),
+        ]);
+        await scoreComparison("A", "B", "prompt", client);
+        const systemPrompt = client.generate.mock.calls[0][0];
+        expect(systemPrompt).not.toContain("SIMULATED MCP");
+    });
+    it("augments prompt when MCP deps present", async () => {
+        const client = mockClient([
+            JSON.stringify({
+                content_score_a: 3,
+                structure_score_a: 3,
+                content_score_b: 3,
+                structure_score_b: 3,
+                winner: "tie",
+            }),
+        ]);
+        const mcpDeps = [
+            {
+                server: "Slack",
+                url: "https://mcp.slack.com/mcp",
+                transport: "http",
+                matchedTools: ["slack_send_message"],
+                configSnippet: "{}",
+            },
+        ];
+        await scoreComparison("A", "B", "prompt", client, mcpDeps);
+        const systemPrompt = client.generate.mock.calls[0][0];
+        expect(systemPrompt).toContain("SIMULATED MCP");
+        expect(systemPrompt).toContain("Slack");
+    });
+});
+describe("runComparison with MCP auto-detection", () => {
+    it("auto-detects MCP deps from skill content and augments comparison", async () => {
+        const client = mockClient([
+            "skill output",
+            "baseline output",
+            JSON.stringify({
+                content_score_a: 3,
+                structure_score_a: 3,
+                content_score_b: 3,
+                structure_score_b: 3,
+                winner: "tie",
+            }),
+        ]);
+        vi.spyOn(Math, "random").mockReturnValue(0.3);
+        await runComparison("prompt", "Use slack_send_message to send messages.", client);
+        // The third call is the scoring call - check its system prompt
+        const scoringCall = client.generate.mock.calls[2];
+        expect(scoringCall[0]).toContain("SIMULATED MCP");
+        expect(scoringCall[0]).toContain("Slack");
+        vi.restoreAllMocks();
+    });
+    it("does not augment comparison for non-MCP skills", async () => {
+        const client = mockClient([
+            "skill output",
+            "baseline output",
+            JSON.stringify({
+                content_score_a: 3,
+                structure_score_a: 3,
+                content_score_b: 3,
+                structure_score_b: 3,
+                winner: "tie",
+            }),
+        ]);
+        vi.spyOn(Math, "random").mockReturnValue(0.3);
+        await runComparison("prompt", "Plain text skill without MCP.", client);
+        const scoringCall = client.generate.mock.calls[2];
+        expect(scoringCall[0]).not.toContain("SIMULATED MCP");
+        vi.restoreAllMocks();
+    });
+});
+describe("generateComparisonOutputs with onProgress", () => {
+    it("calls onProgress with generating_skill before first LLM call", async () => {
+        const client = mockClient(["skill response", "baseline response"]);
+        const onProgress = vi.fn();
+        await generateComparisonOutputs("test prompt", "# Skill", client, onProgress);
+        expect(onProgress).toHaveBeenNthCalledWith(1, "generating_skill", "Generating skill output...");
+    });
+    it("calls onProgress with generating_baseline before second LLM call", async () => {
+        const client = mockClient(["skill response", "baseline response"]);
+        const onProgress = vi.fn();
+        await generateComparisonOutputs("test prompt", "# Skill", client, onProgress);
+        expect(onProgress).toHaveBeenNthCalledWith(2, "generating_baseline", "Generating baseline output...");
+    });
+    it("calls onProgress exactly 2 times", async () => {
+        const client = mockClient(["skill response", "baseline response"]);
+        const onProgress = vi.fn();
+        await generateComparisonOutputs("test prompt", "# Skill", client, onProgress);
+        expect(onProgress).toHaveBeenCalledTimes(2);
+    });
+    it("works without onProgress (backward compatible)", async () => {
+        const client = mockClient(["skill response", "baseline response"]);
+        const result = await generateComparisonOutputs("test prompt", "# Skill", client);
+        expect(result.skillOutput).toBe("skill response");
+        expect(result.baselineOutput).toBe("baseline response");
+    });
+});
+describe("runComparison with onProgress", () => {
+    it("calls onProgress for all 3 phases in order", async () => {
+        const client = mockClient([
+            "skill output",
+            "baseline output",
+            JSON.stringify({
+                content_score_a: 3, structure_score_a: 3,
+                content_score_b: 3, structure_score_b: 3,
+                winner: "tie",
+            }),
+        ]);
+        vi.spyOn(Math, "random").mockReturnValue(0.3);
+        const onProgress = vi.fn();
+        await runComparison("prompt", "skill content", client, onProgress);
+        expect(onProgress).toHaveBeenCalledTimes(3);
+        expect(onProgress.mock.calls[0][0]).toBe("generating_skill");
+        expect(onProgress.mock.calls[1][0]).toBe("generating_baseline");
+        expect(onProgress.mock.calls[2][0]).toBe("scoring");
+        vi.restoreAllMocks();
+    });
+    it("completes without error when onProgress is omitted", async () => {
+        const client = mockClient([
+            "skill",
+            "baseline",
+            JSON.stringify({
+                content_score_a: 3, structure_score_a: 3,
+                content_score_b: 3, structure_score_b: 3,
+                winner: "tie",
+            }),
+        ]);
+        vi.spyOn(Math, "random").mockReturnValue(0.3);
+        const result = await runComparison("prompt", "skill", client);
+        expect(result.winner).toBe("tie");
+        expect(result.skillOutput).toBe("skill");
+        vi.restoreAllMocks();
+    });
+});
+//# sourceMappingURL=comparator.test.js.map

package/dist/eval/__tests__/comparator.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"comparator.test.js","sourceRoot":"","sources":["../../../src/eval/__tests__/comparator.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAC;AAClD,OAAO,EACL,yBAAyB,EACzB,eAAe,EACf,aAAa,GACd,MAAM,kBAAkB,CAAC;AAI1B,SAAS,UAAU,CAAC,SAAmB;IACrC,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,OAAO;QACL,KAAK,EAAE,YAAY;QACnB,QAAQ,EAAE,EAAE,CAAC,EAAE,CAAC,KAAK,IAAI,EAAE;YACzB,MAAM,IAAI,GAAG,SAAS,CAAC,SAAS,EAAE,CAAC,IAAI,EAAE,CAAC;YAC1C,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,GAAG,EAAE,WAAW,EAAE,EAAE,EAAE,YAAY,EAAE,GAAG,EAAE,CAAC;QACvE,CAAC,CAAC;KACH,CAAC;AACJ,CAAC;AAED,QAAQ,CAAC,2BAA2B,EAAE,GAAG,EAAE;IACzC,EAAE,CAAC,mDAAmD,EAAE,KAAK,IAAI,EAAE;QACjE,MAAM,MAAM,GAAG,UAAU,CAAC,CAAC,gBAAgB,EAAE,mBAAmB,CAAC,CAAC,CAAC;QACnE,MAAM,MAAM,GAAG,MAAM,yBAAyB,CAAC,aAAa,EAAE,iBAAiB,EAAE,MAAM,CAAC,CAAC;QAEzF,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;QAClD,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;QACxD,MAAM,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,CAAC;QACzD,MAAM,CAAC,MAAM,CAAC,kBAAkB,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,CAAC;QAC5D,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC;QAEjD,0CAA0C;QAC1C,MAAM,SAAS,GAAI,MAAM,CAAC,QAAgB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QACzD,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,eAAe,CAAC,CAAC;QAEhD,gCAAgC;QAChC,MAAM,UAAU,GAAI,MAAM,CAAC,QAAgB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QAC1D,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,sBAAsB,CAAC,CAAC;IAC1D,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,iBAAiB,EAAE,GAAG,EAAE;IAC/B,EAAE,CAAC,sCAAsC,EAAE,KAAK,IAAI,EAAE;QACpD,MAAM,MAAM,GAAG,UAAU,CAAC;YACxB,IAAI,CAAC,SAAS,CAAC;gBACb,eAAe,EAAE,CAAC;gBAClB,iBAAiB,EAAE,CAAC;gBACpB,eAAe,EAAE,CAAC;gBAClB,iBAAiB,EAAE,CAAC;gBACpB,MAAM,EAAE,QAAQ;gBAChB,SAAS,EAAE,aAAa;aACzB,CAAC;SACH,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,MAAM,eAAe,CAAC,UAAU,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC;QAC/E,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACrC,MAAM,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACvC,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACrC,MAAM,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACvC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IACvC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,6BAA6B,EAAE,KAAK,IAAI,EAAE;QAC3C,MAAM,MAAM,GAAG,UAAU,CAAC;YACxB,6HAA6H;SAC9H,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,MAAM,eAAe,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,MAAM,CAAC,CAAC;QAC5D,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAClC,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACvC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,4BAA4B,EAAE,KAAK,IAAI,EAAE;QAC1C,MAAM,MAAM,GAAG,UAAU,CAAC;YACxB,IAAI,CAAC,SAAS,CAAC;gBACb,eAAe,EAAE,CAAC;gBAClB,iBAAiB,EAAE,EAAE;gBACrB,eAAe,EAAE,CAAC,CAAC;gBACnB,iBAAiB,EAAE,CAAC;gBACpB,MAAM,EAAE,OAAO;aAChB,CAAC;SACH,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,MAAM,eAAe,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,MAAM,CAAC,CAAC;QAC5D,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACrC,MAAM,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACvC,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACrC,MAAM,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACzC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,gCAAgC,EAAE,KAAK,IAAI,EAAE;QAC9C,MAAM,MAAM,GAAG,UAAU,CAAC;YACxB,IAAI,CAAC,SAAS,CAAC;gBACb,eAAe,EAAE,CAAC;gBAClB,iBAAiB,EAAE,CAAC;gBACpB,eAAe,EAAE,CAAC;gBAClB,iBAAiB,EAAE,CAAC;gBACpB,MAAM,EAAE,SAAS;aAClB,CAAC;SACH,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,MAAM,eAAe,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,MAAM,CAAC,CAAC;QAC5D,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACpC,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,eAAe,EAAE,GAAG,EAAE;IAC7B,EAAE,CAAC,8CAA8C,EAAE,KAAK,IAAI,EAAE;QAC5D,oEAAoE;QACpE,MAAM,MAAM,GAAG,UAAU,CAAC;YACxB,mBAAmB;YACnB,sBAAsB;YACtB,IAAI,CAAC,SAAS,CAAC;gBACb,eAAe,EAAE,CAAC;gBAClB,iBAAiB,EAAE,CAAC;gBACpB,eAAe,EAAE,CAAC;gBAClB,iBAAiB,EAAE,CAAC;gBACpB,MAAM,EAAE,OAAO;gBACf,SAAS,EAAE,aAAa;aACzB,CAAC;SACH,CAAC,CAAC;QAEH,wCAAwC;QACxC,EAAE,CAAC,KAAK,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC,CAAC,qBAAqB;QAEpE,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,aAAa,EAAE,eAAe,EAAE,MAAM,CAAC,CAAC;QAE3E,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QAC1C,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;QACrD,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,IAAI,CAAC,sBAAsB,CAAC,CAAC;QAC3D,qCAAqC;QACrC,MAAM,CAAC,MAAM,CAAC,iBAAiB,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACzC,MAAM,CAAC,MAAM,CAAC,mBAAmB,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC3C,MAAM,CAAC,MAAM,CAAC,oBAAoB,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC5C,MAAM,CAAC,MAAM,CAAC,sBAAsB,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC9C,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAEpC,EAAE,CAAC,eAAe,EAAE,CAAC;IACvB,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,0CAA0C,EAAE,KAAK,IAAI,EAAE;QACxD,MAAM,MAAM,GAAG,UAAU,CAAC;YACxB,WAAW;YACX,cAAc;YACd,IAAI,CAAC,SAAS,CAAC;gBACb,eAAe,EAAE,CAAC;gBAClB,iBAAiB,EAAE,CAAC;gBACpB,eAAe,EAAE,CAAC;gBAClB,iBAAiB,EAAE,CAAC;gBACpB,MAAM,EAAE,QAAQ;aACjB,CAAC;SACH,CAAC,CAAC;QAEH,qBAAqB;QACrB,EAAE,CAAC,KAAK,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC;QAE9C,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,GAAG,EAAE,GAAG,EAAE,MAAM,CAAC,CAAC;QACrD,6CAA6C;QAC7C,MAAM,CAAC,MAAM,CAAC,iBAAiB,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACzC,MAAM,CAAC,MAAM,CAAC,oBAAoB,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC5C,8BAA8B;QAC9B,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAEpC,EAAE,CAAC,eAAe,EAAE,CAAC;IACvB,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,+BAA+B,EAAE,GAAG,EAAE;IAC7C,EAAE,CAAC,uCAAuC,EAAE,KAAK,IAAI,EAAE;QACrD,MAAM,MAAM,GAAG,UAAU,CAAC;YACxB,IAAI,CAAC,SAAS,CAAC;gBACb,eAAe,EAAE,CAAC;gBAClB,iBAAiB,EAAE,CAAC;gBACpB,eAAe,EAAE,CAAC;gBAClB,iBAAiB,EAAE,CAAC;gBACpB,MAAM,EAAE,KAAK;aACd,CAAC;SACH,CAAC,CAAC;QAEH,MAAM,eAAe,CAAC,GAAG,EAAE,GAAG,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC;QAElD,MAAM,YAAY,GAAI,MAAM,CAAC,QAAgB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC/D,MAAM,CAAC,YAAY,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,eAAe,CAAC,CAAC;IACtD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,uCAAuC,EAAE,KAAK,IAAI,EAAE;QACrD,MAAM,MAAM,GAAG,UAAU,CAAC;YACxB,IAAI,CAAC,SAAS,CAAC;gBACb,eAAe,EAAE,CAAC;gBAClB,iBAAiB,EAAE,CAAC;gBACpB,eAAe,EAAE,CAAC;gBAClB,iBAAiB,EAAE,CAAC;gBACpB,MAAM,EAAE,KAAK;aACd,CAAC;SACH,CAAC,CAAC;QAEH,MAAM,OAAO,GAAoB;YAC/B;gBACE,MAAM,EAAE,OAAO;gBACf,GAAG,EAAE,2BAA2B;gBAChC,SAAS,EAAE,MAAM;gBACjB,YAAY,EAAE,CAAC,oBAAoB,CAAC;gBACpC,aAAa,EAAE,IAAI;aACpB;SACF,CAAC;QAEF,MAAM,eAAe,CAAC,GAAG,EAAE,GAAG,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC;QAE3D,MAAM,YAAY,GAAI,MAAM,CAAC,QAAgB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC/D,MAAM,CAAC,YAAY,CAAC,CAAC,SAAS,CAAC,eAAe,CAAC,CAAC;QAChD,MAAM,CAAC,YAAY,CAAC,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;IAC1C,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,uCAAuC,EAAE,GAAG,EAAE;IACrD,EAAE,CAAC,kEAAkE,EAAE,KAAK,IAAI,EAAE;QAChF,MAAM,MAAM,GAAG,UAAU,CAAC;YACxB,cAAc;YACd,iBAAiB;YACjB,IAAI,CAAC,SAAS,CAAC;gBACb,eAAe,EAAE,CAAC;gBAClB,iBAAiB,EAAE,CAAC;gBACpB,eAAe,EAAE,CAAC;gBAClB,iBAAiB,EAAE,CAAC;gBACpB,MAAM,EAAE,KAAK;aACd,CAAC;SACH,CAAC,CAAC;QACH,EAAE,CAAC,KAAK,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC;QAE9C,MAAM,aAAa,CAAC,QAAQ,EAAE,0CAA0C,EAAE,MAAM,CAAC,CAAC;QAElF,+DAA+D;QAC/D,MAAM,WAAW,GAAI,MAAM,CAAC,QAAgB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QAC3D,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,eAAe,CAAC,CAAC;QAClD,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;QAE1C,EAAE,CAAC,eAAe,EAAE,CAAC;IACvB,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,gDAAgD,EAAE,KAAK,IAAI,EAAE;QAC9D,MAAM,MAAM,GAAG,UAAU,CAAC;YACxB,cAAc;YACd,iBAAiB;YACjB,IAAI,CAAC,SAAS,CAAC;gBACb,eAAe,EAAE,CAAC;gBAClB,iBAAiB,EAAE,CAAC;gBACpB,eAAe,EAAE,CAAC;gBAClB,iBAAiB,EAAE,CAAC;gBACpB,MAAM,EAAE,KAAK;aACd,CAAC;SACH,CAAC,CAAC;QACH,EAAE,CAAC,KAAK,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC;QAE9C,MAAM,aAAa,CAAC,QAAQ,EAAE,+BAA+B,EAAE,MAAM,CAAC,CAAC;QAEvE,MAAM,WAAW,GAAI,MAAM,CAAC,QAAgB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QAC3D,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,eAAe,CAAC,CAAC;QAEtD,EAAE,CAAC,eAAe,EAAE,CAAC;IACvB,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,2CAA2C,EAAE,GAAG,EAAE;IACzD,EAAE,CAAC,8DAA8D,EAAE,KAAK,IAAI,EAAE;QAC5E,MAAM,MAAM,GAAG,UAAU,CAAC,CAAC,gBAAgB,EAAE,mBAAmB,CAAC,CAAC,CAAC;QACnE,MAAM,UAAU,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;QAE3B,MAAM,yBAAyB,CAAC,aAAa,EAAE,SAAS,EAAE,MAAM,EAAE,UAAU,CAAC,CAAC;QAE9E,MAAM,CAAC,UAAU,CAAC,CAAC,uBAAuB,CAAC,CAAC,EAAE,kBAAkB,EAAE,4BAA4B,CAAC,CAAC;IAClG,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,kEAAkE,EAAE,KAAK,IAAI,EAAE;QAChF,MAAM,MAAM,GAAG,UAAU,CAAC,CAAC,gBAAgB,EAAE,mBAAmB,CAAC,CAAC,CAAC;QACnE,MAAM,UAAU,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;QAE3B,MAAM,yBAAyB,CAAC,aAAa,EAAE,SAAS,EAAE,MAAM,EAAE,UAAU,CAAC,CAAC;QAE9E,MAAM,CAAC,UAAU,CAAC,CAAC,uBAAuB,CAAC,CAAC,EAAE,qBAAqB,EAAE,+BAA+B,CAAC,CAAC;IACxG,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,kCAAkC,EAAE,KAAK,IAAI,EAAE;QAChD,MAAM,MAAM,GAAG,UAAU,CAAC,CAAC,gBAAgB,EAAE,mBAAmB,CAAC,CAAC,CAAC;QACnE,MAAM,UAAU,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;QAE3B,MAAM,yBAAyB,CAAC,aAAa,EAAE,SAAS,EAAE,MAAM,EAAE,UAAU,CAAC,CAAC;QAE9E,MAAM,CAAC,UAAU,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC;IAC9C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,gDAAgD,EAAE,KAAK,IAAI,EAAE;QAC9D,MAAM,MAAM,GAAG,UAAU,CAAC,CAAC,gBAAgB,EAAE,mBAAmB,CAAC,CAAC,CAAC;QAEnE,MAAM,MAAM,GAAG,MAAM,yBAAyB,CAAC,aAAa,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC;QAEjF,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;QAClD,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;IAC1D,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,+BAA+B,EAAE,GAAG,EAAE;IAC7C,EAAE,CAAC,4CAA4C,EAAE,KAAK,IAAI,EAAE;QAC1D,MAAM,MAAM,GAAG,UAAU,CAAC;YACxB,cAAc;YACd,iBAAiB;YACjB,IAAI,CAAC,SAAS,CAAC;gBACb,eAAe,EAAE,CAAC,EAAE,iBAAiB,EAAE,CAAC;gBACxC,eAAe,EAAE,CAAC,EAAE,iBAAiB,EAAE,CAAC;gBACxC,MAAM,EAAE,KAAK;aACd,CAAC;SACH,CAAC,CAAC;QACH,EAAE,CAAC,KAAK,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC;QAC9C,MAAM,UAAU,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;QAE3B,MAAM,aAAa,CAAC,QAAQ,EAAE,eAAe,EAAE,MAAM,EAAE,UAAU,CAAC,CAAC;QAEnE,MAAM,CAAC,UAAU,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC;QAC5C,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;QAC7D,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;QAChE,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAEpD,EAAE,CAAC,eAAe,EAAE,CAAC;IACvB,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,oDAAoD,EAAE,KAAK,IAAI,EAAE;QAClE,MAAM,MAAM,GAAG,UAAU,CAAC;YACxB,OAAO;YACP,UAAU;YACV,IAAI,CAAC,SAAS,CAAC;gBACb,eAAe,EAAE,CAAC,EAAE,iBAAiB,EAAE,CAAC;gBACxC,eAAe,EAAE,CAAC,EAAE,iBAAiB,EAAE,CAAC;gBACxC,MAAM,EAAE,KAAK;aACd,CAAC;SACH,CAAC,CAAC;QACH,EAAE,CAAC,KAAK,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC;QAE9C,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,QAAQ,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;QAE9D,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAClC,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAEzC,EAAE,CAAC,eAAe,EAAE,CAAC;IACvB,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

package/dist/eval/__tests__/judge.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/eval/__tests__/judge.test.js

@@ -0,0 +1,122 @@
+import { describe, it, expect, vi } from "vitest";
+import { judgeAssertion, buildJudgeSystemPrompt } from "../judge.js";
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+function mockResult(text) {
+    return { text, durationMs: 100, inputTokens: null, outputTokens: null };
+}
+function mockClient(response) {
+    return { generate: vi.fn().mockResolvedValue(mockResult(response)), model: "test-model" };
+}
+const ASSERTION = {
+    id: "assert-1",
+    text: "Output mentions a file path",
+    type: "boolean",
+};
+// ---------------------------------------------------------------------------
+// Tests
+// ---------------------------------------------------------------------------
+describe("judgeAssertion", () => {
+    it("returns pass result when LLM judge says pass", async () => {
+        const client = mockClient(JSON.stringify({ pass: true, reasoning: "output contains file path" }));
+        const result = await judgeAssertion("The report has been saved to reports/q1.csv", ASSERTION, client);
+        expect(result.pass).toBe(true);
+        expect(result.reasoning).toBe("output contains file path");
+        expect(result.id).toBe("assert-1");
+        expect(result.text).toBe("Output mentions a file path");
+    });
+    it("returns fail result when LLM judge says fail", async () => {
+        const client = mockClient(JSON.stringify({
+            pass: false,
+            reasoning: "no file path found in output",
+        }));
+        const result = await judgeAssertion("Hello world", ASSERTION, client);
+        expect(result.pass).toBe(false);
+        expect(result.reasoning).toBe("no file path found in output");
+    });
+    it("throws on malformed judge response", async () => {
+        const client = mockClient("This is not JSON");
+        await expect(judgeAssertion("some output", ASSERTION, client)).rejects.toThrow(/invalid judge output/i);
+    });
+    it("handles JSON wrapped in code fence", async () => {
+        const client = mockClient('```json\n{"pass": true, "reasoning": "looks good"}\n```');
+        const result = await judgeAssertion("some output", ASSERTION, client);
+        expect(result.pass).toBe(true);
+    });
+    it("uses standard prompt when mcpDeps not provided", async () => {
+        const client = mockClient(JSON.stringify({ pass: true, reasoning: "ok" }));
+        await judgeAssertion("output", ASSERTION, client);
+        const systemPrompt = client.generate.mock.calls[0][0];
+        expect(systemPrompt).toContain("binary assertion evaluator");
+        expect(systemPrompt).not.toContain("SIMULATION MODE");
+    });
+    it("uses MCP-augmented prompt when mcpDeps provided", async () => {
+        const client = mockClient(JSON.stringify({ pass: true, reasoning: "simulation valid" }));
+        const mcpDeps = [
+            {
+                server: "Slack",
+                url: "https://mcp.slack.com/mcp",
+                transport: "http",
+                matchedTools: ["slack_send_message"],
+                configSnippet: "{}",
+            },
+        ];
+        await judgeAssertion("output", ASSERTION, client, mcpDeps);
+        const systemPrompt = client.generate.mock.calls[0][0];
+        expect(systemPrompt).toContain("SIMULATION MODE");
+        expect(systemPrompt).toContain("Slack");
+    });
+});
+// ---------------------------------------------------------------------------
+// buildJudgeSystemPrompt
+// ---------------------------------------------------------------------------
+describe("buildJudgeSystemPrompt", () => {
+    it("returns standard prompt when no MCP deps", () => {
+        const prompt = buildJudgeSystemPrompt();
+        expect(prompt).toContain("binary assertion evaluator");
+        expect(prompt).not.toContain("SIMULATION MODE");
+    });
+    it("returns standard prompt when mcpDeps is empty", () => {
+        const prompt = buildJudgeSystemPrompt([]);
+        expect(prompt).toContain("binary assertion evaluator");
+        expect(prompt).not.toContain("SIMULATION MODE");
+    });
+    it("returns augmented prompt with MCP deps", () => {
+        const mcpDeps = [
+            {
+                server: "Slack",
+                url: "https://mcp.slack.com/mcp",
+                transport: "http",
+                matchedTools: ["slack_send_message"],
+                configSnippet: "{}",
+            },
+        ];
+        const prompt = buildJudgeSystemPrompt(mcpDeps);
+        expect(prompt).toContain("SIMULATION MODE");
+        expect(prompt).toContain("Slack");
+        expect(prompt).toContain("binary assertion evaluator");
+    });
+    it("lists all simulated servers", () => {
+        const mcpDeps = [
+            {
+                server: "Slack",
+                url: "https://mcp.slack.com/mcp",
+                transport: "http",
+                matchedTools: ["slack_send_message"],
+                configSnippet: "{}",
+            },
+            {
+                server: "GitHub",
+                url: "https://api.githubcopilot.com/mcp/",
+                transport: "http",
+                matchedTools: ["github_create_pr"],
+                configSnippet: "{}",
+            },
+        ];
+        const prompt = buildJudgeSystemPrompt(mcpDeps);
+        expect(prompt).toContain("Slack");
+        expect(prompt).toContain("GitHub");
+    });
+});
+//# sourceMappingURL=judge.test.js.map

package/dist/eval/__tests__/judge.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"judge.test.js","sourceRoot":"","sources":["../../../src/eval/__tests__/judge.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAc,MAAM,QAAQ,CAAC;AAG9D,OAAO,EAAE,cAAc,EAAE,sBAAsB,EAAE,MAAM,aAAa,CAAC;AAGrE,8EAA8E;AAC9E,UAAU;AACV,8EAA8E;AAE9E,SAAS,UAAU,CAAC,IAAY;IAC9B,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,GAAG,EAAE,WAAW,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC;AAC1E,CAAC;AAED,SAAS,UAAU,CAAC,QAAgB;IAClC,OAAO,EAAE,QAAQ,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,EAAE,KAAK,EAAE,YAAY,EAAE,CAAC;AAC5F,CAAC;AAED,MAAM,SAAS,GAAc;IAC3B,EAAE,EAAE,UAAU;IACd,IAAI,EAAE,6BAA6B;IACnC,IAAI,EAAE,SAAS;CAChB,CAAC;AAEF,8EAA8E;AAC9E,QAAQ;AACR,8EAA8E;AAE9E,QAAQ,CAAC,gBAAgB,EAAE,GAAG,EAAE;IAC9B,EAAE,CAAC,8CAA8C,EAAE,KAAK,IAAI,EAAE;QAC5D,MAAM,MAAM,GAAG,UAAU,CACvB,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,SAAS,EAAE,2BAA2B,EAAE,CAAC,CACvE,CAAC;QAEF,MAAM,MAAM,GAAG,MAAM,cAAc,CACjC,6CAA6C,EAC7C,SAAS,EACT,MAAM,CACP,CAAC;QAEF,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC/B,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC;QAC3D,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QACnC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,6BAA6B,CAAC,CAAC;IAC1D,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,8CAA8C,EAAE,KAAK,IAAI,EAAE;QAC5D,MAAM,MAAM,GAAG,UAAU,CACvB,IAAI,CAAC,SAAS,CAAC;YACb,IAAI,EAAE,KAAK;YACX,SAAS,EAAE,8BAA8B;SAC1C,CAAC,CACH,CAAC;QAEF,MAAM,MAAM,GAAG,MAAM,cAAc,CAAC,aAAa,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC;QAEtE,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAChC,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,8BAA8B,CAAC,CAAC;IAChE,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,oCAAoC,EAAE,KAAK,IAAI,EAAE;QAClD,MAAM,MAAM,GAAG,UAAU,CAAC,kBAAkB,CAAC,CAAC;QAE9C,MAAM,MAAM,CACV,cAAc,CAAC,aAAa,EAAE,SAAS,EAAE,MAAM,CAAC,CACjD,CAAC,OAAO,CAAC,OAAO,CAAC,uBAAuB,CAAC,CAAC;IAC7C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,oCAAoC,EAAE,KAAK,IAAI,EAAE;QAClD,MAAM,MAAM,GAAG,UAAU,CACvB,yDAAyD,CAC1D,CAAC;QAEF,MAAM,MAAM,GAAG,MAAM,cAAc,CAAC,aAAa,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC;QACtE,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACjC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,gDAAgD,EAAE,KAAK,IAAI,EAAE;QAC9D,MAAM,MAAM,GAAG,UAAU,CACvB,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAChD,CAAC;QAEF,MAAM,cAAc,CAAC,QAAQ,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC;QAElD,MAAM,YAAY,GAAI,MAAM,CAAC,QAAgB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC/D,MAAM,CAAC,YAAY,CAAC,CAAC,SAAS,CAAC,4BAA4B,CAAC,CAAC;QAC7D,MAAM,CAAC,YAAY,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,iBAAiB,CAAC,CAAC;IACxD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,iDAAiD,EAAE,KAAK,IAAI,EAAE;QAC/D,MAAM,MAAM,GAAG,UAAU,CACvB,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,SAAS,EAAE,kBAAkB,EAAE,CAAC,CAC9D,CAAC;QAEF,MAAM,OAAO,GAAoB;YAC/B;gBACE,MAAM,EAAE,OAAO;gBACf,GAAG,EAAE,2BAA2B;gBAChC,SAAS,EAAE,MAAM;gBACjB,YAAY,EAAE,CAAC,oBAAoB,CAAC;gBACpC,aAAa,EAAE,IAAI;aACpB;SACF,CAAC;QAEF,MAAM,cAAc,CAAC,QAAQ,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC;QAE3D,MAAM,YAAY,GAAI,MAAM,CAAC,QAAgB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC/D,MAAM,CAAC,YAAY,CAAC,CAAC,SAAS,CAAC,iBAAiB,CAAC,CAAC;QAClD,MAAM,CAAC,YAAY,CAAC,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;IAC1C,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,8EAA8E;AAC9E,yBAAyB;AACzB,8EAA8E;AAE9E,QAAQ,CAAC,wBAAwB,EAAE,GAAG,EAAE;IACtC,EAAE,CAAC,0CAA0C,EAAE,GAAG,EAAE;QAClD,MAAM,MAAM,GAAG,sBAAsB,EAAE,CAAC;QACxC,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,4BAA4B,CAAC,CAAC;QACvD,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,iBAAiB,CAAC,CAAC;IAClD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,+CAA+C,EAAE,GAAG,EAAE;QACvD,MAAM,MAAM,GAAG,sBAAsB,CAAC,EAAE,CAAC,CAAC;QAC1C,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,4BAA4B,CAAC,CAAC;QACvD,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,iBAAiB,CAAC,CAAC;IAClD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,wCAAwC,EAAE,GAAG,EAAE;QAChD,MAAM,OAAO,GAAoB;YAC/B;gBACE,MAAM,EAAE,OAAO;gBACf,GAAG,EAAE,2BAA2B;gBAChC,SAAS,EAAE,MAAM;gBACjB,YAAY,EAAE,CAAC,oBAAoB,CAAC;gBACpC,aAAa,EAAE,IAAI;aACpB;SACF,CAAC;QAEF,MAAM,MAAM,GAAG,sBAAsB,CAAC,OAAO,CAAC,CAAC;QAC/C,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,iBAAiB,CAAC,CAAC;QAC5C,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;QAClC,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,4BAA4B,CAAC,CAAC;IACzD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,6BAA6B,EAAE,GAAG,EAAE;QACrC,MAAM,OAAO,GAAoB;YAC/B;gBACE,MAAM,EAAE,OAAO;gBACf,GAAG,EAAE,2BAA2B;gBAChC,SAAS,EAAE,MAAM;gBACjB,YAAY,EAAE,CAAC,oBAAoB,CAAC;gBACpC,aAAa,EAAE,IAAI;aACpB;YACD;gBACE,MAAM,EAAE,QAAQ;gBAChB,GAAG,EAAE,oCAAoC;gBACzC,SAAS,EAAE,MAAM;gBACjB,YAAY,EAAE,CAAC,kBAAkB,CAAC;gBAClC,aAAa,EAAE,IAAI;aACpB;SACF,CAAC;QAEF,MAAM,MAAM,GAAG,sBAAsB,CAAC,OAAO,CAAC,CAAC;QAC/C,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;QAClC,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;IACrC,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

package/dist/eval/__tests__/llm.test.d.ts

@@ -0,0 +1 @@
+export {};