vskill 0.2.26 → 0.2.28

package/dist/commands/eval/__tests__/generate-all.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"generate-all.test.js","sourceRoot":"","sources":["../../../../src/commands/eval/__tests__/generate-all.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AACzE,OAAO,EAAE,SAAS,EAAE,aAAa,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AACvE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AAEjC,8EAA8E;AAC9E,QAAQ;AACR,8EAA8E;AAE9E,MAAM,YAAY,GAAG,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;AAE/C,EAAE,CAAC,IAAI,CAAC,sBAAsB,EAAE,GAAG,EAAE,CAAC,CAAC;IACrC,eAAe,EAAE,GAAG,EAAE,CAAC,CAAC;QACtB,QAAQ,EAAE,YAAY;QACtB,KAAK,EAAE,YAAY;KACpB,CAAC;CACH,CAAC,CAAC,CAAC;AAEJ,8EAA8E;AAC9E,uCAAuC;AACvC,8EAA8E;AAE9E,MAAM,EAAE,kBAAkB,EAAE,GAAG,MAAM,MAAM,CAAC,oBAAoB,CAAC,CAAC;AAElE,8EAA8E;AAC9E,UAAU;AACV,8EAA8E;AAE9E,IAAI,OAAe,CAAC;AAEpB,SAAS,WAAW,CAClB,MAAc,EACd,KAAa,EACb,OAA4B,EAAE;IAE9B,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC;IACxD,SAAS,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACzC,aAAa,CAAC,IAAI,CAAC,QAAQ,EAAE,UAAU,CAAC,EAAE,KAAK,KAAK,UAAU,KAAK,UAAU,CAAC,CAAC;IAE/E,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;QACf,SAAS,CAAC,IAAI,CAAC,QAAQ,EAAE,OAAO,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACxD,aAAa,CACX,IAAI,CAAC,QAAQ,EAAE,OAAO,EAAE,YAAY,CAAC,EACrC,IAAI,CAAC,SAAS,CAAC;YACb,UAAU,EAAE,KAAK;YACjB,KAAK,EAAE;gBACL;oBACE,EAAE,EAAE,CAAC;oBACL,IAAI,EAAE,UAAU;oBAChB,MAAM,EAAE,MAAM;oBACd,eAAe,EAAE,QAAQ;oBACzB,UAAU,EAAE,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC;iBAC3D;aACF;SACF,CAAC,CACH,CAAC;IACJ,CAAC;AACH,CAAC;AAED,MAAM,eAAe,GAAG;;;;;;;;;;;;;;;;OAgBjB,CAAC;AAER,8EAA8E;AAC9E,QAAQ;AACR,8EAA8E;AAE9E,QAAQ,CAAC,oBAAoB,EAAE,GAAG,EAAE;IAClC,UAAU,CAAC,GAAG,EAAE;QACd,OAAO,GAAG,IAAI,CAAC,MAAM,EAAE,EAAE,iBAAiB,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;QACxD,SAAS,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACxC,EAAE,CAAC,aAAa,EAAE,CAAC;QACnB,YAAY,CAAC,iBAAiB,CAAC,eAAe,CAAC,CAAC;IAClD,CAAC,CAAC,CAAC;IAEH,SAAS,CAAC,GAAG,EAAE;QACb,MAAM,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QAClD,EAAE,CAAC,eAAe,EAAE,CAAC;IACvB,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,oCAAoC,EAAE,KAAK,IAAI,EAAE;QAClD,WAAW,CAAC,WAAW,EAAE,SAAS,CAAC,CAAC;QACpC,WAAW,CAAC,WAAW,EAAE,SAAS,CAAC,CAAC;QACpC,WAAW,CAAC,WAAW,EAAE,SAAS,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACrD,MAAM,UAAU,GAAG,EAAE,CAAC,KAAK,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,kBAAkB,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QAEzE,MAAM,kBAAkB,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;QAEzC,4CAA4C;QAC5C,MAAM,CACJ,UAAU,CACR,IAAI,CAAC,OAAO,EAAE,2CAA2C,CAAC,CAC3D,CACF,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACb,MAAM,CACJ,UAAU,CACR,IAAI,CAAC,OAAO,EAAE,2CAA2C,CAAC,CAC3D,CACF,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAEb,MAAM,MAAM,GAAG,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACjE,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,cAAc,CAAC,CAAC;QACzC,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,YAAY,CAAC,CAAC;IACzC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,kCAAkC,EAAE,KAAK,IAAI,EAAE;QAChD,WAAW,CAAC,WAAW,EAAE,gBAAgB,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QAC5D,MAAM,UAAU,GAAG,EAAE,CAAC,KAAK,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,kBAAkB,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QAEzE,MAAM,kBAAkB,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;QAEzC,MAAM,MAAM,GAAG,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACjE,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,YAAY,CAAC,CAAC;QACvC,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,cAAc,CAAC,CAAC;IAC3C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,4CAA4C,EAAE,KAAK,IAAI,EAAE;QAC1D,WAAW,CAAC,WAAW,EAAE,YAAY,CAAC,CAAC;QACvC,WAAW,CAAC,WAAW,EAAE,UAAU,CAAC,CAAC;QACrC,IAAI,SAAS,GAAG,CAAC,CAAC;QAClB,YAAY,CAAC,kBAAkB,CAAC,KAAK,IAAI,EAAE;YACzC,SAAS,EAAE,CAAC;YACZ,IAAI,SAAS,IAAI,CAAC;gBAAE,MAAM,IAAI,KAAK,CAAC,WAAW,CAAC,CAAC;YACjD,OAAO,eAAe,CAAC;QACzB,CAAC,CAAC,CAAC;QACH,MAAM,UAAU,GAAG,EAAE,CAAC,KAAK,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,kBAAkB,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QACzE,EAAE,CAAC,KAAK,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,kBAAkB,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QAExD,MAAM,kBAAkB,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;QAEzC,MAAM,MAAM,GAAG,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACjE,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC;IACxC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,+BAA+B,EAAE,KAAK,IAAI,EAAE;QAC7C,WAAW,CAAC,WAAW,EAAE,gBAAgB,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QAC5D,MAAM,UAAU,GAAG,EAAE,CAAC,KAAK,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,kBAAkB,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QAEzE,MAAM,kBAAkB,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;QAExC,MAAM,MAAM,GAAG,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACjE,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,YAAY,CAAC,CAAC;QACvC,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,cAAc,CAAC,CAAC;IAC3C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,iDAAiD,EAAE,KAAK,IAAI,EAAE;QAC/D,+BAA+B;QAC/B,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,EAAE,mBAAmB,CAAC,CAAC;QACtD,SAAS,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC3C,MAAM,QAAQ,GAAG,IAAI,CAAC,UAAU,EAAE,wBAAwB,CAAC,CAAC;QAC5D,SAAS,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACzC,aAAa,CAAC,IAAI,CAAC,QAAQ,EAAE,UAAU,CAAC,EAAE,sBAAsB,CAAC,CAAC;QAElE,MAAM,UAAU,GAAG,EAAE,CAAC,KAAK,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,kBAAkB,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QAEzE,MAAM,kBAAkB,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC;QAE5C,MAAM,MAAM,GAAG,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACjE,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,YAAY,CAAC,CAAC;QACvC,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,cAAc,CAAC,CAAC;IAC3C,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

package/dist/commands/eval/__tests__/init.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/commands/eval/__tests__/init.test.js

@@ -0,0 +1,116 @@
+import { describe, it, expect, vi, beforeEach } from "vitest";
+// ---------------------------------------------------------------------------
+// Mocks
+// ---------------------------------------------------------------------------
+const mocks = vi.hoisted(() => ({
+    readFileSync: vi.fn(),
+    writeFileSync: vi.fn(),
+    existsSync: vi.fn(),
+    mkdirSync: vi.fn(),
+    generate: vi.fn(),
+}));
+vi.mock("node:fs", () => ({
+    readFileSync: mocks.readFileSync,
+    writeFileSync: mocks.writeFileSync,
+    existsSync: mocks.existsSync,
+    mkdirSync: mocks.mkdirSync,
+}));
+vi.mock("../../../eval/llm.js", () => ({
+    createLlmClient: () => ({
+        generate: mocks.generate,
+        model: "test-model",
+    }),
+}));
+// ---------------------------------------------------------------------------
+// Import module under test AFTER mocks
+// ---------------------------------------------------------------------------
+const { runEvalInit } = await import("../init.js");
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+const VALID_GENERATED = `Here is the evals.json:
+
+\`\`\`json
+{
+  "skill_name": "test-skill",
+  "evals": [
+    {
+      "id": 1,
+      "name": "Basic test",
+      "prompt": "Test prompt",
+      "expected_output": "Expected output",
+      "files": [],
+      "assertions": [
+        { "id": "a1", "text": "Check result", "type": "boolean" }
+      ]
+    }
+  ]
+}
+\`\`\``;
+// ---------------------------------------------------------------------------
+// Tests
+// ---------------------------------------------------------------------------
+describe("runEvalInit", () => {
+    beforeEach(() => {
+        vi.resetAllMocks();
+        // Default: SKILL.md exists, evals.json does not
+        mocks.existsSync.mockImplementation((p) => {
+            if (p.includes("SKILL.md"))
+                return true;
+            if (p.includes("evals.json"))
+                return false;
+            return false;
+        });
+        mocks.readFileSync.mockReturnValue("# My Skill\nDoes things.");
+        mocks.generate.mockResolvedValue(VALID_GENERATED);
+    });
+    it("creates evals.json when absent", async () => {
+        await runEvalInit("/root/plugins/marketing/skills/social-media-posting", false);
+        expect(mocks.mkdirSync).toHaveBeenCalled();
+        expect(mocks.writeFileSync).toHaveBeenCalledOnce();
+        const writtenContent = JSON.parse(mocks.writeFileSync.mock.calls[0][1]);
+        expect(writtenContent.skill_name).toBe("test-skill");
+        expect(writtenContent.evals).toHaveLength(1);
+    });
+    it("exits with message when evals.json exists and no --force", async () => {
+        mocks.existsSync.mockImplementation((p) => {
+            if (p.includes("SKILL.md"))
+                return true;
+            if (p.includes("evals.json"))
+                return true;
+            return false;
+        });
+        const consoleSpy = vi.spyOn(console, "log").mockImplementation(() => { });
+        await runEvalInit("/root/plugins/marketing/skills/smp", false);
+        expect(consoleSpy).toHaveBeenCalledWith(expect.stringContaining("already exists"));
+        expect(mocks.writeFileSync).not.toHaveBeenCalled();
+        consoleSpy.mockRestore();
+    });
+    it("overwrites when --force flag is passed", async () => {
+        mocks.existsSync.mockImplementation((p) => {
+            if (p.includes("SKILL.md"))
+                return true;
+            if (p.includes("evals.json"))
+                return true;
+            return false;
+        });
+        await runEvalInit("/root/plugins/marketing/skills/smp", true);
+        expect(mocks.writeFileSync).toHaveBeenCalledOnce();
+    });
+    it("handles LLM failure gracefully", async () => {
+        mocks.generate.mockRejectedValue(new Error("API rate limited"));
+        const consoleSpy = vi.spyOn(console, "error").mockImplementation(() => { });
+        await runEvalInit("/root/plugins/marketing/skills/smp", false);
+        expect(consoleSpy).toHaveBeenCalledWith(expect.stringContaining("API rate limited"));
+        expect(mocks.writeFileSync).not.toHaveBeenCalled();
+        consoleSpy.mockRestore();
+    });
+    it("throws when SKILL.md does not exist", async () => {
+        mocks.existsSync.mockReturnValue(false);
+        const consoleSpy = vi.spyOn(console, "error").mockImplementation(() => { });
+        await runEvalInit("/root/plugins/marketing/skills/smp", false);
+        expect(consoleSpy).toHaveBeenCalledWith(expect.stringContaining("SKILL.md"));
+        consoleSpy.mockRestore();
+    });
+});
+//# sourceMappingURL=init.test.js.map

package/dist/commands/eval/__tests__/init.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"init.test.js","sourceRoot":"","sources":["../../../../src/commands/eval/__tests__/init.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AAE9D,8EAA8E;AAC9E,QAAQ;AACR,8EAA8E;AAE9E,MAAM,KAAK,GAAG,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;IAC9B,YAAY,EAAE,EAAE,CAAC,EAAE,EAAE;IACrB,aAAa,EAAE,EAAE,CAAC,EAAE,EAAE;IACtB,UAAU,EAAE,EAAE,CAAC,EAAE,EAAE;IACnB,SAAS,EAAE,EAAE,CAAC,EAAE,EAAE;IAClB,QAAQ,EAAE,EAAE,CAAC,EAAE,EAAE;CAClB,CAAC,CAAC,CAAC;AAEJ,EAAE,CAAC,IAAI,CAAC,SAAS,EAAE,GAAG,EAAE,CAAC,CAAC;IACxB,YAAY,EAAE,KAAK,CAAC,YAAY;IAChC,aAAa,EAAE,KAAK,CAAC,aAAa;IAClC,UAAU,EAAE,KAAK,CAAC,UAAU;IAC5B,SAAS,EAAE,KAAK,CAAC,SAAS;CAC3B,CAAC,CAAC,CAAC;AAEJ,EAAE,CAAC,IAAI,CAAC,sBAAsB,EAAE,GAAG,EAAE,CAAC,CAAC;IACrC,eAAe,EAAE,GAAG,EAAE,CAAC,CAAC;QACtB,QAAQ,EAAE,KAAK,CAAC,QAAQ;QACxB,KAAK,EAAE,YAAY;KACpB,CAAC;CACH,CAAC,CAAC,CAAC;AAEJ,8EAA8E;AAC9E,uCAAuC;AACvC,8EAA8E;AAE9E,MAAM,EAAE,WAAW,EAAE,GAAG,MAAM,MAAM,CAAC,YAAY,CAAC,CAAC;AAEnD,8EAA8E;AAC9E,UAAU;AACV,8EAA8E;AAE9E,MAAM,eAAe,GAAG;;;;;;;;;;;;;;;;;;OAkBjB,CAAC;AAER,8EAA8E;AAC9E,QAAQ;AACR,8EAA8E;AAE9E,QAAQ,CAAC,aAAa,EAAE,GAAG,EAAE;IAC3B,UAAU,CAAC,GAAG,EAAE;QACd,EAAE,CAAC,aAAa,EAAE,CAAC;QACnB,gDAAgD;QAChD,KAAK,CAAC,UAAU,CAAC,kBAAkB,CAAC,CAAC,CAAS,EAAE,EAAE;YAChD,IAAI,CAAC,CAAC,QAAQ,CAAC,UAAU,CAAC;gBAAE,OAAO,IAAI,CAAC;YACxC,IAAI,CAAC,CAAC,QAAQ,CAAC,YAAY,CAAC;gBAAE,OAAO,KAAK,CAAC;YAC3C,OAAO,KAAK,CAAC;QACf,CAAC,CAAC,CAAC;QACH,KAAK,CAAC,YAAY,CAAC,eAAe,CAAC,0BAA0B,CAAC,CAAC;QAC/D,KAAK,CAAC,QAAQ,CAAC,iBAAiB,CAAC,eAAe,CAAC,CAAC;IACpD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,gCAAgC,EAAE,KAAK,IAAI,EAAE;QAC9C,MAAM,WAAW,CAAC,qDAAqD,EAAE,KAAK,CAAC,CAAC;QAEhF,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,gBAAgB,EAAE,CAAC;QAC3C,MAAM,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,oBAAoB,EAAE,CAAC;QACnD,MAAM,cAAc,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACxE,MAAM,CAAC,cAAc,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;QACrD,MAAM,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;IAC/C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,0DAA0D,EAAE,KAAK,IAAI,EAAE;QACxE,KAAK,CAAC,UAAU,CAAC,kBAAkB,CAAC,CAAC,CAAS,EAAE,EAAE;YAChD,IAAI,CAAC,CAAC,QAAQ,CAAC,UAAU,CAAC;gBAAE,OAAO,IAAI,CAAC;YACxC,IAAI,CAAC,CAAC,QAAQ,CAAC,YAAY,CAAC;gBAAE,OAAO,IAAI,CAAC;YAC1C,OAAO,KAAK,CAAC;QACf,CAAC,CAAC,CAAC;QAEH,MAAM,UAAU,GAAG,EAAE,CAAC,KAAK,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,kBAAkB,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QAEzE,MAAM,WAAW,CAAC,oCAAoC,EAAE,KAAK,CAAC,CAAC;QAE/D,MAAM,CAAC,UAAU,CAAC,CAAC,oBAAoB,CACrC,MAAM,CAAC,gBAAgB,CAAC,gBAAgB,CAAC,CAC1C,CAAC;QACF,MAAM,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,GAAG,CAAC,gBAAgB,EAAE,CAAC;QACnD,UAAU,CAAC,WAAW,EAAE,CAAC;IAC3B,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,wCAAwC,EAAE,KAAK,IAAI,EAAE;QACtD,KAAK,CAAC,UAAU,CAAC,kBAAkB,CAAC,CAAC,CAAS,EAAE,EAAE;YAChD,IAAI,CAAC,CAAC,QAAQ,CAAC,UAAU,CAAC;gBAAE,OAAO,IAAI,CAAC;YACxC,IAAI,CAAC,CAAC,QAAQ,CAAC,YAAY,CAAC;gBAAE,OAAO,IAAI,CAAC;YAC1C,OAAO,KAAK,CAAC;QACf,CAAC,CAAC,CAAC;QAEH,MAAM,WAAW,CAAC,oCAAoC,EAAE,IAAI,CAAC,CAAC;QAE9D,MAAM,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,oBAAoB,EAAE,CAAC;IACrD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,gCAAgC,EAAE,KAAK,IAAI,EAAE;QAC9C,KAAK,CAAC,QAAQ,CAAC,iBAAiB,CAAC,IAAI,KAAK,CAAC,kBAAkB,CAAC,CAAC,CAAC;QAChE,MAAM,UAAU,GAAG,EAAE,CAAC,KAAK,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,kBAAkB,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QAE3E,MAAM,WAAW,CAAC,oCAAoC,EAAE,KAAK,CAAC,CAAC;QAE/D,MAAM,CAAC,UAAU,CAAC,CAAC,oBAAoB,CACrC,MAAM,CAAC,gBAAgB,CAAC,kBAAkB,CAAC,CAC5C,CAAC;QACF,MAAM,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,GAAG,CAAC,gBAAgB,EAAE,CAAC;QACnD,UAAU,CAAC,WAAW,EAAE,CAAC;IAC3B,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,qCAAqC,EAAE,KAAK,IAAI,EAAE;QACnD,KAAK,CAAC,UAAU,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;QACxC,MAAM,UAAU,GAAG,EAAE,CAAC,KAAK,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,kBAAkB,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QAE3E,MAAM,WAAW,CAAC,oCAAoC,EAAE,KAAK,CAAC,CAAC;QAE/D,MAAM,CAAC,UAAU,CAAC,CAAC,oBAAoB,CACrC,MAAM,CAAC,gBAAgB,CAAC,UAAU,CAAC,CACpC,CAAC;QACF,UAAU,CAAC,WAAW,EAAE,CAAC;IAC3B,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

package/dist/commands/eval/__tests__/run.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/commands/eval/__tests__/run.test.js

@@ -0,0 +1,149 @@
+import { describe, it, expect, vi, beforeEach } from "vitest";
+// ---------------------------------------------------------------------------
+// Mocks
+// ---------------------------------------------------------------------------
+const mocks = vi.hoisted(() => ({
+    readFileSync: vi.fn(),
+    writeFileSync: vi.fn(),
+    existsSync: vi.fn(),
+    mkdirSync: vi.fn(),
+    generate: vi.fn(),
+}));
+vi.mock("node:fs", () => ({
+    readFileSync: mocks.readFileSync,
+    writeFileSync: mocks.writeFileSync,
+    existsSync: mocks.existsSync,
+    mkdirSync: mocks.mkdirSync,
+}));
+vi.mock("../../../eval/llm.js", () => ({
+    createLlmClient: () => ({
+        generate: mocks.generate,
+        model: "test-model",
+    }),
+}));
+// ---------------------------------------------------------------------------
+// Import module under test AFTER mocks
+// ---------------------------------------------------------------------------
+const { runEvalRun } = await import("../run.js");
+// ---------------------------------------------------------------------------
+// Fixtures
+// ---------------------------------------------------------------------------
+const VALID_EVALS = {
+    skill_name: "test-skill",
+    evals: [
+        {
+            id: 1,
+            name: "Basic test",
+            prompt: "Test prompt 1",
+            expected_output: "Expected output 1",
+            files: [],
+            assertions: [
+                { id: "a1", text: "Output mentions AI", type: "boolean" },
+                { id: "a2", text: "Output is concise", type: "boolean" },
+            ],
+        },
+        {
+            id: 2,
+            name: "Edge case test",
+            prompt: "Test prompt 2",
+            expected_output: "Expected output 2",
+            files: [],
+            assertions: [
+                { id: "b1", text: "Output handles edge case", type: "boolean" },
+            ],
+        },
+    ],
+};
+// ---------------------------------------------------------------------------
+// Tests
+// ---------------------------------------------------------------------------
+describe("runEvalRun", () => {
+    beforeEach(() => {
+        vi.resetAllMocks();
+        // Default: evals.json exists with valid content
+        mocks.existsSync.mockReturnValue(true);
+        mocks.readFileSync.mockReturnValue(JSON.stringify(VALID_EVALS));
+    });
+    it("prints results table on success", async () => {
+        // Mock LLM: first call returns output, subsequent calls judge assertions
+        let callCount = 0;
+        mocks.generate.mockImplementation(async () => {
+            callCount++;
+            // First call per case = generate output, remaining = judge assertions
+            if (callCount === 1)
+                return "AI is great";
+            if (callCount <= 3)
+                return JSON.stringify({ pass: true, reasoning: "ok" });
+            if (callCount === 4)
+                return "Edge case handled";
+            return JSON.stringify({ pass: false, reasoning: "not quite" });
+        });
+        const consoleSpy = vi.spyOn(console, "log").mockImplementation(() => { });
+        await runEvalRun("/skills/test-skill");
+        // Check table was printed
+        const output = consoleSpy.mock.calls.map((c) => c[0]).join("\n");
+        expect(output).toContain("a1");
+        expect(output).toContain("b1");
+        consoleSpy.mockRestore();
+    });
+    it("writes benchmark.json after run", async () => {
+        mocks.generate.mockImplementation(async (_sys, prompt) => {
+            if (prompt.includes("Test prompt"))
+                return "LLM output here";
+            return JSON.stringify({ pass: true, reasoning: "ok" });
+        });
+        vi.spyOn(console, "log").mockImplementation(() => { });
+        await runEvalRun("/skills/test-skill");
+        expect(mocks.writeFileSync).toHaveBeenCalled();
+        const writtenPath = mocks.writeFileSync.mock.calls[0][0];
+        expect(writtenPath).toContain("benchmark.json");
+        const writtenContent = JSON.parse(mocks.writeFileSync.mock.calls[0][1]);
+        expect(writtenContent.skill_name).toBe("test-skill");
+        expect(writtenContent.cases).toHaveLength(2);
+        vi.restoreAllMocks();
+    });
+    it("marks error case on LLM failure and continues", async () => {
+        let callCount = 0;
+        mocks.generate.mockImplementation(async () => {
+            callCount++;
+            // First case: output generation fails
+            if (callCount === 1)
+                throw new Error("API timeout");
+            // Second case: works fine
+            if (callCount === 2)
+                return "Edge case handled";
+            return JSON.stringify({ pass: true, reasoning: "ok" });
+        });
+        vi.spyOn(console, "log").mockImplementation(() => { });
+        await runEvalRun("/skills/test-skill");
+        const writtenContent = JSON.parse(mocks.writeFileSync.mock.calls[0][1]);
+        expect(writtenContent.cases[0].status).toBe("error");
+        expect(writtenContent.cases[0].error_message).toContain("API timeout");
+        expect(writtenContent.cases[1].status).toBe("pass");
+        vi.restoreAllMocks();
+    });
+    it("exits with error for missing evals.json", async () => {
+        mocks.existsSync.mockReturnValue(false);
+        const consoleSpy = vi.spyOn(console, "error").mockImplementation(() => { });
+        const exitSpy = vi
+            .spyOn(process, "exit")
+            .mockImplementation((() => { }));
+        await runEvalRun("/skills/test-skill");
+        expect(consoleSpy).toHaveBeenCalledWith(expect.stringContaining("No evals.json found"));
+        consoleSpy.mockRestore();
+        exitSpy.mockRestore();
+    });
+    it("exits with error for invalid evals.json", async () => {
+        mocks.existsSync.mockReturnValue(true);
+        mocks.readFileSync.mockReturnValue("{ broken json");
+        const consoleSpy = vi.spyOn(console, "error").mockImplementation(() => { });
+        const exitSpy = vi
+            .spyOn(process, "exit")
+            .mockImplementation((() => { }));
+        await runEvalRun("/skills/test-skill");
+        expect(consoleSpy).toHaveBeenCalledWith(expect.stringContaining("Invalid evals.json"));
+        consoleSpy.mockRestore();
+        exitSpy.mockRestore();
+    });
+});
+//# sourceMappingURL=run.test.js.map

package/dist/commands/eval/__tests__/run.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"run.test.js","sourceRoot":"","sources":["../../../../src/commands/eval/__tests__/run.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AAE9D,8EAA8E;AAC9E,QAAQ;AACR,8EAA8E;AAE9E,MAAM,KAAK,GAAG,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;IAC9B,YAAY,EAAE,EAAE,CAAC,EAAE,EAAE;IACrB,aAAa,EAAE,EAAE,CAAC,EAAE,EAAE;IACtB,UAAU,EAAE,EAAE,CAAC,EAAE,EAAE;IACnB,SAAS,EAAE,EAAE,CAAC,EAAE,EAAE;IAClB,QAAQ,EAAE,EAAE,CAAC,EAAE,EAAE;CAClB,CAAC,CAAC,CAAC;AAEJ,EAAE,CAAC,IAAI,CAAC,SAAS,EAAE,GAAG,EAAE,CAAC,CAAC;IACxB,YAAY,EAAE,KAAK,CAAC,YAAY;IAChC,aAAa,EAAE,KAAK,CAAC,aAAa;IAClC,UAAU,EAAE,KAAK,CAAC,UAAU;IAC5B,SAAS,EAAE,KAAK,CAAC,SAAS;CAC3B,CAAC,CAAC,CAAC;AAEJ,EAAE,CAAC,IAAI,CAAC,sBAAsB,EAAE,GAAG,EAAE,CAAC,CAAC;IACrC,eAAe,EAAE,GAAG,EAAE,CAAC,CAAC;QACtB,QAAQ,EAAE,KAAK,CAAC,QAAQ;QACxB,KAAK,EAAE,YAAY;KACpB,CAAC;CACH,CAAC,CAAC,CAAC;AAEJ,8EAA8E;AAC9E,uCAAuC;AACvC,8EAA8E;AAE9E,MAAM,EAAE,UAAU,EAAE,GAAG,MAAM,MAAM,CAAC,WAAW,CAAC,CAAC;AAEjD,8EAA8E;AAC9E,WAAW;AACX,8EAA8E;AAE9E,MAAM,WAAW,GAAG;IAClB,UAAU,EAAE,YAAY;IACxB,KAAK,EAAE;QACL;YACE,EAAE,EAAE,CAAC;YACL,IAAI,EAAE,YAAY;YAClB,MAAM,EAAE,eAAe;YACvB,eAAe,EAAE,mBAAmB;YACpC,KAAK,EAAE,EAAE;YACT,UAAU,EAAE;gBACV,EAAE,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,oBAAoB,EAAE,IAAI,EAAE,SAAS,EAAE;gBACzD,EAAE,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,mBAAmB,EAAE,IAAI,EAAE,SAAS,EAAE;aACzD;SACF;QACD;YACE,EAAE,EAAE,CAAC;YACL,IAAI,EAAE,gBAAgB;YACtB,MAAM,EAAE,eAAe;YACvB,eAAe,EAAE,mBAAmB;YACpC,KAAK,EAAE,EAAE;YACT,UAAU,EAAE;gBACV,EAAE,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,0BAA0B,EAAE,IAAI,EAAE,SAAS,EAAE;aAChE;SACF;KACF;CACF,CAAC;AAEF,8EAA8E;AAC9E,QAAQ;AACR,8EAA8E;AAE9E,QAAQ,CAAC,YAAY,EAAE,GAAG,EAAE;IAC1B,UAAU,CAAC,GAAG,EAAE;QACd,EAAE,CAAC,aAAa,EAAE,CAAC;QACnB,gDAAgD;QAChD,KAAK,CAAC,UAAU,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC;QACvC,KAAK,CAAC,YAAY,CAAC,eAAe,CAAC,IAAI,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC,CAAC;IAClE,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,iCAAiC,EAAE,KAAK,IAAI,EAAE;QAC/C,yEAAyE;QACzE,IAAI,SAAS,GAAG,CAAC,CAAC;QAClB,KAAK,CAAC,QAAQ,CAAC,kBAAkB,CAAC,KAAK,IAAI,EAAE;YAC3C,SAAS,EAAE,CAAC;YACZ,sEAAsE;YACtE,IAAI,SAAS,KAAK,CAAC;gBAAE,OAAO,aAAa,CAAC;YAC1C,IAAI,SAAS,IAAI,CAAC;gBAAE,OAAO,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YAC3E,IAAI,SAAS,KAAK,CAAC;gBAAE,OAAO,mBAAmB,CAAC;YAChD,OAAO,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,SAAS,EAAE,WAAW,EAAE,CAAC,CAAC;QACjE,CAAC,CAAC,CAAC;QAEH,MAAM,UAAU,GAAG,EAAE,CAAC,KAAK,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,kBAAkB,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QAEzE,MAAM,UAAU,CAAC,oBAAoB,CAAC,CAAC;QAEvC,0BAA0B;QAC1B,MAAM,MAAM,GAAG,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACjE,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;QAC/B,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;QAC/B,UAAU,CAAC,WAAW,EAAE,CAAC;IAC3B,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,iCAAiC,EAAE,KAAK,IAAI,EAAE;QAC/C,KAAK,CAAC,QAAQ,CAAC,kBAAkB,CAAC,KAAK,EAAE,IAAY,EAAE,MAAc,EAAE,EAAE;YACvE,IAAI,MAAM,CAAC,QAAQ,CAAC,aAAa,CAAC;gBAAE,OAAO,iBAAiB,CAAC;YAC7D,OAAO,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACzD,CAAC,CAAC,CAAC;QACH,EAAE,CAAC,KAAK,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,kBAAkB,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QAEtD,MAAM,UAAU,CAAC,oBAAoB,CAAC,CAAC;QAEvC,MAAM,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,gBAAgB,EAAE,CAAC;QAC/C,MAAM,WAAW,GAAG,KAAK,CAAC,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACzD,MAAM,CAAC,WAAW,CAAC,CAAC,SAAS,CAAC,gBAAgB,CAAC,CAAC;QAChD,MAAM,cAAc,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACxE,MAAM,CAAC,cAAc,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;QACrD,MAAM,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAE7C,EAAE,CAAC,eAAe,EAAE,CAAC;IACvB,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,+CAA+C,EAAE,KAAK,IAAI,EAAE;QAC7D,IAAI,SAAS,GAAG,CAAC,CAAC;QAClB,KAAK,CAAC,QAAQ,CAAC,kBAAkB,CAAC,KAAK,IAAI,EAAE;YAC3C,SAAS,EAAE,CAAC;YACZ,sCAAsC;YACtC,IAAI,SAAS,KAAK,CAAC;gBAAE,MAAM,IAAI,KAAK,CAAC,aAAa,CAAC,CAAC;YACpD,0BAA0B;YAC1B,IAAI,SAAS,KAAK,CAAC;gBAAE,OAAO,mBAAmB,CAAC;YAChD,OAAO,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACzD,CAAC,CAAC,CAAC;QACH,EAAE,CAAC,KAAK,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,kBAAkB,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QAEtD,MAAM,UAAU,CAAC,oBAAoB,CAAC,CAAC;QAEvC,MAAM,cAAc,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACxE,MAAM,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACrD,MAAM,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC;QACvE,MAAM,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAEpD,EAAE,CAAC,eAAe,EAAE,CAAC;IACvB,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,yCAAyC,EAAE,KAAK,IAAI,EAAE;QACvD,KAAK,CAAC,UAAU,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;QACxC,MAAM,UAAU,GAAG,EAAE,CAAC,KAAK,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,kBAAkB,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QAC3E,MAAM,OAAO,GAAG,EAAE;aACf,KAAK,CAAC,OAAO,EAAE,MAAM,CAAC;aACtB,kBAAkB,CAAC,CAAC,GAAG,EAAE,GAAE,CAAC,CAAQ,CAAC,CAAC;QAEzC,MAAM,UAAU,CAAC,oBAAoB,CAAC,CAAC;QAEvC,MAAM,CAAC,UAAU,CAAC,CAAC,oBAAoB,CACrC,MAAM,CAAC,gBAAgB,CAAC,qBAAqB,CAAC,CAC/C,CAAC;QACF,UAAU,CAAC,WAAW,EAAE,CAAC;QACzB,OAAO,CAAC,WAAW,EAAE,CAAC;IACxB,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,yCAAyC,EAAE,KAAK,IAAI,EAAE;QACvD,KAAK,CAAC,UAAU,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC;QACvC,KAAK,CAAC,YAAY,CAAC,eAAe,CAAC,eAAe,CAAC,CAAC;QACpD,MAAM,UAAU,GAAG,EAAE,CAAC,KAAK,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,kBAAkB,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QAC3E,MAAM,OAAO,GAAG,EAAE;aACf,KAAK,CAAC,OAAO,EAAE,MAAM,CAAC;aACtB,kBAAkB,CAAC,CAAC,GAAG,EAAE,GAAE,CAAC,CAAQ,CAAC,CAAC;QAEzC,MAAM,UAAU,CAAC,oBAAoB,CAAC,CAAC;QAEvC,MAAM,CAAC,UAAU,CAAC,CAAC,oBAAoB,CACrC,MAAM,CAAC,gBAAgB,CAAC,oBAAoB,CAAC,CAC9C,CAAC;QACF,UAAU,CAAC,WAAW,EAAE,CAAC;QACzB,OAAO,CAAC,WAAW,EAAE,CAAC;IACxB,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

package/dist/commands/eval/coverage.d.ts

@@ -0,0 +1 @@
+export declare function runEvalCoverage(root: string): Promise<void>;

package/dist/commands/eval/coverage.js

@@ -0,0 +1,79 @@
+// ---------------------------------------------------------------------------
+// vskill eval coverage -- show eval coverage across all skills
+// ---------------------------------------------------------------------------
+import { scanSkills } from "../../eval/skill-scanner.js";
+import { loadAndValidateEvals } from "../../eval/schema.js";
+import { readBenchmark } from "../../eval/benchmark.js";
+import { green, red, yellow, cyan, bold, dim, table } from "../../utils/output.js";
+export async function runEvalCoverage(root) {
+    const skills = await scanSkills(root);
+    if (skills.length === 0) {
+        console.log(dim("No skills found in " + root));
+        return;
+    }
+    const headers = ["PLUGIN", "SKILL", "CASES", "ASSERTIONS", "LAST RUN", "STATUS"];
+    const rows = [];
+    let missing = 0;
+    let pending = 0;
+    let passing = 0;
+    let failing = 0;
+    for (const skill of skills) {
+        if (!skill.hasEvals) {
+            rows.push([skill.plugin, skill.skill, "-", "-", "-", yellow("MISSING")]);
+            missing++;
+            continue;
+        }
+        let evalsFile;
+        try {
+            evalsFile = loadAndValidateEvals(skill.dir);
+        }
+        catch {
+            rows.push([skill.plugin, skill.skill, "-", "-", "-", red("INVALID")]);
+            failing++;
+            continue;
+        }
+        const caseCount = String(evalsFile.evals.length);
+        const assertionCount = String(evalsFile.evals.reduce((sum, e) => sum + e.assertions.length, 0));
+        const benchmark = await readBenchmark(skill.dir);
+        if (!benchmark) {
+            rows.push([
+                skill.plugin,
+                skill.skill,
+                caseCount,
+                assertionCount,
+                "-",
+                cyan("PENDING"),
+            ]);
+            pending++;
+            continue;
+        }
+        const anyFailed = benchmark.cases.some((c) => c.status === "fail" || c.status === "error");
+        const lastRun = benchmark.timestamp.split("T")[0];
+        if (anyFailed) {
+            rows.push([
+                skill.plugin,
+                skill.skill,
+                caseCount,
+                assertionCount,
+                lastRun,
+                red("FAIL"),
+            ]);
+            failing++;
+        }
+        else {
+            rows.push([
+                skill.plugin,
+                skill.skill,
+                caseCount,
+                assertionCount,
+                lastRun,
+                green("PASS"),
+            ]);
+            passing++;
+        }
+    }
+    console.log(bold(`\nEval Coverage Report\n`));
+    console.log(table(headers, rows));
+    console.log(`\n${bold("Summary:")} ${skills.length} skills | ${green(`${passing} pass`)} | ${red(`${failing} fail`)} | ${cyan(`${pending} pending`)} | ${yellow(`${missing} missing`)}`);
+}
+//# sourceMappingURL=coverage.js.map

package/dist/commands/eval/coverage.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"coverage.js","sourceRoot":"","sources":["../../../src/commands/eval/coverage.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,+DAA+D;AAC/D,8EAA8E;AAE9E,OAAO,EAAE,UAAU,EAAE,MAAM,6BAA6B,CAAC;AACzD,OAAO,EAAE,oBAAoB,EAAE,MAAM,sBAAsB,CAAC;AAC5D,OAAO,EAAE,aAAa,EAAE,MAAM,yBAAyB,CAAC;AACxD,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,EAAE,KAAK,EAAE,MAAM,uBAAuB,CAAC;AAEnF,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,IAAY;IAChD,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,IAAI,CAAC,CAAC;IAEtC,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACxB,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,qBAAqB,GAAG,IAAI,CAAC,CAAC,CAAC;QAC/C,OAAO;IACT,CAAC;IAED,MAAM,OAAO,GAAG,CAAC,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,QAAQ,CAAC,CAAC;IACjF,MAAM,IAAI,GAAe,EAAE,CAAC;IAE5B,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,IAAI,OAAO,GAAG,CAAC,CAAC;IAEhB,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,IAAI,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC;YACpB,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,KAAK,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;YACzE,OAAO,EAAE,CAAC;YACV,SAAS;QACX,CAAC;QAED,IAAI,SAAS,CAAC;QACd,IAAI,CAAC;YACH,SAAS,GAAG,oBAAoB,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAC9C,CAAC;QAAC,MAAM,CAAC;YACP,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,KAAK,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;YACtE,OAAO,EAAE,CAAC;YACV,SAAS;QACX,CAAC;QAED,MAAM,SAAS,GAAG,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QACjD,MAAM,cAAc,GAAG,MAAM,CAC3B,SAAS,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC,CACjE,CAAC;QAEF,MAAM,SAAS,GAAG,MAAM,aAAa,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACjD,IAAI,CAAC,SAAS,EAAE,CAAC;YACf,IAAI,CAAC,IAAI,CAAC;gBACR,KAAK,CAAC,MAAM;gBACZ,KAAK,CAAC,KAAK;gBACX,SAAS;gBACT,cAAc;gBACd,GAAG;gBACH,IAAI,CAAC,SAAS,CAAC;aAChB,CAAC,CAAC;YACH,OAAO,EAAE,CAAC;YACV,SAAS;QACX,CAAC;QAED,MAAM,SAAS,GAAG,SAAS,CAAC,KAAK,CAAC,IAAI,CACpC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,MAAM,IAAI,CAAC,CAAC,MAAM,KAAK,OAAO,CACnD,CAAC;QACF,MAAM,OAAO,GAAG,SAAS,CAAC,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAElD,IAAI,SAAS,EAAE,CAAC;YACd,IAAI,CAAC,IAAI,CAAC;gBACR,KAAK,CAAC,MAAM;gBACZ,KAAK,CAAC,KAAK;gBACX,SAAS;gBACT,cAAc;gBACd,OAAO;gBACP,GAAG,CAAC,MAAM,CAAC;aACZ,CAAC,CAAC;YACH,OAAO,EAAE,CAAC;QACZ,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,IAAI,CAAC;gBACR,KAAK,CAAC,MAAM;gBACZ,KAAK,CAAC,KAAK;gBACX,SAAS;gBACT,cAAc;gBACd,OAAO;gBACP,KAAK,CAAC,MAAM,CAAC;aACd,CAAC,CAAC;YACH,OAAO,EAAE,CAAC;QACZ,CAAC;IACH,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC,CAAC;IAC9C,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC;IAClC,OAAO,CAAC,GAAG,CACT,KAAK,IAAI,CAAC,UAAU,CAAC,IAAI,MAAM,CAAC,MAAM,aAAa,KAAK,CAAC,GAAG,OAAO,OAAO,CAAC,MAAM,GAAG,CAAC,GAAG,OAAO,OAAO,CAAC,MAAM,IAAI,CAAC,GAAG,OAAO,UAAU,CAAC,MAAM,MAAM,CAAC,GAAG,OAAO,UAAU,CAAC,EAAE,CAC5K,CAAC;AACJ,CAAC"}

package/dist/commands/eval/generate-all.d.ts

@@ -0,0 +1 @@
+export declare function runEvalGenerateAll(root: string, force: boolean): Promise<void>;

package/dist/commands/eval/generate-all.js

@@ -0,0 +1,64 @@
+// ---------------------------------------------------------------------------
+// vskill eval generate-all -- batch scaffold evals.json for all skills
+// ---------------------------------------------------------------------------
+import { readFileSync, writeFileSync, existsSync, mkdirSync } from "node:fs";
+import { join } from "node:path";
+import { scanSkills } from "../../eval/skill-scanner.js";
+import { createLlmClient } from "../../eval/llm.js";
+import { buildEvalInitPrompt, parseGeneratedEvals } from "../../eval/prompt-builder.js";
+import { green, red, yellow, bold, dim } from "../../utils/output.js";
+export async function runEvalGenerateAll(root, force) {
+    const skills = await scanSkills(root);
+    if (skills.length === 0) {
+        console.log(dim("No skills found in " + root));
+        return;
+    }
+    const client = createLlmClient();
+    let generated = 0;
+    let skipped = 0;
+    let failed = 0;
+    const failedPaths = [];
+    for (const skill of skills) {
+        const evalsPath = join(skill.dir, "evals", "evals.json");
+        // Skip if evals already exist and not forcing
+        if (skill.hasEvals && !force) {
+            skipped++;
+            continue;
+        }
+        const skillMdPath = join(skill.dir, "SKILL.md");
+        if (!existsSync(skillMdPath)) {
+            failed++;
+            failedPaths.push(`${skill.plugin}/${skill.skill} (no SKILL.md)`);
+            continue;
+        }
+        try {
+            const skillContent = readFileSync(skillMdPath, "utf-8");
+            const prompt = buildEvalInitPrompt(skillContent);
+            const raw = await client.generate("You generate eval test cases for AI skills. Output only valid JSON in a code fence.", prompt);
+            const evalsFile = parseGeneratedEvals(raw);
+            mkdirSync(join(skill.dir, "evals"), { recursive: true });
+            writeFileSync(evalsPath, JSON.stringify(evalsFile, null, 2), "utf-8");
+            generated++;
+            console.log(green(`  Generated: ${skill.plugin}/${skill.skill}`));
+        }
+        catch (err) {
+            failed++;
+            failedPaths.push(`${skill.plugin}/${skill.skill}`);
+            console.error(red(`  Failed: ${skill.plugin}/${skill.skill} - `) +
+                dim(err.message));
+        }
+    }
+    // Print summary
+    console.log(bold(`\nBatch Generation Summary`));
+    console.log(`  Scanned: ${skills.length}`);
+    console.log(`  ${green(`Generated: ${generated}`)}`);
+    console.log(`  ${yellow(`Skipped: ${skipped}`)}`);
+    console.log(`  ${failed > 0 ? red(`Failed: ${failed}`) : `Failed: ${failed}`}`);
+    if (failedPaths.length > 0) {
+        console.log(red("\nFailed skills:"));
+        for (const path of failedPaths) {
+            console.log(red(`  - ${path}`));
+        }
+    }
+}
+//# sourceMappingURL=generate-all.js.map

package/dist/commands/eval/generate-all.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"generate-all.js","sourceRoot":"","sources":["../../../src/commands/eval/generate-all.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,uEAAuE;AACvE,8EAA8E;AAE9E,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAC7E,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,UAAU,EAAE,MAAM,6BAA6B,CAAC;AACzD,OAAO,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AACpD,OAAO,EAAE,mBAAmB,EAAE,mBAAmB,EAAE,MAAM,8BAA8B,CAAC;AACxF,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,EAAE,MAAM,uBAAuB,CAAC;AAEtE,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACtC,IAAY,EACZ,KAAc;IAEd,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,IAAI,CAAC,CAAC;IAEtC,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACxB,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,qBAAqB,GAAG,IAAI,CAAC,CAAC,CAAC;QAC/C,OAAO;IACT,CAAC;IAED,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;IACjC,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,MAAM,WAAW,GAAa,EAAE,CAAC;IAEjC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,OAAO,EAAE,YAAY,CAAC,CAAC;QAEzD,8CAA8C;QAC9C,IAAI,KAAK,CAAC,QAAQ,IAAI,CAAC,KAAK,EAAE,CAAC;YAC7B,OAAO,EAAE,CAAC;YACV,SAAS;QACX,CAAC;QAED,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,UAAU,CAAC,CAAC;QAChD,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC,EAAE,CAAC;YAC7B,MAAM,EAAE,CAAC;YACT,WAAW,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,MAAM,IAAI,KAAK,CAAC,KAAK,gBAAgB,CAAC,CAAC;YACjE,SAAS;QACX,CAAC;QAED,IAAI,CAAC;YACH,MAAM,YAAY,GAAG,YAAY,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC;YACxD,MAAM,MAAM,GAAG,mBAAmB,CAAC,YAAY,CAAC,CAAC;YAEjD,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,QAAQ,CAC/B,qFAAqF,EACrF,MAAM,CACP,CAAC;YAEF,MAAM,SAAS,GAAG,mBAAmB,CAAC,GAAG,CAAC,CAAC;YAE3C,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,OAAO,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YACzD,aAAa,CAAC,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;YAEtE,SAAS,EAAE,CAAC;YACZ,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,gBAAgB,KAAK,CAAC,MAAM,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;QACpE,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,EAAE,CAAC;YACT,WAAW,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,MAAM,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC;YACnD,OAAO,CAAC,KAAK,CACX,GAAG,CAAC,aAAa,KAAK,CAAC,MAAM,IAAI,KAAK,CAAC,KAAK,KAAK,CAAC;gBAChD,GAAG,CAAE,GAAa,CAAC,OAAO,CAAC,CAC9B,CAAC;QACJ,CAAC;IACH,CAAC;IAED,gBAAgB;IAChB,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,4BAA4B,CAAC,CAAC,CAAC;IAChD,OAAO,CAAC,GAAG,CAAC,cAAc,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;IAC3C,OAAO,CAAC,GAAG,CAAC,KAAK,KAAK,CAAC,cAAc,SAAS,EAAE,CAAC,EAAE,CAAC,CAAC;IACrD,OAAO,CAAC,GAAG,CAAC,KAAK,MAAM,CAAC,YAAY,OAAO,EAAE,CAAC,EAAE,CAAC,CAAC;IAClD,OAAO,CAAC,GAAG,CAAC,KAAK,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,WAAW,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC,WAAW,MAAM,EAAE,EAAE,CAAC,CAAC;IAEhF,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC3B,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,kBAAkB,CAAC,CAAC,CAAC;QACrC,KAAK,MAAM,IAAI,IAAI,WAAW,EAAE,CAAC;YAC/B,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC,CAAC;QAClC,CAAC;IACH,CAAC;AACH,CAAC"}

package/dist/commands/eval/init.d.ts

@@ -0,0 +1 @@
+export declare function runEvalInit(skillDir: string, force: boolean): Promise<void>;

package/dist/commands/eval/init.js

@@ -0,0 +1,38 @@
+// ---------------------------------------------------------------------------
+// vskill eval init -- scaffold evals.json for a skill using LLM
+// ---------------------------------------------------------------------------
+import { readFileSync, writeFileSync, existsSync, mkdirSync } from "node:fs";
+import { join } from "node:path";
+import { createLlmClient } from "../../eval/llm.js";
+import { buildEvalInitPrompt, parseGeneratedEvals } from "../../eval/prompt-builder.js";
+import { green, red, dim, yellow } from "../../utils/output.js";
+export async function runEvalInit(skillDir, force) {
+    const skillMdPath = join(skillDir, "SKILL.md");
+    const evalsDir = join(skillDir, "evals");
+    const evalsPath = join(evalsDir, "evals.json");
+    // Check SKILL.md exists
+    if (!existsSync(skillMdPath)) {
+        console.error(red(`SKILL.md not found at ${skillMdPath}`));
+        return;
+    }
+    // Check existing evals.json
+    if (existsSync(evalsPath) && !force) {
+        console.log(yellow("evals.json already exists, use --force to overwrite"));
+        return;
+    }
+    const skillContent = readFileSync(skillMdPath, "utf-8");
+    const prompt = buildEvalInitPrompt(skillContent);
+    try {
+        const client = createLlmClient();
+        const raw = await client.generate("You generate eval test cases for AI skills. Output only valid JSON in a code fence.", prompt);
+        const evalsFile = parseGeneratedEvals(raw);
+        mkdirSync(evalsDir, { recursive: true });
+        writeFileSync(evalsPath, JSON.stringify(evalsFile, null, 2), "utf-8");
+        console.log(green(`Created ${evalsPath}`));
+        console.log(dim(`  ${evalsFile.evals.length} eval cases, ${evalsFile.evals.reduce((sum, e) => sum + e.assertions.length, 0)} assertions`));
+    }
+    catch (err) {
+        console.error(red("Failed to generate evals: ") + dim(err.message));
+    }
+}
+//# sourceMappingURL=init.js.map

package/dist/commands/eval/init.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"init.js","sourceRoot":"","sources":["../../../src/commands/eval/init.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,gEAAgE;AAChE,8EAA8E;AAE9E,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAC7E,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AACpD,OAAO,EAAE,mBAAmB,EAAE,mBAAmB,EAAE,MAAM,8BAA8B,CAAC;AACxF,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAC;AAEhE,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,QAAgB,EAChB,KAAc;IAEd,MAAM,WAAW,GAAG,IAAI,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;IAC/C,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IACzC,MAAM,SAAS,GAAG,IAAI,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;IAE/C,wBAAwB;IACxB,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC,EAAE,CAAC;QAC7B,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,yBAAyB,WAAW,EAAE,CAAC,CAAC,CAAC;QAC3D,OAAO;IACT,CAAC;IAED,4BAA4B;IAC5B,IAAI,UAAU,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;QACpC,OAAO,CAAC,GAAG,CACT,MAAM,CAAC,qDAAqD,CAAC,CAC9D,CAAC;QACF,OAAO;IACT,CAAC;IAED,MAAM,YAAY,GAAG,YAAY,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC;IACxD,MAAM,MAAM,GAAG,mBAAmB,CAAC,YAAY,CAAC,CAAC;IAEjD,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;QACjC,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,QAAQ,CAC/B,qFAAqF,EACrF,MAAM,CACP,CAAC;QAEF,MAAM,SAAS,GAAG,mBAAmB,CAAC,GAAG,CAAC,CAAC;QAE3C,SAAS,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACzC,aAAa,CAAC,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;QAEtE,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,WAAW,SAAS,EAAE,CAAC,CAAC,CAAC;QAC3C,OAAO,CAAC,GAAG,CACT,GAAG,CAAC,KAAK,SAAS,CAAC,KAAK,CAAC,MAAM,gBAAgB,SAAS,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC,aAAa,CAAC,CAC9H,CAAC;IACJ,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,OAAO,CAAC,KAAK,CACX,GAAG,CAAC,4BAA4B,CAAC,GAAG,GAAG,CAAE,GAAa,CAAC,OAAO,CAAC,CAChE,CAAC;IACJ,CAAC;AACH,CAAC"}

package/dist/commands/eval/run.d.ts

@@ -0,0 +1 @@
+export declare function runEvalRun(skillDir: string): Promise<void>;

package/dist/commands/eval/run.js

@@ -0,0 +1,107 @@
+// ---------------------------------------------------------------------------
+// vskill eval run -- execute eval cases and grade assertions
+// ---------------------------------------------------------------------------
+import { loadAndValidateEvals, EvalValidationError } from "../../eval/schema.js";
+import { createLlmClient } from "../../eval/llm.js";
+import { judgeAssertion } from "../../eval/judge.js";
+import { writeBenchmark } from "../../eval/benchmark.js";
+import { green, red, yellow, bold, dim, table } from "../../utils/output.js";
+export async function runEvalRun(skillDir) {
+    // Load and validate evals.json
+    let evalsFile;
+    try {
+        evalsFile = loadAndValidateEvals(skillDir);
+    }
+    catch (err) {
+        if (err instanceof EvalValidationError) {
+            const firstMsg = err.errors[0]?.message || "";
+            if (firstMsg.includes("No evals.json")) {
+                console.error(red(`No evals.json found at ${skillDir}/evals/evals.json`));
+            }
+            else {
+                console.error(red(`Invalid evals.json: ${err.message}`));
+            }
+        }
+        else {
+            console.error(red(`Error loading evals: ${err.message}`));
+        }
+        process.exit(1);
+        return;
+    }
+    const client = createLlmClient();
+    const model = client.model;
+    const benchmarkCases = [];
+    const tableRows = [];
+    for (const evalCase of evalsFile.evals) {
+        try {
+            // Step 1: Send prompt to LLM
+            const output = await client.generate("You are an AI skill being evaluated. Respond to the prompt as the skill would.", evalCase.prompt);
+            // Step 2: Judge each assertion
+            const assertionResults = [];
+            let passCount = 0;
+            for (const assertion of evalCase.assertions) {
+                const result = await judgeAssertion(output, assertion, client);
+                assertionResults.push(result);
+                if (result.pass)
+                    passCount++;
+                const truncatedText = assertion.text.length > 60
+                    ? assertion.text.slice(0, 57) + "..."
+                    : assertion.text;
+                tableRows.push([
+                    evalCase.name,
+                    assertion.id,
+                    truncatedText,
+                    result.pass ? green("PASS") : red("FAIL"),
+                ]);
+            }
+            const passRate = evalCase.assertions.length > 0
+                ? passCount / evalCase.assertions.length
+                : 0;
+            const allPassed = passCount === evalCase.assertions.length;
+            benchmarkCases.push({
+                eval_id: evalCase.id,
+                eval_name: evalCase.name,
+                status: allPassed ? "pass" : "fail",
+                error_message: null,
+                pass_rate: passRate,
+                assertions: assertionResults,
+            });
+        }
+        catch (err) {
+            // Mark case as error, continue with remaining
+            benchmarkCases.push({
+                eval_id: evalCase.id,
+                eval_name: evalCase.name,
+                status: "error",
+                error_message: err.message,
+                pass_rate: 0,
+                assertions: [],
+            });
+            tableRows.push([
+                evalCase.name,
+                "-",
+                dim("Error: " + err.message.slice(0, 50)),
+                yellow("ERROR"),
+            ]);
+        }
+    }
+    // Print results table
+    const headers = ["EVAL", "ASSERTION", "TEXT", "STATUS"];
+    console.log(bold(`\nEval Results: ${evalsFile.skill_name}\n`));
+    console.log(table(headers, tableRows));
+    // Compute summary
+    const passed = benchmarkCases.filter((c) => c.status === "pass").length;
+    const failed = benchmarkCases.filter((c) => c.status === "fail").length;
+    const errors = benchmarkCases.filter((c) => c.status === "error").length;
+    console.log(`\n${green(`${passed} passed`)} ${failed > 0 ? red(`${failed} failed`) : ""} ${errors > 0 ? yellow(`${errors} errors`) : ""}`.trim());
+    // Write benchmark.json
+    const benchmark = {
+        timestamp: new Date().toISOString(),
+        model,
+        skill_name: evalsFile.skill_name,
+        cases: benchmarkCases,
+    };
+    await writeBenchmark(skillDir, benchmark);
+    console.log(dim(`\nBenchmark written to ${skillDir}/evals/benchmark.json`));
+}
+//# sourceMappingURL=run.js.map

package/dist/commands/eval/run.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"run.js","sourceRoot":"","sources":["../../../src/commands/eval/run.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,6DAA6D;AAC7D,8EAA8E;AAE9E,OAAO,EAAE,oBAAoB,EAAE,mBAAmB,EAAE,MAAM,sBAAsB,CAAC;AACjF,OAAO,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AACpD,OAAO,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AACrD,OAAO,EAAE,cAAc,EAAE,MAAM,yBAAyB,CAAC;AAEzD,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,EAAE,KAAK,EAAE,MAAM,uBAAuB,CAAC;AAE7E,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,QAAgB;IAC/C,+BAA+B;IAC/B,IAAI,SAAS,CAAC;IACd,IAAI,CAAC;QACH,SAAS,GAAG,oBAAoB,CAAC,QAAQ,CAAC,CAAC;IAC7C,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,IAAI,GAAG,YAAY,mBAAmB,EAAE,CAAC;YACvC,MAAM,QAAQ,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,OAAO,IAAI,EAAE,CAAC;YAC9C,IAAI,QAAQ,CAAC,QAAQ,CAAC,eAAe,CAAC,EAAE,CAAC;gBACvC,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,0BAA0B,QAAQ,mBAAmB,CAAC,CAAC,CAAC;YAC5E,CAAC;iBAAM,CAAC;gBACN,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,uBAAuB,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;YAC3D,CAAC;QACH,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,wBAAyB,GAAa,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;QACvE,CAAC;QACD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAChB,OAAO;IACT,CAAC;IAED,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;IACjC,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC;IAC3B,MAAM,cAAc,GAAoB,EAAE,CAAC;IAC3C,MAAM,SAAS,GAAe,EAAE,CAAC;IAEjC,KAAK,MAAM,QAAQ,IAAI,SAAS,CAAC,KAAK,EAAE,CAAC;QACvC,IAAI,CAAC;YACH,6BAA6B;YAC7B,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,QAAQ,CAClC,gFAAgF,EAChF,QAAQ,CAAC,MAAM,CAChB,CAAC;YAEF,+BAA+B;YAC/B,MAAM,gBAAgB,GAAG,EAAE,CAAC;YAC5B,IAAI,SAAS,GAAG,CAAC,CAAC;YAElB,KAAK,MAAM,SAAS,IAAI,QAAQ,CAAC,UAAU,EAAE,CAAC;gBAC5C,MAAM,MAAM,GAAG,MAAM,cAAc,CAAC,MAAM,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC;gBAC/D,gBAAgB,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;gBAC9B,IAAI,MAAM,CAAC,IAAI;oBAAE,SAAS,EAAE,CAAC;gBAE7B,MAAM,aAAa,GACjB,SAAS,CAAC,IAAI,CAAC,MAAM,GAAG,EAAE;oBACxB,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,KAAK;oBACrC,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC;gBAErB,SAAS,CAAC,IAAI,CAAC;oBACb,QAAQ,CAAC,IAAI;oBACb,SAAS,CAAC,EAAE;oBACZ,aAAa;oBACb,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC;iBAC1C,CAAC,CAAC;YACL,CAAC;YAED,MAAM,QAAQ,GAAG,QAAQ,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC;gBAC7C,CAAC,CAAC,SAAS,GAAG,QAAQ,CAAC,UAAU,CAAC,MAAM;gBACxC,CAAC,CAAC,CAAC,CAAC;YACN,MAAM,SAAS,GAAG,SAAS,KAAK,QAAQ,CAAC,UAAU,CAAC,MAAM,CAAC;YAE3D,cAAc,CAAC,IAAI,CAAC;gBAClB,OAAO,EAAE,QAAQ,CAAC,EAAE;gBACpB,SAAS,EAAE,QAAQ,CAAC,IAAI;gBACxB,MAAM,EAAE,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM;gBACnC,aAAa,EAAE,IAAI;gBACnB,SAAS,EAAE,QAAQ;gBACnB,UAAU,EAAE,gBAAgB;aAC7B,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,8CAA8C;YAC9C,cAAc,CAAC,IAAI,CAAC;gBAClB,OAAO,EAAE,QAAQ,CAAC,EAAE;gBACpB,SAAS,EAAE,QAAQ,CAAC,IAAI;gBACxB,MAAM,EAAE,OAAO;gBACf,aAAa,EAAG,GAAa,CAAC,OAAO;gBACrC,SAAS,EAAE,CAAC;gBACZ,UAAU,EAAE,EAAE;aACf,CAAC,CAAC;YAEH,SAAS,CAAC,IAAI,CAAC;gBACb,QAAQ,CAAC,IAAI;gBACb,GAAG;gBACH,GAAG,CAAC,SAAS,GAAI,GAAa,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;gBACpD,MAAM,CAAC,OAAO,CAAC;aAChB,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,sBAAsB;IACtB,MAAM,OAAO,GAAG,CAAC,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC;IACxD,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,mBAAmB,SAAS,CAAC,UAAU,IAAI,CAAC,CAAC,CAAC;IAC/D,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC,CAAC;IAEvC,kBAAkB;IAClB,MAAM,MAAM,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC,MAAM,CAAC;IACxE,MAAM,MAAM,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC,MAAM,CAAC;IACxE,MAAM,MAAM,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,OAAO,CAAC,CAAC,MAAM,CAAC;IACzE,OAAO,CAAC,GAAG,CACT,KAAK,KAAK,CAAC,GAAG,MAAM,SAAS,CAAC,IAAI,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,MAAM,SAAS,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,MAAM,SAAS,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,EAAE,CACrI,CAAC;IAEF,uBAAuB;IACvB,MAAM,SAAS,GAAoB;QACjC,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACnC,KAAK;QACL,UAAU,EAAE,SAAS,CAAC,UAAU;QAChC,KAAK,EAAE,cAAc;KACtB,CAAC;IAEF,MAAM,cAAc,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;IAC1C,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,0BAA0B,QAAQ,uBAAuB,CAAC,CAAC,CAAC;AAC9E,CAAC"}