vskill 0.2.26 → 0.2.28

package/dist/eval/__tests__/skill-scanner.test.js

@@ -0,0 +1,78 @@
+import { describe, it, expect, beforeEach, afterEach } from "vitest";
+import { mkdirSync, writeFileSync, rmSync } from "node:fs";
+import { join } from "node:path";
+import { tmpdir } from "node:os";
+import { scanSkills } from "../skill-scanner.js";
+// ---------------------------------------------------------------------------
+// Test helpers
+// ---------------------------------------------------------------------------
+let testDir;
+function createSkill(plugin, skill, opts = {}) {
+    const skillDir = join(testDir, plugin, "skills", skill);
+    mkdirSync(skillDir, { recursive: true });
+    writeFileSync(join(skillDir, "SKILL.md"), `# ${skill}`);
+    if (opts.evals) {
+        const evalsDir = join(skillDir, "evals");
+        mkdirSync(evalsDir, { recursive: true });
+        writeFileSync(join(evalsDir, "evals.json"), JSON.stringify({ skill_name: skill, evals: [] }));
+    }
+    if (opts.benchmark) {
+        const evalsDir = join(skillDir, "evals");
+        mkdirSync(evalsDir, { recursive: true });
+        writeFileSync(join(evalsDir, "benchmark.json"), JSON.stringify({ timestamp: "2026-03-01T00:00:00Z" }));
+    }
+}
+// ---------------------------------------------------------------------------
+// Tests
+// ---------------------------------------------------------------------------
+describe("scanSkills", () => {
+    beforeEach(() => {
+        testDir = join(tmpdir(), `vskill-test-${Date.now()}`);
+        mkdirSync(testDir, { recursive: true });
+    });
+    afterEach(() => {
+        rmSync(testDir, { recursive: true, force: true });
+    });
+    it("discovers skills in plugins directory", async () => {
+        createSkill("marketing", "social-media-posting");
+        createSkill("devtools", "code-review");
+        const skills = await scanSkills(testDir);
+        expect(skills).toHaveLength(2);
+        const names = skills.map((s) => s.skill).sort();
+        expect(names).toEqual(["code-review", "social-media-posting"]);
+    });
+    it("sets hasEvals=true when evals.json exists", async () => {
+        createSkill("marketing", "social-media-posting", { evals: true });
+        const skills = await scanSkills(testDir);
+        expect(skills[0].hasEvals).toBe(true);
+    });
+    it("sets hasEvals=false when evals.json is absent", async () => {
+        createSkill("marketing", "social-media-posting");
+        const skills = await scanSkills(testDir);
+        expect(skills[0].hasEvals).toBe(false);
+    });
+    it("sets hasBenchmark=true when benchmark.json exists", async () => {
+        createSkill("marketing", "social-media-posting", {
+            evals: true,
+            benchmark: true,
+        });
+        const skills = await scanSkills(testDir);
+        expect(skills[0].hasBenchmark).toBe(true);
+    });
+    it("sets hasBenchmark=false when benchmark.json is absent", async () => {
+        createSkill("marketing", "social-media-posting", { evals: true });
+        const skills = await scanSkills(testDir);
+        expect(skills[0].hasBenchmark).toBe(false);
+    });
+    it("returns correct plugin and skill names", async () => {
+        createSkill("marketing", "social-media-posting");
+        const skills = await scanSkills(testDir);
+        expect(skills[0].plugin).toBe("marketing");
+        expect(skills[0].skill).toBe("social-media-posting");
+    });
+    it("returns empty array for empty root", async () => {
+        const skills = await scanSkills(testDir);
+        expect(skills).toEqual([]);
+    });
+});
+//# sourceMappingURL=skill-scanner.test.js.map

package/dist/eval/__tests__/skill-scanner.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"skill-scanner.test.js","sourceRoot":"","sources":["../../../src/eval/__tests__/skill-scanner.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AACrE,OAAO,EAAE,SAAS,EAAE,aAAa,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AAC3D,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AACjC,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAEjD,8EAA8E;AAC9E,eAAe;AACf,8EAA8E;AAE9E,IAAI,OAAe,CAAC;AAEpB,SAAS,WAAW,CAClB,MAAc,EACd,KAAa,EACb,OAAiD,EAAE;IAEnD,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC;IACxD,SAAS,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACzC,aAAa,CAAC,IAAI,CAAC,QAAQ,EAAE,UAAU,CAAC,EAAE,KAAK,KAAK,EAAE,CAAC,CAAC;IAExD,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;QACf,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QACzC,SAAS,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACzC,aAAa,CACX,IAAI,CAAC,QAAQ,EAAE,YAAY,CAAC,EAC5B,IAAI,CAAC,SAAS,CAAC,EAAE,UAAU,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,CACjD,CAAC;IACJ,CAAC;IAED,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;QACnB,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QACzC,SAAS,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACzC,aAAa,CACX,IAAI,CAAC,QAAQ,EAAE,gBAAgB,CAAC,EAChC,IAAI,CAAC,SAAS,CAAC,EAAE,SAAS,EAAE,sBAAsB,EAAE,CAAC,CACtD,CAAC;IACJ,CAAC;AACH,CAAC;AAED,8EAA8E;AAC9E,QAAQ;AACR,8EAA8E;AAE9E,QAAQ,CAAC,YAAY,EAAE,GAAG,EAAE;IAC1B,UAAU,CAAC,GAAG,EAAE;QACd,OAAO,GAAG,IAAI,CAAC,MAAM,EAAE,EAAE,eAAe,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;QACtD,SAAS,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC1C,CAAC,CAAC,CAAC;IAEH,SAAS,CAAC,GAAG,EAAE;QACb,MAAM,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IACpD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,uCAAuC,EAAE,KAAK,IAAI,EAAE;QACrD,WAAW,CAAC,WAAW,EAAE,sBAAsB,CAAC,CAAC;QACjD,WAAW,CAAC,UAAU,EAAE,aAAa,CAAC,CAAC;QAEvC,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,OAAO,CAAC,CAAC;QAEzC,MAAM,CAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAC/B,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC;QAChD,MAAM,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,CAAC,aAAa,EAAE,sBAAsB,CAAC,CAAC,CAAC;IACjE,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,2CAA2C,EAAE,KAAK,IAAI,EAAE;QACzD,WAAW,CAAC,WAAW,EAAE,sBAAsB,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QAElE,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,OAAO,CAAC,CAAC;QAEzC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACxC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,+CAA+C,EAAE,KAAK,IAAI,EAAE;QAC7D,WAAW,CAAC,WAAW,EAAE,sBAAsB,CAAC,CAAC;QAEjD,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,OAAO,CAAC,CAAC;QAEzC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACzC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,mDAAmD,EAAE,KAAK,IAAI,EAAE;QACjE,WAAW,CAAC,WAAW,EAAE,sBAAsB,EAAE;YAC/C,KAAK,EAAE,IAAI;YACX,SAAS,EAAE,IAAI;SAChB,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,OAAO,CAAC,CAAC;QAEzC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC5C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,uDAAuD,EAAE,KAAK,IAAI,EAAE;QACrE,WAAW,CAAC,WAAW,EAAE,sBAAsB,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QAElE,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,OAAO,CAAC,CAAC;QAEzC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC7C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,wCAAwC,EAAE,KAAK,IAAI,EAAE;QACtD,WAAW,CAAC,WAAW,EAAE,sBAAsB,CAAC,CAAC;QAEjD,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,OAAO,CAAC,CAAC;QAEzC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QAC3C,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,sBAAsB,CAAC,CAAC;IACvD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,oCAAoC,EAAE,KAAK,IAAI,EAAE;QAClD,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,OAAO,CAAC,CAAC;QACzC,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IAC7B,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

package/dist/eval/benchmark.d.ts

@@ -0,0 +1,22 @@
+export interface BenchmarkAssertionResult {
+    id: string;
+    text: string;
+    pass: boolean;
+    reasoning: string;
+}
+export interface BenchmarkCase {
+    eval_id: number;
+    eval_name: string;
+    status: "pass" | "fail" | "error";
+    error_message: string | null;
+    pass_rate: number;
+    assertions: BenchmarkAssertionResult[];
+}
+export interface BenchmarkResult {
+    timestamp: string;
+    model: string;
+    skill_name: string;
+    cases: BenchmarkCase[];
+}
+export declare function writeBenchmark(skillDir: string, result: BenchmarkResult): Promise<void>;
+export declare function readBenchmark(skillDir: string): Promise<BenchmarkResult | null>;

package/dist/eval/benchmark.js

@@ -0,0 +1,24 @@
+// ---------------------------------------------------------------------------
+// benchmark.json read/write
+// ---------------------------------------------------------------------------
+import { writeFileSync, readFileSync, existsSync, mkdirSync } from "node:fs";
+import { join } from "node:path";
+export async function writeBenchmark(skillDir, result) {
+    const evalsDir = join(skillDir, "evals");
+    mkdirSync(evalsDir, { recursive: true });
+    const filePath = join(evalsDir, "benchmark.json");
+    writeFileSync(filePath, JSON.stringify(result, null, 2), "utf-8");
+}
+export async function readBenchmark(skillDir) {
+    const filePath = join(skillDir, "evals", "benchmark.json");
+    if (!existsSync(filePath))
+        return null;
+    try {
+        const raw = readFileSync(filePath, "utf-8");
+        return JSON.parse(raw);
+    }
+    catch {
+        return null;
+    }
+}
+//# sourceMappingURL=benchmark.js.map

package/dist/eval/benchmark.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"benchmark.js","sourceRoot":"","sources":["../../src/eval/benchmark.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,4BAA4B;AAC5B,8EAA8E;AAE9E,OAAO,EAAE,aAAa,EAAE,YAAY,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAC7E,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAyBjC,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,QAAgB,EAChB,MAAuB;IAEvB,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IACzC,SAAS,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACzC,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,EAAE,gBAAgB,CAAC,CAAC;IAClD,aAAa,CAAC,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;AACpE,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,QAAgB;IAEhB,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,EAAE,OAAO,EAAE,gBAAgB,CAAC,CAAC;IAC3D,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC;QAAE,OAAO,IAAI,CAAC;IAEvC,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAC5C,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAoB,CAAC;IAC5C,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC"}

package/dist/eval/judge.d.ts

@@ -0,0 +1,9 @@
+import type { Assertion } from "./schema.js";
+import type { LlmClient } from "./llm.js";
+export interface AssertionResult {
+    id: string;
+    text: string;
+    pass: boolean;
+    reasoning: string;
+}
+export declare function judgeAssertion(output: string, assertion: Assertion, client: LlmClient): Promise<AssertionResult>;

package/dist/eval/judge.js

@@ -0,0 +1,40 @@
+// ---------------------------------------------------------------------------
+// Assertion-level LLM judge
+// ---------------------------------------------------------------------------
+const JUDGE_SYSTEM = `You are a binary assertion evaluator. Given an LLM output and an assertion, determine if the output satisfies the assertion. Respond with ONLY a JSON object: { "pass": boolean, "reasoning": "brief explanation" }`;
+export async function judgeAssertion(output, assertion, client) {
+    const userPrompt = `## LLM Output
+${output}
+
+## Assertion to Verify
+${assertion.text}
+
+Does the LLM output satisfy this assertion? Respond with JSON only: { "pass": boolean, "reasoning": "..." }`;
+    const raw = await client.generate(JUDGE_SYSTEM, userPrompt);
+    const parsed = parseJudgeResponse(raw);
+    return {
+        id: assertion.id,
+        text: assertion.text,
+        pass: parsed.pass,
+        reasoning: parsed.reasoning,
+    };
+}
+function parseJudgeResponse(raw) {
+    // Try to extract JSON from code fence first
+    const fenceMatch = raw.match(/```(?:json)?\s*\n([\s\S]*?)\n```/);
+    const jsonStr = fenceMatch ? fenceMatch[1] : raw;
+    try {
+        const parsed = JSON.parse(jsonStr);
+        if (typeof parsed.pass !== "boolean") {
+            throw new Error("missing pass field");
+        }
+        return {
+            pass: parsed.pass,
+            reasoning: typeof parsed.reasoning === "string" ? parsed.reasoning : "",
+        };
+    }
+    catch {
+        throw new Error(`Invalid judge output: expected JSON with { pass, reasoning }, got: ${raw.slice(0, 100)}`);
+    }
+}
+//# sourceMappingURL=judge.js.map

package/dist/eval/judge.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"judge.js","sourceRoot":"","sources":["../../src/eval/judge.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,4BAA4B;AAC5B,8EAA8E;AAY9E,MAAM,YAAY,GAAG,qNAAqN,CAAC;AAE3O,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,MAAc,EACd,SAAoB,EACpB,MAAiB;IAEjB,MAAM,UAAU,GAAG;EACnB,MAAM;;;EAGN,SAAS,CAAC,IAAI;;4GAE4F,CAAC;IAE3G,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC,YAAY,EAAE,UAAU,CAAC,CAAC;IAE5D,MAAM,MAAM,GAAG,kBAAkB,CAAC,GAAG,CAAC,CAAC;IAEvC,OAAO;QACL,EAAE,EAAE,SAAS,CAAC,EAAE;QAChB,IAAI,EAAE,SAAS,CAAC,IAAI;QACpB,IAAI,EAAE,MAAM,CAAC,IAAI;QACjB,SAAS,EAAE,MAAM,CAAC,SAAS;KAC5B,CAAC;AACJ,CAAC;AAED,SAAS,kBAAkB,CAAC,GAAW;IACrC,4CAA4C;IAC5C,MAAM,UAAU,GAAG,GAAG,CAAC,KAAK,CAAC,kCAAkC,CAAC,CAAC;IACjE,MAAM,OAAO,GAAG,UAAU,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;IAEjD,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QACnC,IAAI,OAAO,MAAM,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;YACrC,MAAM,IAAI,KAAK,CAAC,oBAAoB,CAAC,CAAC;QACxC,CAAC;QACD,OAAO;YACL,IAAI,EAAE,MAAM,CAAC,IAAI;YACjB,SAAS,EAAE,OAAO,MAAM,CAAC,SAAS,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE;SACxE,CAAC;IACJ,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,KAAK,CACb,sEAAsE,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAC1F,CAAC;IACJ,CAAC;AACH,CAAC"}

package/dist/eval/llm.d.ts

@@ -0,0 +1,5 @@
+export interface LlmClient {
+    generate(systemPrompt: string, userPrompt: string): Promise<string>;
+    readonly model: string;
+}
+export declare function createLlmClient(): LlmClient;

package/dist/eval/llm.js

@@ -0,0 +1,34 @@
+// ---------------------------------------------------------------------------
+// Anthropic LLM client wrapper for eval commands
+// ---------------------------------------------------------------------------
+import Anthropic from "@anthropic-ai/sdk";
+const DEFAULT_MODEL = "claude-sonnet-4-20250514";
+export function createLlmClient() {
+    const apiKey = process.env.ANTHROPIC_API_KEY;
+    if (!apiKey) {
+        throw new Error("ANTHROPIC_API_KEY is not set. Export it before running eval commands:\n  export ANTHROPIC_API_KEY=sk-ant-...");
+    }
+    const client = new Anthropic({ apiKey });
+    const model = process.env.VSKILL_EVAL_MODEL || DEFAULT_MODEL;
+    return {
+        model,
+        async generate(systemPrompt, userPrompt) {
+            const controller = new AbortController();
+            const timeout = setTimeout(() => controller.abort(), 120_000);
+            try {
+                const response = await client.messages.create({
+                    model,
+                    system: systemPrompt,
+                    messages: [{ role: "user", content: userPrompt }],
+                    max_tokens: 4096,
+                }, { signal: controller.signal });
+                const textBlock = response.content.find((b) => b.type === "text");
+                return textBlock && "text" in textBlock ? textBlock.text : "";
+            }
+            finally {
+                clearTimeout(timeout);
+            }
+        },
+    };
+}
+//# sourceMappingURL=llm.js.map

package/dist/eval/llm.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"llm.js","sourceRoot":"","sources":["../../src/eval/llm.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,iDAAiD;AACjD,8EAA8E;AAE9E,OAAO,SAAS,MAAM,mBAAmB,CAAC;AAE1C,MAAM,aAAa,GAAG,0BAA0B,CAAC;AAOjD,MAAM,UAAU,eAAe;IAC7B,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC;IAC7C,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,MAAM,IAAI,KAAK,CACb,8GAA8G,CAC/G,CAAC;IACJ,CAAC;IAED,MAAM,MAAM,GAAG,IAAI,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;IACzC,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,aAAa,CAAC;IAE7D,OAAO;QACL,KAAK;QACL,KAAK,CAAC,QAAQ,CAAC,YAAoB,EAAE,UAAkB;YACrD,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;YACzC,MAAM,OAAO,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,OAAO,CAAC,CAAC;YAC9D,IAAI,CAAC;gBACH,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC,MAAM,CAC3C;oBACE,KAAK;oBACL,MAAM,EAAE,YAAY;oBACpB,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,CAAC;oBACjD,UAAU,EAAE,IAAI;iBACjB,EACD,EAAE,MAAM,EAAE,UAAU,CAAC,MAAM,EAAE,CAC9B,CAAC;gBAEF,MAAM,SAAS,GAAG,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC;gBAClE,OAAO,SAAS,IAAI,MAAM,IAAI,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;YAChE,CAAC;oBAAS,CAAC;gBACT,YAAY,CAAC,OAAO,CAAC,CAAC;YACxB,CAAC;QACH,CAAC;KACF,CAAC;AACJ,CAAC"}

package/dist/eval/prompt-builder.d.ts

@@ -0,0 +1,3 @@
+import type { EvalsFile } from "./schema.js";
+export declare function buildEvalInitPrompt(skillContent: string): string;
+export declare function parseGeneratedEvals(raw: string): EvalsFile;

package/dist/eval/prompt-builder.js

@@ -0,0 +1,155 @@
+// ---------------------------------------------------------------------------
+// Eval generation prompt assembly + response parsing
+// ---------------------------------------------------------------------------
+// ---------------------------------------------------------------------------
+// Schema reference (embedded)
+// ---------------------------------------------------------------------------
+const SCHEMA_REFERENCE = `
+## evals.json Schema
+
+The file MUST be valid JSON with this structure:
+
+{
+  "skill_name": "<string, required - the skill identifier>",
+  "evals": [
+    {
+      "id": <number, required - unique integer per eval case>,
+      "name": "<string, required - descriptive name for this test case>",
+      "prompt": "<string, required - the user prompt to send to the LLM>",
+      "expected_output": "<string, required - description of what correct output looks like>",
+      "files": ["<optional array of file paths relevant to this eval>"],
+      "assertions": [
+        {
+          "id": "<string, required - unique within this eval case, e.g. 'assert-1'>",
+          "text": "<string, required - the assertion to verify against the output>",
+          "type": "boolean"
+        }
+      ]
+    }
+  ]
+}
+
+Every eval case MUST have at least 1 assertion. Assertion IDs must be unique within each eval case.
+`;
+// ---------------------------------------------------------------------------
+// Example (embedded from social-media-posting)
+// ---------------------------------------------------------------------------
+const EXAMPLE_EVALS = `
+## Example: social-media-posting evals.json
+
+{
+  "skill_name": "social-media-posting",
+  "evals": [
+    {
+      "id": 1,
+      "name": "LinkedIn announcement post",
+      "prompt": "Write a LinkedIn post announcing our new AI-powered analytics dashboard. Target audience: B2B SaaS founders. Tone: professional but excited. Include a call to action to sign up for the beta at analytics.example.com/beta",
+      "expected_output": "A professional LinkedIn post with product announcement, value proposition, and CTA",
+      "files": [],
+      "assertions": [
+        { "id": "assert-1", "text": "Post mentions the AI-powered analytics dashboard by name", "type": "boolean" },
+        { "id": "assert-2", "text": "Post includes the beta signup URL analytics.example.com/beta", "type": "boolean" },
+        { "id": "assert-3", "text": "Post uses a professional tone appropriate for B2B SaaS audience", "type": "boolean" },
+        { "id": "assert-4", "text": "Post includes a clear call to action", "type": "boolean" }
+      ]
+    }
+  ]
+}
+`;
+// ---------------------------------------------------------------------------
+// Best practices (embedded)
+// ---------------------------------------------------------------------------
+const BEST_PRACTICES = `
+## Best Practices for Eval Generation
+
+1. **Realistic prompts with substantive detail**: Include specific details like file paths, column names, audience types, or configuration values. Avoid generic prompts like "do something with this skill."
+
+2. **Objectively verifiable assertions**: Each assertion should be checkable by an LLM judge with a clear yes/no answer. "The output mentions X" is verifiable. "The output is good" is not.
+
+3. **Descriptive assertion names/IDs**: Assertion IDs should read clearly in a benchmark viewer. Use descriptive IDs like "mentions-file-path" or "includes-cta-url".
+
+4. **Skip assertions for purely subjective qualities**: Don't assert on tone, creativity, or style unless there's an objective proxy (e.g., "uses formal language" instead of "sounds professional").
+
+5. **Generate 2-3 test cases**: Each representing a different realistic usage scenario for this skill. Cover the primary use case and at least one edge case or variation.
+`;
+// ---------------------------------------------------------------------------
+// Prompt builder
+// ---------------------------------------------------------------------------
+export function buildEvalInitPrompt(skillContent) {
+    return `You are an expert eval generator for AI skills. Your task is to create an evals.json file for the skill described below.
+
+## Skill Content (SKILL.md)
+
+${skillContent}
+
+${SCHEMA_REFERENCE}
+
+${EXAMPLE_EVALS}
+
+${BEST_PRACTICES}
+
+## Instructions
+
+Generate a complete evals.json for this skill. Output ONLY the JSON inside a \`\`\`json code fence. Generate 2-3 eval cases with realistic, specific prompts and objectively verifiable assertions. Each case must have at least 2 assertions.`;
+}
+// ---------------------------------------------------------------------------
+// Response parser
+// ---------------------------------------------------------------------------
+export function parseGeneratedEvals(raw) {
+    // Extract JSON from ```json ... ``` code fence
+    const match = raw.match(/```(?:json)?\s*\n([\s\S]*?)\n```/);
+    if (!match) {
+        throw new Error("No JSON code block found in LLM response. Expected ```json ... ``` fence.");
+    }
+    const jsonStr = match[1];
+    let parsed;
+    try {
+        parsed = JSON.parse(jsonStr);
+    }
+    catch (err) {
+        throw new Error(`Invalid JSON in code block: ${err.message}`);
+    }
+    // Validate against schema
+    validateParsedEvals(parsed);
+    // Normalize
+    return {
+        skill_name: parsed.skill_name,
+        evals: parsed.evals.map((e) => ({
+            id: e.id,
+            name: e.name,
+            prompt: e.prompt,
+            expected_output: e.expected_output,
+            files: Array.isArray(e.files) ? e.files : [],
+            assertions: e.assertions.map((a) => ({
+                id: a.id,
+                text: a.text,
+                type: a.type || "boolean",
+            })),
+        })),
+    };
+}
+function validateParsedEvals(parsed) {
+    const errors = [];
+    if (typeof parsed.skill_name !== "string" || !parsed.skill_name) {
+        errors.push("missing skill_name");
+    }
+    if (!Array.isArray(parsed.evals) || parsed.evals.length === 0) {
+        errors.push("missing or empty evals array");
+    }
+    if (Array.isArray(parsed.evals)) {
+        for (let i = 0; i < parsed.evals.length; i++) {
+            const e = parsed.evals[i];
+            if (!e.prompt)
+                errors.push(`evals[${i}] missing prompt`);
+            if (!e.name)
+                errors.push(`evals[${i}] missing name`);
+            if (!Array.isArray(e.assertions) || e.assertions.length === 0) {
+                errors.push(`evals[${i}] missing or empty assertions`);
+            }
+        }
+    }
+    if (errors.length > 0) {
+        throw new Error(`Invalid evals structure: ${errors.join(", ")}`);
+    }
+}
+//# sourceMappingURL=prompt-builder.js.map

package/dist/eval/prompt-builder.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"prompt-builder.js","sourceRoot":"","sources":["../../src/eval/prompt-builder.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,qDAAqD;AACrD,8EAA8E;AAI9E,8EAA8E;AAC9E,8BAA8B;AAC9B,8EAA8E;AAE9E,MAAM,gBAAgB,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;CA0BxB,CAAC;AAEF,8EAA8E;AAC9E,+CAA+C;AAC/C,8EAA8E;AAE9E,MAAM,aAAa,GAAG;;;;;;;;;;;;;;;;;;;;;CAqBrB,CAAC;AAEF,8EAA8E;AAC9E,4BAA4B;AAC5B,8EAA8E;AAE9E,MAAM,cAAc,GAAG;;;;;;;;;;;;CAYtB,CAAC;AAEF,8EAA8E;AAC9E,iBAAiB;AACjB,8EAA8E;AAE9E,MAAM,UAAU,mBAAmB,CAAC,YAAoB;IACtD,OAAO;;;;EAIP,YAAY;;EAEZ,gBAAgB;;EAEhB,aAAa;;EAEb,cAAc;;;;+OAI+N,CAAC;AAChP,CAAC;AAED,8EAA8E;AAC9E,kBAAkB;AAClB,8EAA8E;AAE9E,MAAM,UAAU,mBAAmB,CAAC,GAAW;IAC7C,+CAA+C;IAC/C,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,kCAAkC,CAAC,CAAC;IAC5D,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,MAAM,IAAI,KAAK,CACb,2EAA2E,CAC5E,CAAC;IACJ,CAAC;IAED,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;IACzB,IAAI,MAAW,CAAC;IAChB,IAAI,CAAC;QACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IAC/B,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,IAAI,KAAK,CACb,+BAAgC,GAAa,CAAC,OAAO,EAAE,CACxD,CAAC;IACJ,CAAC;IAED,0BAA0B;IAC1B,mBAAmB,CAAC,MAAM,CAAC,CAAC;IAE5B,YAAY;IACZ,OAAO;QACL,UAAU,EAAE,MAAM,CAAC,UAAU;QAC7B,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC;YACnC,EAAE,EAAE,CAAC,CAAC,EAAE;YACR,IAAI,EAAE,CAAC,CAAC,IAAI;YACZ,MAAM,EAAE,CAAC,CAAC,MAAM;YAChB,eAAe,EAAE,CAAC,CAAC,eAAe;YAClC,KAAK,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE;YAC5C,UAAU,EAAE,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC;gBACxC,EAAE,EAAE,CAAC,CAAC,EAAE;gBACR,IAAI,EAAE,CAAC,CAAC,IAAI;gBACZ,IAAI,EAAE,CAAC,CAAC,IAAI,IAAI,SAAS;aAC1B,CAAC,CAAC;SACJ,CAAC,CAAC;KACJ,CAAC;AACJ,CAAC;AAED,SAAS,mBAAmB,CAAC,MAAW;IACtC,MAAM,MAAM,GAAa,EAAE,CAAC;IAE5B,IAAI,OAAO,MAAM,CAAC,UAAU,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,CAAC;QAChE,MAAM,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;IACpC,CAAC;IACD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,MAAM,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC9D,MAAM,CAAC,IAAI,CAAC,8BAA8B,CAAC,CAAC;IAC9C,CAAC;IAED,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;QAChC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC7C,MAAM,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;YAC1B,IAAI,CAAC,CAAC,CAAC,MAAM;gBAAE,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,kBAAkB,CAAC,CAAC;YACzD,IAAI,CAAC,CAAC,CAAC,IAAI;gBAAE,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,gBAAgB,CAAC,CAAC;YACrD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAC9D,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,+BAA+B,CAAC,CAAC;YACzD,CAAC;QACH,CAAC;IACH,CAAC;IAED,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACtB,MAAM,IAAI,KAAK,CAAC,4BAA4B,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACnE,CAAC;AACH,CAAC"}

package/dist/eval/schema.d.ts

@@ -0,0 +1,26 @@
+export interface Assertion {
+    id: string;
+    text: string;
+    type: "boolean";
+}
+export interface EvalCase {
+    id: number;
+    name: string;
+    prompt: string;
+    expected_output: string;
+    files: string[];
+    assertions: Assertion[];
+}
+export interface EvalsFile {
+    skill_name: string;
+    evals: EvalCase[];
+}
+export interface ValidationError {
+    path: string;
+    message: string;
+}
+export declare class EvalValidationError extends Error {
+    errors: ValidationError[];
+    constructor(errors: ValidationError[]);
+}
+export declare function loadAndValidateEvals(skillDir: string): EvalsFile;

package/dist/eval/schema.js

@@ -0,0 +1,128 @@
+// ---------------------------------------------------------------------------
+// evals.json schema validation
+// ---------------------------------------------------------------------------
+import { readFileSync, existsSync } from "node:fs";
+import { join } from "node:path";
+export class EvalValidationError extends Error {
+    errors;
+    constructor(errors) {
+        const msg = errors.map((e) => `${e.path}: ${e.message}`).join("; ");
+        super(`Eval validation failed: ${msg}`);
+        this.name = "EvalValidationError";
+        this.errors = errors;
+    }
+}
+// ---------------------------------------------------------------------------
+// Validator
+// ---------------------------------------------------------------------------
+export function loadAndValidateEvals(skillDir) {
+    const filePath = join(skillDir, "evals", "evals.json");
+    if (!existsSync(filePath)) {
+        throw new EvalValidationError([
+            { path: filePath, message: "No evals.json found" },
+        ]);
+    }
+    const raw = readFileSync(filePath, "utf-8");
+    let parsed;
+    try {
+        parsed = JSON.parse(raw);
+    }
+    catch (err) {
+        const syntaxErr = err;
+        throw new EvalValidationError([
+            {
+                path: filePath,
+                message: `JSON parse error: ${syntaxErr.message}`,
+            },
+        ]);
+    }
+    const errors = [];
+    if (typeof parsed.skill_name !== "string" || !parsed.skill_name) {
+        errors.push({ path: "skill_name", message: "required string field" });
+    }
+    if (!Array.isArray(parsed.evals)) {
+        errors.push({ path: "evals", message: "required array field" });
+        throw new EvalValidationError(errors);
+    }
+    for (let i = 0; i < parsed.evals.length; i++) {
+        const evalCase = parsed.evals[i];
+        const prefix = `evals[${i}]`;
+        if (typeof evalCase.id !== "number") {
+            errors.push({ path: `${prefix}.id`, message: "required number field" });
+        }
+        if (typeof evalCase.name !== "string" || !evalCase.name) {
+            errors.push({
+                path: `${prefix}.name`,
+                message: "required string field",
+            });
+        }
+        if (typeof evalCase.prompt !== "string" || !evalCase.prompt) {
+            errors.push({
+                path: `${prefix}.prompt`,
+                message: "required string field",
+            });
+        }
+        if (typeof evalCase.expected_output !== "string" ||
+            !evalCase.expected_output) {
+            errors.push({
+                path: `${prefix}.expected_output`,
+                message: "required string field",
+            });
+        }
+        if (!Array.isArray(evalCase.assertions)) {
+            errors.push({
+                path: `${prefix}.assertions`,
+                message: "required array field",
+            });
+            continue;
+        }
+        if (evalCase.assertions.length === 0) {
+            errors.push({
+                path: `${prefix}.assertions`,
+                message: "must have at least 1 assertion",
+            });
+            continue;
+        }
+        // Check for duplicate assertion IDs
+        const seenIds = new Set();
+        for (const assertion of evalCase.assertions) {
+            if (typeof assertion.id !== "string" || !assertion.id) {
+                errors.push({
+                    path: `${prefix}.assertions[].id`,
+                    message: "required string field",
+                });
+            }
+            if (typeof assertion.text !== "string" || !assertion.text) {
+                errors.push({
+                    path: `${prefix}.assertions[].text`,
+                    message: "required string field",
+                });
+            }
+            if (assertion.id && seenIds.has(assertion.id)) {
+                errors.push({
+                    path: `${prefix}.assertions`,
+                    message: `duplicate assertion ID: ${assertion.id}`,
+                });
+            }
+            seenIds.add(assertion.id);
+        }
+    }
+    if (errors.length > 0) {
+        throw new EvalValidationError(errors);
+    }
+    // Normalize: default files to []
+    const evals = parsed.evals.map((e) => ({
+        id: e.id,
+        name: e.name,
+        prompt: e.prompt,
+        expected_output: e.expected_output,
+        files: Array.isArray(e.files) ? e.files : [],
+        assertions: e.assertions.map((a) => ({
+            id: a.id,
+            text: a.text,
+            type: a.type || "boolean",
+        })),
+    }));
+    return { skill_name: parsed.skill_name, evals };
+}
+//# sourceMappingURL=schema.js.map

package/dist/eval/schema.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"schema.js","sourceRoot":"","sources":["../../src/eval/schema.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,+BAA+B;AAC/B,8EAA8E;AAE9E,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AACnD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AA+BjC,MAAM,OAAO,mBAAoB,SAAQ,KAAK;IAC5C,MAAM,CAAoB;IAE1B,YAAY,MAAyB;QACnC,MAAM,GAAG,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACpE,KAAK,CAAC,2BAA2B,GAAG,EAAE,CAAC,CAAC;QACxC,IAAI,CAAC,IAAI,GAAG,qBAAqB,CAAC;QAClC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;IACvB,CAAC;CACF;AAED,8EAA8E;AAC9E,YAAY;AACZ,8EAA8E;AAE9E,MAAM,UAAU,oBAAoB,CAAC,QAAgB;IACnD,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,EAAE,OAAO,EAAE,YAAY,CAAC,CAAC;IAEvD,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC1B,MAAM,IAAI,mBAAmB,CAAC;YAC5B,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,qBAAqB,EAAE;SACnD,CAAC,CAAC;IACL,CAAC;IAED,MAAM,GAAG,GAAG,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAE5C,IAAI,MAAW,CAAC;IAChB,IAAI,CAAC;QACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAC3B,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,SAAS,GAAG,GAAkB,CAAC;QACrC,MAAM,IAAI,mBAAmB,CAAC;YAC5B;gBACE,IAAI,EAAE,QAAQ;gBACd,OAAO,EAAE,qBAAqB,SAAS,CAAC,OAAO,EAAE;aAClD;SACF,CAAC,CAAC;IACL,CAAC;IAED,MAAM,MAAM,GAAsB,EAAE,CAAC;IAErC,IAAI,OAAO,MAAM,CAAC,UAAU,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,CAAC;QAChE,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,YAAY,EAAE,OAAO,EAAE,uBAAuB,EAAE,CAAC,CAAC;IACxE,CAAC;IAED,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;QACjC,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,sBAAsB,EAAE,CAAC,CAAC;QAChE,MAAM,IAAI,mBAAmB,CAAC,MAAM,CAAC,CAAC;IACxC,CAAC;IAED,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC7C,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QACjC,MAAM,MAAM,GAAG,SAAS,CAAC,GAAG,CAAC;QAE7B,IAAI,OAAO,QAAQ,CAAC,EAAE,KAAK,QAAQ,EAAE,CAAC;YACpC,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,GAAG,MAAM,KAAK,EAAE,OAAO,EAAE,uBAAuB,EAAE,CAAC,CAAC;QAC1E,CAAC;QACD,IAAI,OAAO,QAAQ,CAAC,IAAI,KAAK,QAAQ,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;YACxD,MAAM,CAAC,IAAI,CAAC;gBACV,IAAI,EAAE,GAAG,MAAM,OAAO;gBACtB,OAAO,EAAE,uBAAuB;aACjC,CAAC,CAAC;QACL,CAAC;QACD,IAAI,OAAO,QAAQ,CAAC,MAAM,KAAK,QAAQ,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC;YAC5D,MAAM,CAAC,IAAI,CAAC;gBACV,IAAI,EAAE,GAAG,MAAM,SAAS;gBACxB,OAAO,EAAE,uBAAuB;aACjC,CAAC,CAAC;QACL,CAAC;QACD,IACE,OAAO,QAAQ,CAAC,eAAe,KAAK,QAAQ;YAC5C,CAAC,QAAQ,CAAC,eAAe,EACzB,CAAC;YACD,MAAM,CAAC,IAAI,CAAC;gBACV,IAAI,EAAE,GAAG,MAAM,kBAAkB;gBACjC,OAAO,EAAE,uBAAuB;aACjC,CAAC,CAAC;QACL,CAAC;QAED,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC;YACxC,MAAM,CAAC,IAAI,CAAC;gBACV,IAAI,EAAE,GAAG,MAAM,aAAa;gBAC5B,OAAO,EAAE,sBAAsB;aAChC,CAAC,CAAC;YACH,SAAS;QACX,CAAC;QAED,IAAI,QAAQ,CAAC,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACrC,MAAM,CAAC,IAAI,CAAC;gBACV,IAAI,EAAE,GAAG,MAAM,aAAa;gBAC5B,OAAO,EAAE,gCAAgC;aAC1C,CAAC,CAAC;YACH,SAAS;QACX,CAAC;QAED,oCAAoC;QACpC,MAAM,OAAO,GAAG,IAAI,GAAG,EAAU,CAAC;QAClC,KAAK,MAAM,SAAS,IAAI,QAAQ,CAAC,UAAU,EAAE,CAAC;YAC5C,IAAI,OAAO,SAAS,CAAC,EAAE,KAAK,QAAQ,IAAI,CAAC,SAAS,CAAC,EAAE,EAAE,CAAC;gBACtD,MAAM,CAAC,IAAI,CAAC;oBACV,IAAI,EAAE,GAAG,MAAM,kBAAkB;oBACjC,OAAO,EAAE,uBAAuB;iBACjC,CAAC,CAAC;YACL,CAAC;YACD,IAAI,OAAO,SAAS,CAAC,IAAI,KAAK,QAAQ,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC;gBAC1D,MAAM,CAAC,IAAI,CAAC;oBACV,IAAI,EAAE,GAAG,MAAM,oBAAoB;oBACnC,OAAO,EAAE,uBAAuB;iBACjC,CAAC,CAAC;YACL,CAAC;YACD,IAAI,SAAS,CAAC,EAAE,IAAI,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC,EAAE,CAAC;gBAC9C,MAAM,CAAC,IAAI,CAAC;oBACV,IAAI,EAAE,GAAG,MAAM,aAAa;oBAC5B,OAAO,EAAE,2BAA2B,SAAS,CAAC,EAAE,EAAE;iBACnD,CAAC,CAAC;YACL,CAAC;YACD,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QAC5B,CAAC;IACH,CAAC;IAED,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACtB,MAAM,IAAI,mBAAmB,CAAC,MAAM,CAAC,CAAC;IACxC,CAAC;IAED,iCAAiC;IACjC,MAAM,KAAK,GAAe,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC;QACtD,EAAE,EAAE,CAAC,CAAC,EAAE;QACR,IAAI,EAAE,CAAC,CAAC,IAAI;QACZ,MAAM,EAAE,CAAC,CAAC,MAAM;QAChB,eAAe,EAAE,CAAC,CAAC,eAAe;QAClC,KAAK,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE;QAC5C,UAAU,EAAE,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC;YACxC,EAAE,EAAE,CAAC,CAAC,EAAE;YACR,IAAI,EAAE,CAAC,CAAC,IAAI;YACZ,IAAI,EAAE,CAAC,CAAC,IAAI,IAAI,SAAS;SAC1B,CAAC,CAAC;KACJ,CAAC,CAAC,CAAC;IAEJ,OAAO,EAAE,UAAU,EAAE,MAAM,CAAC,UAAU,EAAE,KAAK,EAAE,CAAC;AAClD,CAAC"}

package/dist/eval/skill-scanner.d.ts

@@ -0,0 +1,8 @@
+export interface SkillInfo {
+    plugin: string;
+    skill: string;
+    dir: string;
+    hasEvals: boolean;
+    hasBenchmark: boolean;
+}
+export declare function scanSkills(root: string): Promise<SkillInfo[]>;

package/dist/eval/skill-scanner.js

@@ -0,0 +1,44 @@
+// ---------------------------------------------------------------------------
+// Filesystem scanner for plugin skills
+// ---------------------------------------------------------------------------
+import { readdirSync, existsSync } from "node:fs";
+import { join } from "node:path";
+export async function scanSkills(root) {
+    const skills = [];
+    if (!existsSync(root))
+        return skills;
+    let plugins;
+    try {
+        plugins = readdirSync(root, { withFileTypes: true })
+            .filter((d) => d.isDirectory())
+            .map((d) => d.name);
+    }
+    catch {
+        return skills;
+    }
+    for (const plugin of plugins) {
+        const skillsDir = join(root, plugin, "skills");
+        if (!existsSync(skillsDir))
+            continue;
+        let skillDirs;
+        try {
+            skillDirs = readdirSync(skillsDir, { withFileTypes: true })
+                .filter((d) => d.isDirectory())
+                .map((d) => d.name);
+        }
+        catch {
+            continue;
+        }
+        for (const skill of skillDirs) {
+            const skillDir = join(skillsDir, skill);
+            const skillMd = join(skillDir, "SKILL.md");
+            if (!existsSync(skillMd))
+                continue;
+            const hasEvals = existsSync(join(skillDir, "evals", "evals.json"));
+            const hasBenchmark = existsSync(join(skillDir, "evals", "benchmark.json"));
+            skills.push({ plugin, skill, dir: skillDir, hasEvals, hasBenchmark });
+        }
+    }
+    return skills;
+}
+//# sourceMappingURL=skill-scanner.js.map

package/dist/eval/skill-scanner.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"skill-scanner.js","sourceRoot":"","sources":["../../src/eval/skill-scanner.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,uCAAuC;AACvC,8EAA8E;AAE9E,OAAO,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AAClD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAUjC,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,IAAY;IAC3C,MAAM,MAAM,GAAgB,EAAE,CAAC;IAE/B,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;QAAE,OAAO,MAAM,CAAC;IAErC,IAAI,OAAiB,CAAC;IACtB,IAAI,CAAC;QACH,OAAO,GAAG,WAAW,CAAC,IAAI,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC;aACjD,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC;aAC9B,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IACxB,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC;QAC/C,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC;YAAE,SAAS;QAErC,IAAI,SAAmB,CAAC;QACxB,IAAI,CAAC;YACH,SAAS,GAAG,WAAW,CAAC,SAAS,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC;iBACxD,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC;iBAC9B,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QACxB,CAAC;QAAC,MAAM,CAAC;YACP,SAAS;QACX,CAAC;QAED,KAAK,MAAM,KAAK,IAAI,SAAS,EAAE,CAAC;YAC9B,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;YACxC,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;YAE3C,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC;gBAAE,SAAS;YAEnC,MAAM,QAAQ,GAAG,UAAU,CAAC,IAAI,CAAC,QAAQ,EAAE,OAAO,EAAE,YAAY,CAAC,CAAC,CAAC;YACnE,MAAM,YAAY,GAAG,UAAU,CAC7B,IAAI,CAAC,QAAQ,EAAE,OAAO,EAAE,gBAAgB,CAAC,CAC1C,CAAC;YAEF,MAAM,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,EAAE,QAAQ,EAAE,QAAQ,EAAE,YAAY,EAAE,CAAC,CAAC;QACxE,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC"}

package/dist/index.js

@@ -131,5 +131,14 @@ program
     const { blocklistCommand } = await import("./commands/blocklist.js");
     await blocklistCommand(subcommand || "list", name);
 });
+program
+    .command("eval [subcommand] [target]")
+    .description("Eval commands: init, run, coverage, generate-all")
+    .option("--force", "Overwrite existing evals.json")
+    .option("--root <path>", "Root directory for skill plugins (default: plugins/)")
+    .action(async (subcommand, target, opts) => {
+    const { evalCommand } = await import("./commands/eval.js");
+    await evalCommand(subcommand || "coverage", target, opts);
+});
 program.parse();
 //# sourceMappingURL=index.js.map