skill-probe 0.1.0 → 0.3.0

package/README.md

@@ -11,7 +11,7 @@ Complements static linters like `skill-audit` (security/quality) — this is **b
 
 ## Why
 
-- Skill activation is a coin flip: independent studies measured 55–87% trigger rates.
+- Skill activation is stochastic: the same prompt can produce different skill-routing outcomes.
 - A single run lies (we measured the same prompt at 0/5 one batch, 2/3 the next).
 - skill-probe runs each prompt **k times**, reports a **Wilson 95% CI**, stops early when the
   result is statistically decided, and flags **trigger-theft** (a sibling stealing a trigger).
@@ -22,11 +22,27 @@ Complements static linters like `skill-audit` (security/quality) — this is **b
 npm i -g skill-probe      # or: npx skill-probe
 ```
 
+## Generate a config (`skill-probe gen`)
+
+Don't want to hand-write the cases? Draft them from your skills, then review:
+
+```bash
+ANTHROPIC_API_KEY=sk-ant-...  skill-probe gen --cwd . > probe.config.json
+```
+It reads each `.claude/skills/*/SKILL.md`, and an LLM drafts realistic should-fire prompts per
+skill, cross-skill **near-misses** (to surface mis-routing), and off-topic **decoys**. Hallucinated
+skill names are dropped automatically. Flags: `--per-skill <n>` (default 3), `--decoys <n>`
+(default 2), `--model <id>`.
+
+> **Always review the draft before running.** Generators lean toward obvious keyword-matches; you
+> add the messy, oblique phrasings real users type (that's where triggering actually fails). `gen`
+> needs `ANTHROPIC_API_KEY`.
+
 ## Use
 
 ```bash
 # point it at your own config (after `npm i -g skill-probe`):
-skill-probe --config my.config.json --k 15 --threshold 0.9 --json
+skill-probe --config my.config.json --k 10 --threshold 0.7 --json
 
 # or, from a clone of this repo, try the bundled example (needs `claude` installed + auth):
 skill-probe --config examples/audit.config.json
@@ -48,12 +64,15 @@ Output:
 ```
 skill-probe — runtime: claude-code  model: (runtime default)  threshold: 70%
 
-  PASS          reliability 100% [72%, 100%] k=10   expect=commit-writer
+  PASS          expect=commit-writer  | write a commit message
+        reliability 100% [72%, 100%] k=10
         outcomes: commit-writer×10
-  FAIL          reliability 20% [6%, 51%] k=10   expect=pr-describer
+  FAIL          expect=pr-describer  | write a pull request description
+        reliability 20% [6%, 51%] k=10
         outcomes: None×8, commit-writer×2
         ⚠ trigger-theft by: commit-writer
-  PASS          reliability 100% [72%, 100%] k=10   expect=(none)
+  PASS          expect=(none)  | what's the weather?
+        reliability 100% [72%, 100%] k=10
         outcomes: None×10
 
 Result: 2 pass / 1 fail / 0 inconclusive / 0 error  |  exit 1  |  cost $0.18
@@ -65,6 +84,11 @@ Result: 2 pass / 1 fail / 0 inconclusive / 0 error  |  exit 1  |  cost $0.18
 - Exit code: `0` all pass, `1` a behavioral fail / trigger-theft, `2` inconclusive or an
   infrastructure error (runtime down → never a silent pass).
 
+**Output options:** a live `probing [2/3] …` progress line prints to stderr while it runs (use
+`--quiet` to silence it). `--markdown` emits a table you can paste into a PR/README; `--json`
+for machine output; `--no-cost` hides the cost line (handy on a Claude subscription, where the
+dollar figure is just an estimate, not a charge).
+
 **Cross-runtime comparison must pin the model.** A skill can fire on one runtime and not another
 partly because of the *model*, not the runtime. Always set `model` (recorded in the report) so a
 Claude Code vs OpenCode comparison is fair — otherwise you're comparing two confounded variables.
@@ -95,6 +119,7 @@ effect is positive — otherwise reverts. When applied, the original is snapshot
 before: 0% [0%, 49%]   after: 100% [51%, 100%]   (4 paired runs)
 P(rewrite improved reliability) = 100%   Δ = +80% [36%, 99%]
 ✅ APPLIED
+original backed up to: /path/to/SKILL.md.bak.1719000000000
 ```
 
 `fix` needs `ANTHROPIC_API_KEY` (for the rewrite). It changes descriptions on statistical evidence,

package/dist/cli.js

@@ -4,14 +4,17 @@
  *   skill-probe fix --config <f> --skill <n>  rewrite a skill's description & prove the lift
  * Exit: 0 = pass/applied, 1 = behavioral fail/reverted, 2 = inconclusive/infra error/usage. */
 import { parseArgs } from "node:util";
+import { resolve } from "node:path";
 import { runAudit } from "./eval.js";
 import { runFix } from "./fix.js";
-import { loadConfig, ConfigError } from "./config.js";
-import { renderTable, renderJson, renderFix } from "./report.js";
+import { listSkills, generateCases, genConfig } from "./gen.js";
+import { loadConfig, parseProbability, ConfigError } from "./config.js";
+import { renderTable, renderMarkdown, renderJson, renderFix } from "./report.js";
 const HELP = `skill-probe — audit a co-loaded agent skill library by real activation behavior.
 
 USAGE
   skill-probe --config <file.json> [options]                 # audit
+  skill-probe gen [--cwd <dir>] > probe.config.json          # draft a test config from your skills
   skill-probe fix --config <file.json> --skill <name> [opts] # rewrite a description, prove the lift
 
 OPTIONS
@@ -24,6 +27,13 @@ OPTIONS
       --threshold <p>      min acceptable reliability, strictly 0..1 (default 0.7 = smoke test)
       --conf <p>           confidence level: 0.9 | 0.95 | 0.99 (default 0.95)
       --json               emit JSON instead of a table
+      --markdown           emit a Markdown table (paste into a PR/README)
+      --no-cost            hide the cost line (e.g. when on a Claude subscription)
+      --quiet              suppress the live progress line on stderr
+  gen only (needs ANTHROPIC_API_KEY):
+      --cwd <dir>          project dir containing .claude/skills/ (default: .)
+      --per-skill <n>      should-fire prompts to draft per skill (default 3)
+      --decoys <n>         off-topic decoy prompts to draft (default 2)
   fix only:
       --skill <name>       the skill whose description to rewrite (uses the config's cases for it)
       --apply-threshold <p> min P(improvement) to keep the rewrite (default 0.9)
@@ -31,16 +41,63 @@ OPTIONS
 
 Reliability carries a Wilson CI + sample size k. fix rewrites the description, runs an INTERLEAVED
 before/after, and applies it only if the Bayesian P(improvement) clears the apply-threshold.`;
+let progressLen = 0;
+function makeProgress(values) {
+    if (values["json"] || values["quiet"] || !process.stderr.isTTY)
+        return undefined;
+    return (e) => {
+        const p = e.prompt.length > 32 ? e.prompt.slice(0, 29) + "…" : e.prompt;
+        const line = `  probing [${e.caseIndex}/${e.caseTotal}] "${p}" — run ${e.validN}/${e.maxK}`;
+        const pad = Math.max(0, progressLen - line.length);
+        process.stderr.write("\r" + line + " ".repeat(pad));
+        progressLen = line.length;
+    };
+}
+function clearProgress() {
+    if (progressLen) {
+        process.stderr.write("\r" + " ".repeat(progressLen) + "\r");
+        progressLen = 0;
+    }
+}
 async function audit(values) {
     const cfg = loadConfig(values["config"], {
         runtime: values["runtime"], model: values["model"],
         k: values["k"], threshold: values["threshold"],
         conf: values["conf"],
     });
-    const result = await runAudit(cfg);
-    console.log(values["json"] ? renderJson(result) : renderTable(result));
+    const result = await runAudit(cfg, makeProgress(values));
+    clearProgress();
+    const showCost = !values["no-cost"];
+    const text = values["json"] ? renderJson(result)
+        : values["markdown"] ? renderMarkdown(result, { showCost })
+            : renderTable(result, { showCost });
+    console.log(text);
     return result.exitCode;
 }
+function intArg(raw, def, min) {
+    if (raw === undefined)
+        return def;
+    const v = Math.floor(Number(raw));
+    return Number.isFinite(v) && v >= min ? v : def;
+}
+async function gen(values) {
+    const cwd = values["cwd"] ?? ".";
+    const skills = listSkills(resolve(cwd));
+    if (skills.length === 0) {
+        console.error(`no skills found under ${cwd}/.claude/skills/`);
+        return 2;
+    }
+    const model = values["model"];
+    const cases = await generateCases(skills, {
+        perSkill: intArg(values["per-skill"], 3, 1),
+        decoys: intArg(values["decoys"], 2, 0),
+        ...(model ? { model } : {}),
+    });
+    console.log(renderJson(genConfig(cases)));
+    console.error(`# drafted ${cases.length} cases for ${skills.length} skill(s) — REVIEW before ` +
+        `running (add the messy/oblique phrasings the generator misses).`);
+    return 0;
+}
 async function fix(values) {
     const skill = values["skill"];
     if (!skill) {
@@ -51,7 +108,7 @@ async function fix(values) {
         runtime: values["runtime"], model: values["model"],
         k: values["k"], conf: values["conf"],
     });
-    const applyBar = values["apply-threshold"] ? Number(values["apply-threshold"]) : 0.9;
+    const applyBar = parseProbability(values["apply-threshold"], "--apply-threshold", 0.9);
     const result = await runFix(cfg, { skill, k: cfg.k, applyBar, ...(cfg.model ? { model: cfg.model } : {}) });
     console.log(values["json"] ? renderJson(result) : renderFix(result));
     return result.applied ? 0 : 1;
@@ -68,16 +125,32 @@ async function main() {
             conf: { type: "string" },
             skill: { type: "string" },
             "apply-threshold": { type: "string" },
+            cwd: { type: "string" },
+            "per-skill": { type: "string" },
+            decoys: { type: "string" },
             json: { type: "boolean", default: false },
+            markdown: { type: "boolean", default: false },
+            "no-cost": { type: "boolean", default: false },
+            quiet: { type: "boolean", default: false },
             help: { type: "boolean", short: "h", default: false },
         },
     });
-    if (values.help || !values.config) {
+    if (values.help) {
         console.log(HELP);
-        process.exit(values.help ? 0 : 2);
+        process.exit(0);
     }
     const cmd = positionals[0];
-    const code = cmd === "fix" ? await fix(values) : await audit(values);
+    let code;
+    if (cmd === "gen") {
+        code = await gen(values);
+    }
+    else {
+        if (!values.config) {
+            console.log(HELP);
+            process.exit(2);
+        }
+        code = cmd === "fix" ? await fix(values) : await audit(values);
+    }
     process.exit(code);
 }
 main().catch((e) => {

package/dist/config.d.ts

@@ -7,6 +7,8 @@ export declare class ConfigError extends Error {
  * `configPath` is used to resolve a relative `cwd` against the CONFIG FILE's
  * directory — not the caller's shell (finding #3). Throws ConfigError on any problem. */
 export declare function validateConfig(raw: unknown, configPath: string): Config;
+/** Validate a probability-ish flag (e.g. --apply-threshold): must be a number strictly in (0,1). */
+export declare function parseProbability(raw: string | undefined, name: string, def: number): number;
 /** CLI flag overrides (already string-typed from argv); numerics are coerced then validated. */
 export interface Overrides {
     runtime?: string | undefined;

package/dist/config.js

@@ -65,6 +65,16 @@ export function validateConfig(raw, configPath) {
         cases: cases,
     };
 }
+/** Validate a probability-ish flag (e.g. --apply-threshold): must be a number strictly in (0,1). */
+export function parseProbability(raw, name, def) {
+    if (raw === undefined)
+        return def;
+    const v = Number(raw);
+    if (!Number.isFinite(v) || v <= 0 || v >= 1) {
+        throw new ConfigError([`${name} must be a number strictly between 0 and 1 (got ${JSON.stringify(raw)})`]);
+    }
+    return v;
+}
 export function loadConfig(configPath, overrides = {}) {
     let fileRaw;
     try {

package/dist/eval.d.ts

@@ -30,4 +30,11 @@ export interface AuditResult {
     /** 0 = all pass; 1 = a behavioral failure/theft; 2 = inconclusive or infrastructure error */
     exitCode: 0 | 1 | 2;
 }
-export declare function runAudit(cfg: Config): Promise<AuditResult>;
+export interface ProgressEvent {
+    caseIndex: number;
+    caseTotal: number;
+    prompt: string;
+    validN: number;
+    maxK: number;
+}
+export declare function runAudit(cfg: Config, onProgress?: (e: ProgressEvent) => void): Promise<AuditResult>;

package/dist/eval.js

@@ -14,14 +14,20 @@ function verdictFor(stats, decision, theft) {
         return "fail";
     return "inconclusive"; // includes "k too small to decide" — report says how many runs are needed
 }
-export async function runAudit(cfg) {
+export async function runAudit(cfg, onProgress) {
     const adapter = getAdapter(cfg.runtime);
     const cases = [];
     let totalCost = 0;
-    for (const c of cfg.cases) {
+    for (let ci = 0; ci < cfg.cases.length; ci++) {
+        const c = cfg.cases[ci];
         const stats = await measure(adapter, c.prompt, c.expected, {
             cwd: cfg.cwd, maxK: cfg.k, threshold: cfg.threshold, conf: cfg.conf,
             ...(cfg.model ? { model: cfg.model } : {}),
+            ...(onProgress ? {
+                onProbe: (validN, maxK) => onProgress({
+                    caseIndex: ci + 1, caseTotal: cfg.cases.length, prompt: c.prompt, validN, maxK,
+                }),
+            } : {}),
         });
         totalCost += stats.totalCost;
         const rel = reliability(stats.hits, stats.n, cfg.conf);

package/dist/gen.d.ts

@@ -0,0 +1,26 @@
+import type { Config, EvalCase } from "./types.ts";
+export interface SkillInfo {
+    name: string;
+    description: string;
+}
+export declare function listSkills(cwd: string): SkillInfo[];
+export interface GenOpts {
+    perSkill: number;
+    decoys: number;
+    model?: string;
+}
+/** A raw, untrusted candidate from a generator — sanitized by generateCases(). */
+export interface RawCase {
+    prompt?: unknown;
+    expected?: unknown;
+}
+export interface GenDeps {
+    generate?: (skills: SkillInfo[], opts: GenOpts) => Promise<RawCase[]>;
+}
+export declare function defaultGenerate(skills: SkillInfo[], opts: GenOpts): Promise<RawCase[]>;
+/** Runs a generator, then SANITIZES its output (applies to any generator, not just the LLM one):
+ * keep entries with a non-empty prompt and an `expected` that is a real skill name or null —
+ * dropping hallucinated skill names. */
+export declare function generateCases(skills: SkillInfo[], opts: GenOpts, deps?: GenDeps): Promise<EvalCase[]>;
+/** Wrap generated cases in a ready-to-run config with sensible defaults. */
+export declare function genConfig(cases: EvalCase[]): Config;

package/dist/gen.js

@@ -0,0 +1,63 @@
+/** `skill-probe gen` — draft a test config from the skills in a project. The skill descriptions
+ * already say what each skill is for, so an LLM can propose realistic should-fire prompts,
+ * cross-skill near-misses (to surface mis-routing), and decoys. ALWAYS human-reviewed before use
+ * (the generator tends toward obvious keyword-matches; you add the messy real-world phrasings). */
+import { readFileSync, existsSync, readdirSync } from "node:fs";
+import { join } from "node:path";
+import { complete } from "./llm.js";
+const DESC_RE = /^description:\s*(.*)$/m;
+export function listSkills(cwd) {
+    const root = join(cwd, ".claude", "skills");
+    if (!existsSync(root))
+        throw new Error(`no .claude/skills/ directory under ${cwd}`);
+    const skills = [];
+    for (const name of readdirSync(root)) {
+        const file = join(root, name, "SKILL.md");
+        if (!existsSync(file))
+            continue;
+        const m = DESC_RE.exec(readFileSync(file, "utf8"));
+        skills.push({ name, description: m ? m[1].trim() : "" });
+    }
+    return skills;
+}
+export async function defaultGenerate(skills, opts) {
+    const list = skills.map((s) => `- ${s.name}: ${s.description}`).join("\n");
+    const sys = "You build a TEST SUITE that audits whether an AI coding agent loads the correct "
+        + "skill for a user's prompt. Output ONLY a JSON array, no prose.";
+    const user = `Skills available (name: description):\n${list}\n\n`
+        + `Produce a JSON array of {"prompt": string, "expected": string|null}:\n`
+        + `- For EACH skill, ${opts.perSkill} realistic user prompts that SHOULD load it. Vary the `
+        + `phrasing — direct, oblique/indirect, terse, and with surrounding context. Do NOT just echo `
+        + `the description's keywords; write how real users actually type.\n`
+        + `- Where two skills overlap, add a few NEAR-MISS prompts in the gray zone, with "expected" `
+        + `set to the single most-appropriate skill (these surface mis-routing / trigger-theft).\n`
+        + `- ${opts.decoys} DECOY prompts that match NO skill, each with "expected": null.\n`
+        + `"expected" MUST be exactly one of the skill names above, or null. Return ONLY the JSON array.`;
+    const raw = await complete(sys, user, opts.model ? { model: opts.model, maxTokens: 2000 } : { maxTokens: 2000 });
+    const m = raw.match(/\[[\s\S]*\]/);
+    if (!m)
+        throw new Error("generator did not return a JSON array");
+    return JSON.parse(m[0]);
+}
+/** Runs a generator, then SANITIZES its output (applies to any generator, not just the LLM one):
+ * keep entries with a non-empty prompt and an `expected` that is a real skill name or null —
+ * dropping hallucinated skill names. */
+export async function generateCases(skills, opts, deps = {}) {
+    const raw = await (deps.generate ?? defaultGenerate)(skills, opts);
+    const names = new Set(skills.map((s) => s.name));
+    const cases = [];
+    for (const c of raw) {
+        if (typeof c?.prompt !== "string" || !c.prompt)
+            continue;
+        const exp = c.expected;
+        if (exp === null || exp === undefined)
+            cases.push({ prompt: c.prompt, expected: null });
+        else if (typeof exp === "string" && names.has(exp))
+            cases.push({ prompt: c.prompt, expected: exp });
+    }
+    return cases;
+}
+/** Wrap generated cases in a ready-to-run config with sensible defaults. */
+export function genConfig(cases) {
+    return { runtime: "claude-code", cwd: ".", k: 10, threshold: 0.7, conf: 0.95, cases };
+}

package/dist/orchestrator.d.ts

@@ -21,4 +21,5 @@ export declare function measure(adapter: RuntimeAdapter, prompt: string, expecte
     threshold: number;
     conf: number;
     model?: string;
+    onProbe?: (validN: number, maxK: number) => void;
 }): Promise<RunStats>;

package/dist/orchestrator.js

@@ -12,6 +12,7 @@ export async function measure(adapter, prompt, expected, opts) {
         if (r.status === "error") {
             errors++;
             lastError = r.error;
+            opts.onProbe?.(n, opts.maxK);
             continue; // do NOT record as a behavioral outcome
         }
         const key = r.skillFired ?? "None";
@@ -19,6 +20,7 @@ export async function measure(adapter, prompt, expected, opts) {
         if (key === want)
             hits++;
         n++;
+        opts.onProbe?.(n, opts.maxK);
         if (n >= 3) {
             const d = sequentialDecision(hits, n, opts.threshold, opts.conf);
             if (d !== "undecided")

package/dist/report.d.ts

@@ -1,5 +1,10 @@
 import type { AuditResult } from "./eval.ts";
 import type { FixResult } from "./fix.ts";
-export declare function renderTable(a: AuditResult): string;
+export interface RenderOpts {
+    showCost?: boolean;
+}
+export declare function renderTable(a: AuditResult, opts?: RenderOpts): string;
+/** Markdown table — paste into a PR or README. */
+export declare function renderMarkdown(a: AuditResult, opts?: RenderOpts): string;
 export declare function renderJson(a: AuditResult | unknown): string;
 export declare function renderFix(f: FixResult): string;

package/dist/report.js

@@ -14,7 +14,8 @@ function kToCertify(threshold, conf) {
     }
     return 500;
 }
-export function renderTable(a) {
+export function renderTable(a, opts = {}) {
+    const showCost = opts.showCost ?? true;
     const out = [];
     out.push(`skill-probe — runtime: ${a.runtime}  model: ${a.model}  threshold: ${pct(a.threshold)}`);
     out.push("");
@@ -42,8 +43,38 @@ export function renderTable(a) {
     }
     out.push("");
     out.push(`Result: ${a.counts.pass} pass / ${a.counts.fail} fail / ` +
-        `${a.counts.inconclusive} inconclusive / ${a.counts.error} error  |  ` +
-        `exit ${a.exitCode}  |  cost $${a.totalCost.toFixed(4)}`);
+        `${a.counts.inconclusive} inconclusive / ${a.counts.error} error  |  exit ${a.exitCode}` +
+        (showCost ? `  |  cost $${a.totalCost.toFixed(4)}` : ""));
+    return out.join("\n");
+}
+/** Markdown table — paste into a PR or README. */
+export function renderMarkdown(a, opts = {}) {
+    const showCost = opts.showCost ?? true;
+    const mark = {
+        pass: "✅ pass", fail: "❌ fail", inconclusive: "⚠️ inconclusive", error: "⛔ error",
+    };
+    const out = [];
+    const conf = a.cases[0]?.reliability.conf ?? 0.95;
+    out.push(`### skill-probe — \`${a.runtime}\` · model \`${a.model}\` · threshold ${pct(a.threshold)}`);
+    out.push("");
+    out.push(`| Verdict | Skill | Reliability (${pct(conf)} CI) | k | Notes |`);
+    out.push("|---|---|---|---|---|");
+    for (const c of a.cases) {
+        const r = c.reliability;
+        const expect = c.expected ?? "(decoy)";
+        let notes = "";
+        if (c.verdict === "error")
+            notes = `infra error: ${c.stats.lastError ?? "untrustworthy"}`;
+        else if (c.theft.length)
+            notes = `⚠ trigger-theft by ${c.theft.join(", ")}`;
+        else if (c.verdict === "inconclusive")
+            notes = `need ~k=${kToCertify(a.threshold, r.conf)} to certify`;
+        const rel = c.verdict === "error" ? "—" : `${pct(r.pHat)} [${pct(r.ciLow)}, ${pct(r.ciHigh)}]`;
+        out.push(`| ${mark[c.verdict]} | \`${expect}\` | ${rel} | ${r.k} | ${notes} |`);
+    }
+    out.push("");
+    out.push(`**${a.counts.pass} pass · ${a.counts.fail} fail · ${a.counts.inconclusive} inconclusive · ` +
+        `${a.counts.error} error**` + (showCost ? ` · cost $${a.totalCost.toFixed(4)}` : ""));
     return out.join("\n");
 }
 export function renderJson(a) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "skill-probe",
-  "version": "0.1.0",
+  "version": "0.3.0",
   "description": "Runtime-agnostic, co-loaded-aware auditor for AI agent skill libraries: measures which skill actually fires, with statistical confidence.",
   "type": "module",
   "bin": { "skill-probe": "dist/cli.js" },