skill-probe 0.5.0 → 0.6.0

package/README.md

@@ -28,6 +28,35 @@ Complements static linters like `skill-audit` (security/quality) — this is **b
 npm i -g skill-probe      # or: npx skill-probe
 ```
 
+## Start here (`skill-probe doctor`)
+
+Run this first — it catches the setup problems (no skills dir, runtime not installed or not
+authenticated, a config typo) before you spend any probes:
+
+```bash
+skill-probe doctor --cwd .                      # check the project + runtime
+skill-probe doctor --config probe.config.json   # also sanity-check a config
+```
+
+```
+skill-probe doctor
+
+PASS  Node 22.18.0
+PASS  config parsed: probe.config.json
+PASS  found .claude/skills/ with 4 skills
+PASS  expected skills all exist
+PASS  claude CLI found
+PASS  claude-code probe succeeded (CLI is authenticated)
+WARN  threshold 90% with k=10 cannot certify a pass — need ~k=35
+FAIL  skill "greeter" missing description:
+```
+
+It checks: Node version, `.claude/skills/` exists, every skill has a `SKILL.md` with `name:` +
+`description:`, names match folders, the config parses, every `expected` skill exists, the
+threshold/k are statistically achievable, the runtime CLI is installed, and a harmless live probe
+authenticates. Exit `0` healthy · `1` warnings only · `2` hard failures. Use `--skip-probe` to skip
+the live (costing) auth check.
+
 ## Generate a config (`skill-probe gen`)
 
 Don't want to hand-write the cases? Draft them from your skills, then review:

package/dist/cli.js

@@ -8,14 +8,16 @@ import { resolve } from "node:path";
 import { runAudit } from "./eval.js";
 import { runFix } from "./fix.js";
 import { runContextAudit } from "./context.js";
+import { runDoctor } from "./doctor.js";
 import { listSkills, generateCases, genConfig } from "./gen.js";
 import { loadConfig, parseProbability, parseIntFlag, ConfigError } from "./config.js";
-import { renderTable, renderMarkdown, renderJson, renderFix, renderContext } from "./report.js";
+import { renderTable, renderMarkdown, renderJson, renderFix, renderContext, renderDoctor } from "./report.js";
 const HELP = `skill-probe — audit a co-loaded agent skill library by real activation behavior.
 
 USAGE
   skill-probe --config <file.json> [options]                 # audit
   skill-probe context --config <file.json> [options]         # isolation vs co-loaded activation rates
+  skill-probe doctor [--cwd <dir>] [--config <file.json>]    # preflight: setup/auth/config sanity
   skill-probe gen [--cwd <dir>] > probe.config.json          # draft a test config from your skills
   skill-probe fix --config <file.json> --skill <name> [opts] # rewrite a description, prove the lift
 
@@ -32,6 +34,10 @@ OPTIONS
       --markdown           emit a Markdown table (paste into a PR/README)
       --no-cost            hide the cost line (e.g. when on a Claude subscription)
       --quiet              suppress the live progress line on stderr
+  doctor only:
+      --cwd <dir>          project dir containing .claude/skills/ (default: .)
+      --config <file>      also sanity-check a config (expected skills exist, k/threshold feasible)
+      --skip-probe         don't run the harmless live probe (skip the auth check; no cost)
   gen only (uses ANTHROPIC_API_KEY if set, else the logged-in 'claude' CLI):
       --cwd <dir>          project dir containing .claude/skills/ (default: .)
       --per-skill <n>      should-fire prompts to draft per skill (default 3)
@@ -87,6 +93,17 @@ async function audit(values) {
     console.log(text);
     return result.exitCode;
 }
+async function doctor(values) {
+    const result = await runDoctor({
+        cwd: values["cwd"] ?? ".",
+        ...(values["config"] ? { configPath: values["config"] } : {}),
+        ...(values["runtime"] ? { runtime: values["runtime"] } : {}),
+        ...(values["model"] ? { model: values["model"] } : {}),
+        ...(values["skip-probe"] ? { skipProbe: true } : {}),
+    });
+    console.log(values["json"] ? renderJson(result) : renderDoctor(result));
+    return result.exitCode;
+}
 async function context(values) {
     const cfg = loadConfig(values["config"], {
         runtime: values["runtime"], model: values["model"],
@@ -147,6 +164,7 @@ async function main() {
             cwd: { type: "string" },
             "per-skill": { type: "string" },
             decoys: { type: "string" },
+            "skip-probe": { type: "boolean", default: false },
             json: { type: "boolean", default: false },
             markdown: { type: "boolean", default: false },
             "no-cost": { type: "boolean", default: false },
@@ -163,6 +181,9 @@ async function main() {
     if (cmd === "gen") {
         code = await gen(values);
     }
+    else if (cmd === "doctor") {
+        code = await doctor(values); // config is optional for doctor
+    }
     else {
         if (!values.config) {
             console.log(HELP);

package/dist/doctor.d.ts

@@ -0,0 +1,23 @@
+import type { ProbeResult, ProbeOpts } from "./types.ts";
+export type CheckStatus = "pass" | "warn" | "fail" | "info";
+export interface Check {
+    status: CheckStatus;
+    message: string;
+}
+export interface DoctorResult {
+    checks: Check[];
+    exitCode: 0 | 1 | 2;
+}
+export interface DoctorOpts {
+    cwd: string;
+    configPath?: string;
+    runtime?: string;
+    model?: string;
+    skipProbe?: boolean;
+}
+/** Injection seam so tests don't shell out to a real runtime. */
+export interface DoctorDeps {
+    cliInstalled?: (cmd: string) => boolean;
+    probe?: (runtime: string, prompt: string, opts: ProbeOpts) => Promise<ProbeResult>;
+}
+export declare function runDoctor(opts: DoctorOpts, deps?: DoctorDeps): Promise<DoctorResult>;

package/dist/doctor.js

@@ -0,0 +1,138 @@
+/** `skill-probe doctor` — preflight checks. Now that the tool is public, the first failures users
+ * hit are SETUP failures (no skills dir, runtime not installed, not authed, a config typo), not deep
+ * eval problems. doctor surfaces those in one cheap pass before they spend probes on a broken setup.
+ *
+ * Exit: 0 = healthy, 1 = warnings only, 2 = at least one hard failure. */
+import { spawnSync } from "node:child_process";
+import { existsSync, readdirSync, readFileSync } from "node:fs";
+import { join, resolve } from "node:path";
+import { loadConfig } from "./config.js";
+import { wilsonInterval } from "./stats.js";
+import { kToCertify } from "./report.js";
+import { getAdapter } from "./adapters/index.js";
+const NAME_RE = /^name:\s*(.*)$/m;
+const DESC_RE = /^description:\s*(.*)$/m;
+const MIN_NODE = { major: 22, minor: 18 }; // package.json engines: ">=22.18"
+function realCliInstalled(cmd) {
+    const r = spawnSync(cmd, ["--version"], { stdio: "ignore", timeout: 15_000 });
+    // ENOENT = not on PATH. Any other result (even a non-zero exit) means the binary exists.
+    return !(r.error && r.error.code === "ENOENT");
+}
+export async function runDoctor(opts, deps = {}) {
+    const cliInstalled = deps.cliInstalled ?? realCliInstalled;
+    const probe = deps.probe ?? ((rt, p, o) => getAdapter(rt).probe(p, o));
+    const checks = [];
+    const add = (status, message) => { checks.push({ status, message }); };
+    // 1. Node version (declared engine)
+    const [maj = 0, min = 0] = process.versions.node.split(".").map(Number);
+    if (maj > MIN_NODE.major || (maj === MIN_NODE.major && min >= MIN_NODE.minor)) {
+        add("pass", `Node ${process.versions.node}`);
+    }
+    else {
+        add("warn", `Node ${process.versions.node} is below the declared engine >=22.18 ` +
+            `(the compiled CLI may still run, but this is untested)`);
+    }
+    // 2. Config (optional). Load first — it can redirect the skills dir (cwd resolves to the config's dir).
+    let cfg = null;
+    if (opts.configPath) {
+        if (!existsSync(resolve(opts.configPath))) {
+            add("fail", `config not found: ${opts.configPath}`);
+        }
+        else {
+            try {
+                cfg = loadConfig(opts.configPath, { ...(opts.runtime ? { runtime: opts.runtime } : {}) });
+                add("pass", `config parsed: ${opts.configPath}`);
+            }
+            catch (e) {
+                add("fail", `config ${opts.configPath}: ${e instanceof Error ? e.message : String(e)}`);
+            }
+        }
+    }
+    const skillsCwd = cfg ? cfg.cwd : resolve(opts.cwd);
+    const runtime = cfg?.runtime ?? opts.runtime ?? "claude-code";
+    // 3-6. Skills directory + each SKILL.md
+    const skillNames = new Set();
+    const skillsRoot = join(skillsCwd, ".claude", "skills");
+    if (!existsSync(skillsRoot)) {
+        add("fail", `no .claude/skills/ directory under ${skillsCwd}`);
+    }
+    else {
+        const dirs = readdirSync(skillsRoot, { withFileTypes: true }).filter((d) => d.isDirectory());
+        if (dirs.length === 0) {
+            add("fail", `.claude/skills/ exists but contains no skill folders`);
+        }
+        let valid = 0;
+        for (const d of dirs) {
+            const md = join(skillsRoot, d.name, "SKILL.md");
+            if (!existsSync(md)) {
+                add("fail", `skill "${d.name}" missing SKILL.md`);
+                continue;
+            }
+            const text = readFileSync(md, "utf8");
+            const name = NAME_RE.exec(text)?.[1]?.trim() ?? "";
+            const desc = DESC_RE.exec(text)?.[1]?.trim() ?? "";
+            if (!name)
+                add("fail", `skill "${d.name}" missing name:`);
+            else if (name !== d.name)
+                add("warn", `skill "${d.name}" has name: "${name}" (doesn't match folder name)`);
+            if (!desc)
+                add("fail", `skill "${d.name}" missing description:`);
+            else if (desc === "|" || desc === ">")
+                add("warn", `skill "${d.name}" uses a multi-line description block — a single-line description triggers more reliably`);
+            skillNames.add(d.name);
+            valid++;
+        }
+        if (valid > 0)
+            add("pass", `found .claude/skills/ with ${valid} skill${valid === 1 ? "" : "s"}`);
+    }
+    // 7-9. Config-dependent checks
+    if (cfg) {
+        const missing = cfg.cases
+            .map((c) => c.expected).filter((s) => s !== null)
+            .filter((s) => !skillNames.has(s));
+        if (missing.length)
+            add("fail", `config expects skill(s) not in the library: ${[...new Set(missing)].join(", ")}`);
+        else
+            add("pass", `expected skills all exist`);
+        if (cfg.cases.every((c) => c.expected === null)) {
+            add("warn", `config has only decoy cases — no skill is being verified to fire`);
+        }
+        // a strict threshold at a small k can be mathematically impossible to certify
+        if (wilsonInterval(cfg.k, cfg.k, cfg.conf).lo <= cfg.threshold) {
+            add("warn", `threshold ${Math.round(cfg.threshold * 100)}% with k=${cfg.k} cannot certify a pass ` +
+                `even on a perfect run — need ~k=${kToCertify(cfg.threshold, cfg.conf)} (or lower the threshold)`);
+        }
+    }
+    // 10-11. Runtime + a harmless live probe (verifies the CLI is installed AND authed)
+    add("info", `runtime: ${runtime}  model: ${cfg?.model ?? opts.model ?? "(runtime default)"}`);
+    const known = runtime === "claude-code" || runtime === "opencode";
+    const bin = runtime === "claude-code" ? "claude" : runtime === "opencode" ? "opencode" : null;
+    let cliOk = false;
+    if (!known) {
+        add("fail", `unknown runtime "${runtime}" (expected claude-code or opencode)`);
+    }
+    else if (bin && cliInstalled(bin)) {
+        add("pass", `${bin} CLI found`);
+        cliOk = true;
+    }
+    else {
+        add("fail", `${bin} CLI not found on PATH — install it and authenticate to run ${runtime}`);
+    }
+    if (opts.skipProbe) {
+        add("info", `probe skipped (--skip-probe)`);
+    }
+    else if (known && cliOk) {
+        const r = await probe(runtime, "Reply with exactly: ok", {
+            cwd: skillsCwd, timeoutMs: 60_000,
+            ...(cfg?.model ?? opts.model ? { model: (cfg?.model ?? opts.model) } : {}),
+        });
+        if (r.status === "ok")
+            add("pass", `${runtime} probe succeeded (CLI is authenticated)`);
+        else
+            add("fail", `${runtime} probe failed — check auth/login: ${r.error ?? "unknown error"}`);
+    }
+    const hasFail = checks.some((c) => c.status === "fail");
+    const hasWarn = checks.some((c) => c.status === "warn");
+    const exitCode = hasFail ? 2 : hasWarn ? 1 : 0;
+    return { checks, exitCode };
+}

package/dist/report.d.ts

@@ -1,6 +1,9 @@
 import type { AuditResult } from "./eval.ts";
 import type { FixResult } from "./fix.ts";
 import type { ContextAuditResult } from "./context.ts";
+import type { DoctorResult } from "./doctor.ts";
+/** Smallest k at which an all-success run could certify reliability >= threshold. */
+export declare function kToCertify(threshold: number, conf: number): number;
 export interface RenderOpts {
     showCost?: boolean;
 }
@@ -8,6 +11,7 @@ export declare function renderTable(a: AuditResult, opts?: RenderOpts): string;
 /** Markdown table — paste into a PR or README. */
 export declare function renderMarkdown(a: AuditResult, opts?: RenderOpts): string;
 export declare function renderJson(a: AuditResult | unknown): string;
+export declare function renderDoctor(d: DoctorResult): string;
 /** activation-rate-by-context: isolation vs co-loaded, with a Fisher's-exact p on the drop. */
 export declare function renderContext(a: ContextAuditResult, opts?: RenderOpts): string;
 export declare function renderFix(f: FixResult): string;

package/dist/report.js

@@ -8,7 +8,7 @@ const LABEL = {
     inconclusive: "INCONCLUSIVE", error: "ERROR       ",
 };
 /** Smallest k at which an all-success run could certify reliability >= threshold. */
-function kToCertify(threshold, conf) {
+export function kToCertify(threshold, conf) {
     for (let k = 1; k <= 500; k++) {
         if (wilsonInterval(k, k, conf).lo > threshold)
             return k;
@@ -81,6 +81,18 @@ export function renderMarkdown(a, opts = {}) {
 export function renderJson(a) {
     return JSON.stringify(a, null, 2);
 }
+export function renderDoctor(d) {
+    const out = ["skill-probe doctor", ""];
+    for (const c of d.checks)
+        out.push(`${c.status.toUpperCase().padEnd(4)}  ${c.message}`);
+    out.push("");
+    const f = d.checks.filter((c) => c.status === "fail").length;
+    const w = d.checks.filter((c) => c.status === "warn").length;
+    out.push(f ? `${f} failure(s)` + (w ? `, ${w} warning(s)` : "") + `  |  exit ${d.exitCode}`
+        : w ? `${w} warning(s)  |  exit ${d.exitCode}`
+            : `all healthy  |  exit 0`);
+    return out.join("\n");
+}
 /** activation-rate-by-context: isolation vs co-loaded, with a Fisher's-exact p on the drop. */
 export function renderContext(a, opts = {}) {
     const showCost = opts.showCost ?? true;

package/package.json

@@ -1,9 +1,11 @@
 {
   "name": "skill-probe",
-  "version": "0.5.0",
+  "version": "0.6.0",
   "description": "Runtime-agnostic, co-loaded-aware auditor for AI agent skill libraries: measures which skill actually fires, with statistical confidence.",
   "type": "module",
-  "bin": { "skill-probe": "dist/cli.js" },
+  "bin": {
+    "skill-probe": "dist/cli.js"
+  },
   "scripts": {
     "dev": "node src/cli.ts",
     "test": "node --test test/*.test.ts",
@@ -11,13 +13,36 @@
     "build": "tsc",
     "prepublishOnly": "npm run build && npm test"
   },
-  "keywords": ["agent-skills", "skill", "eval", "claude-code", "mcp", "SKILL.md", "trigger"],
+  "keywords": [
+    "agent-skills",
+    "skill",
+    "eval",
+    "claude-code",
+    "mcp",
+    "SKILL.md",
+    "trigger"
+  ],
   "author": "HystonKayange",
   "license": "MIT",
-  "repository": { "type": "git", "url": "git+https://github.com/HystonKayange/skill-probe.git" },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/HystonKayange/skill-probe.git"
+  },
   "homepage": "https://github.com/HystonKayange/skill-probe#readme",
-  "bugs": { "url": "https://github.com/HystonKayange/skill-probe/issues" },
-  "engines": { "node": ">=22.18" },
-  "devDependencies": { "typescript": "^5.5.0", "@types/node": "^22.0.0" },
-  "files": ["dist", "examples", "README.md", "LICENSE"]
+  "bugs": {
+    "url": "https://github.com/HystonKayange/skill-probe/issues"
+  },
+  "engines": {
+    "node": ">=22.18"
+  },
+  "devDependencies": {
+    "typescript": "^5.5.0",
+    "@types/node": "^22.0.0"
+  },
+  "files": [
+    "dist",
+    "examples",
+    "README.md",
+    "LICENSE"
+  ]
 }