skill-probe 0.3.0 → 0.4.0

package/README.md

@@ -9,6 +9,12 @@ which skill *actually* fires (by real activation, not keyword matching) when you
 
 Complements static linters like `skill-audit` (security/quality) — this is **behavioral**.
 
+> **How it works / requirements.** skill-probe is a **terminal CLI**, *not* an in-agent slash
+> command. It drives your local runtime CLI under the hood, so you need that runtime installed and
+> authenticated: **`claude`** (logged in) for `runtime: claude-code`, or **`opencode`** for
+> `runtime: opencode`. `gen`/`fix` use `ANTHROPIC_API_KEY` if set, otherwise the local `claude` CLI.
+> Codex/Gemini runtimes aren't supported yet (their traces don't expose which skill fired).
+
 ## Why
 
 - Skill activation is stochastic: the same prompt can produce different skill-routing outcomes.
@@ -35,8 +41,11 @@ skill names are dropped automatically. Flags: `--per-skill <n>` (default 3), `--
 (default 2), `--model <id>`.
 
 > **Always review the draft before running.** Generators lean toward obvious keyword-matches; you
-> add the messy, oblique phrasings real users type (that's where triggering actually fails). `gen`
-> needs `ANTHROPIC_API_KEY`.
+> add the messy, oblique phrasings real users type (that's where triggering actually fails).
+
+`gen` (and `fix`) use `ANTHROPIC_API_KEY` if it's set, otherwise they fall back to the logged-in
+**`claude` CLI** — so on a Claude subscription the whole tool (audit, gen, fix) works with **no API
+key at all**.
 
 ## Use
 
@@ -122,12 +131,13 @@ P(rewrite improved reliability) = 100%   Δ = +80% [36%, 99%]
 original backed up to: /path/to/SKILL.md.bak.1719000000000
 ```
 
-`fix` needs `ANTHROPIC_API_KEY` (for the rewrite). It changes descriptions on statistical evidence,
-not on "the new one looks nicer" — a rewrite that doesn't measurably help is reverted.
+`fix` does the rewrite via `ANTHROPIC_API_KEY` if set, else the logged-in `claude` CLI (subscription).
+It changes descriptions on statistical evidence, not on "the new one looks nicer" — a rewrite that
+doesn't measurably help is reverted.
 
 ## Status
 
-Early (v0.1). **Audit (`skill-probe`):** Wilson confidence intervals + sequential stopping +
+Early (v0.4). **Audit (`skill-probe`):** Wilson confidence intervals + sequential stopping +
 four-state verdict (pass / fail / inconclusive / **error**), across two runtimes (Claude Code,
 OpenCode). Infrastructure failures (timeout / auth / crash / empty output) are reported as
 `error`, never as a behavioral pass/fail — a decoy can't falsely pass because the runtime was down.

package/dist/cli.js

@@ -8,7 +8,7 @@ import { resolve } from "node:path";
 import { runAudit } from "./eval.js";
 import { runFix } from "./fix.js";
 import { listSkills, generateCases, genConfig } from "./gen.js";
-import { loadConfig, parseProbability, ConfigError } from "./config.js";
+import { loadConfig, parseProbability, parseIntFlag, ConfigError } from "./config.js";
 import { renderTable, renderMarkdown, renderJson, renderFix } from "./report.js";
 const HELP = `skill-probe — audit a co-loaded agent skill library by real activation behavior.
 
@@ -30,7 +30,7 @@ OPTIONS
       --markdown           emit a Markdown table (paste into a PR/README)
       --no-cost            hide the cost line (e.g. when on a Claude subscription)
       --quiet              suppress the live progress line on stderr
-  gen only (needs ANTHROPIC_API_KEY):
+  gen only (uses ANTHROPIC_API_KEY if set, else the logged-in 'claude' CLI):
       --cwd <dir>          project dir containing .claude/skills/ (default: .)
       --per-skill <n>      should-fire prompts to draft per skill (default 3)
       --decoys <n>         off-topic decoy prompts to draft (default 2)
@@ -74,14 +74,10 @@ async function audit(values) {
     console.log(text);
     return result.exitCode;
 }
-function intArg(raw, def, min) {
-    if (raw === undefined)
-        return def;
-    const v = Math.floor(Number(raw));
-    return Number.isFinite(v) && v >= min ? v : def;
-}
 async function gen(values) {
     const cwd = values["cwd"] ?? ".";
+    const perSkill = parseIntFlag(values["per-skill"], "--per-skill", 3, 1);
+    const decoys = parseIntFlag(values["decoys"], "--decoys", 2, 0);
     const skills = listSkills(resolve(cwd));
     if (skills.length === 0) {
         console.error(`no skills found under ${cwd}/.claude/skills/`);
@@ -89,9 +85,7 @@ async function gen(values) {
     }
     const model = values["model"];
     const cases = await generateCases(skills, {
-        perSkill: intArg(values["per-skill"], 3, 1),
-        decoys: intArg(values["decoys"], 2, 0),
-        ...(model ? { model } : {}),
+        perSkill, decoys, ...(model ? { model } : {}),
     });
     console.log(renderJson(genConfig(cases)));
     console.error(`# drafted ${cases.length} cases for ${skills.length} skill(s) — REVIEW before ` +

package/dist/config.d.ts

@@ -9,6 +9,8 @@ export declare class ConfigError extends Error {
 export declare function validateConfig(raw: unknown, configPath: string): Config;
 /** Validate a probability-ish flag (e.g. --apply-threshold): must be a number strictly in (0,1). */
 export declare function parseProbability(raw: string | undefined, name: string, def: number): number;
+/** Validate an integer flag (e.g. --per-skill, --decoys): must be an integer >= min if provided. */
+export declare function parseIntFlag(raw: string | undefined, name: string, def: number, min: number): number;
 /** CLI flag overrides (already string-typed from argv); numerics are coerced then validated. */
 export interface Overrides {
     runtime?: string | undefined;

package/dist/config.js

@@ -75,6 +75,16 @@ export function parseProbability(raw, name, def) {
     }
     return v;
 }
+/** Validate an integer flag (e.g. --per-skill, --decoys): must be an integer >= min if provided. */
+export function parseIntFlag(raw, name, def, min) {
+    if (raw === undefined)
+        return def;
+    const v = Number(raw);
+    if (!Number.isInteger(v) || v < min) {
+        throw new ConfigError([`${name} must be an integer >= ${min} (got ${JSON.stringify(raw)})`]);
+    }
+    return v;
+}
 export function loadConfig(configPath, overrides = {}) {
     let fileRaw;
     try {

package/dist/llm.d.ts

@@ -1,4 +1,15 @@
-export declare function complete(system: string, user: string, opts?: {
+export type LlmBackend = "api" | "cli";
+/** Which backend complete() will use, given the environment. Exported for testing/diagnostics. */
+export declare function llmBackend(): LlmBackend;
+export interface CompleteOpts {
     model?: string;
     maxTokens?: number;
-}): Promise<string>;
+}
+export declare function complete(system: string, user: string, opts?: CompleteOpts): Promise<string>;
+/** Pure interpreter of a `claude -p --output-format json` run. Treats a non-zero exit, an
+ * `is_error: true` payload, a timeout, or empty output as ERRORS — so an error string can never
+ * be mistaken for a valid generation (critical: fix would otherwise write it as a description). */
+export declare function interpretClaudeJson(code: number | null, out: string, err: string, timedOut: boolean): {
+    text?: string;
+    error?: string;
+};

package/dist/llm.js

@@ -1,33 +1,72 @@
-/** Minimal Anthropic client for the `fix` command's description rewrite. Uses Node's built-in
- * fetch (Node >=18) — zero deps. Needs ANTHROPIC_API_KEY (only `fix` uses this, not the audit). */
+/** LLM text completion for `gen` and `fix`. Unified auth (v0.4):
+ *   - ANTHROPIC_API_KEY set  → call the Anthropic API directly (fetch, Node >=18).
+ *   - otherwise              → fall back to the `claude` CLI (works on a Claude subscription,
+ *                              no API key needed).
+ * So the whole tool — audit, gen, fix — runs on just a logged-in `claude`. */
+import { spawn } from "node:child_process";
+import { tmpdir } from "node:os";
 const ANTHROPIC_URL = "https://api.anthropic.com/v1/messages";
+/** Which backend complete() will use, given the environment. Exported for testing/diagnostics. */
+export function llmBackend() {
+    return process.env["ANTHROPIC_API_KEY"] ? "api" : "cli";
+}
 export async function complete(system, user, opts = {}) {
+    return llmBackend() === "api"
+        ? completeViaApi(system, user, opts)
+        : completeViaClaudeCli(system, user, opts);
+}
+async function completeViaApi(system, user, opts) {
     const key = process.env["ANTHROPIC_API_KEY"];
-    if (!key) {
-        throw new Error("ANTHROPIC_API_KEY is required for `skill-probe fix` (it rewrites the description).");
-    }
     const model = opts.model ?? process.env["SKILL_PROBE_FIX_MODEL"] ?? "claude-sonnet-4-6";
     const res = await fetch(ANTHROPIC_URL, {
         method: "POST",
-        headers: {
-            "content-type": "application/json",
-            "x-api-key": key,
-            "anthropic-version": "2023-06-01",
-        },
-        body: JSON.stringify({
-            model,
-            max_tokens: opts.maxTokens ?? 400,
-            system,
-            messages: [{ role: "user", content: user }],
-        }),
+        headers: { "content-type": "application/json", "x-api-key": key, "anthropic-version": "2023-06-01" },
+        body: JSON.stringify({ model, max_tokens: opts.maxTokens ?? 400, system, messages: [{ role: "user", content: user }] }),
     });
-    if (!res.ok) {
+    if (!res.ok)
         throw new Error(`Anthropic API ${res.status}: ${(await res.text()).slice(0, 200)}`);
-    }
     const body = (await res.json());
-    return (body.content ?? [])
-        .filter((b) => b.type === "text")
-        .map((b) => b.text ?? "")
-        .join("")
-        .trim();
+    return (body.content ?? []).filter((b) => b.type === "text").map((b) => b.text ?? "").join("").trim();
+}
+/** Pure interpreter of a `claude -p --output-format json` run. Treats a non-zero exit, an
+ * `is_error: true` payload, a timeout, or empty output as ERRORS — so an error string can never
+ * be mistaken for a valid generation (critical: fix would otherwise write it as a description). */
+export function interpretClaudeJson(code, out, err, timedOut) {
+    if (timedOut)
+        return { error: "claude CLI timed out after 120s" };
+    let parsed = null;
+    try {
+        parsed = JSON.parse(out);
+    }
+    catch { /* may be plain text */ }
+    if (code !== 0 || parsed?.is_error === true) {
+        const ctx = (parsed && typeof parsed.result === "string" ? parsed.result : "") || err || out;
+        return { error: `claude CLI error (exit ${code}): ${ctx.trim().slice(0, 200)}` };
+    }
+    let text = parsed ? (typeof parsed.result === "string" ? parsed.result : "") : out;
+    text = text.trim();
+    return text ? { text } : { error: `claude CLI returned no text (${(err || out).slice(0, 200)})` };
+}
+function completeViaClaudeCli(system, user, opts) {
+    return new Promise((resolve, reject) => {
+        const args = ["-p", user, "--append-system-prompt", system, "--output-format", "json"];
+        if (opts.model)
+            args.push("--model", opts.model);
+        // run in a neutral cwd so the project's own skills/AGENTS.md don't bias the generation
+        const child = spawn("claude", args, { cwd: tmpdir(), stdio: ["ignore", "pipe", "pipe"] });
+        let out = "", err = "", settled = false, timedOut = false;
+        const timer = setTimeout(() => { timedOut = true; child.kill("SIGKILL"); }, 120_000);
+        const done = (fn) => { if (settled)
+            return; settled = true; clearTimeout(timer); fn(); };
+        child.stdout.on("data", (d) => { out += d.toString(); });
+        child.stderr.on("data", (d) => { err += d.toString(); });
+        child.on("error", (e) => done(() => reject(new Error(`claude CLI failed: ${e.message} — set ANTHROPIC_API_KEY or install/login 'claude'.`))));
+        child.on("close", (code) => done(() => {
+            const r = interpretClaudeJson(code, out, err, timedOut);
+            if (r.text)
+                resolve(r.text);
+            else
+                reject(new Error(r.error ?? "claude CLI failed"));
+        }));
+    });
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "skill-probe",
-  "version": "0.3.0",
+  "version": "0.4.0",
   "description": "Runtime-agnostic, co-loaded-aware auditor for AI agent skill libraries: measures which skill actually fires, with statistical confidence.",
   "type": "module",
   "bin": { "skill-probe": "dist/cli.js" },