skill-probe 0.8.0 → 0.10.0

package/dist/doctor.js

@@ -4,15 +4,17 @@
  *
  * Exit: 0 = healthy, 1 = warnings only, 2 = at least one hard failure. */
 import { spawnSync } from "node:child_process";
-import { existsSync, readdirSync, readFileSync } from "node:fs";
+import { existsSync } from "node:fs";
 import { join, resolve } from "node:path";
+import { probeSettings } from "./types.js";
 import { loadConfig } from "./config.js";
-import { wilsonInterval } from "./stats.js";
-import { kToCertify } from "./report.js";
+import { kToCertify } from "./stats.js";
 import { getAdapter } from "./adapters/index.js";
-const NAME_RE = /^name:\s*(.*)$/m;
-const DESC_RE = /^description:\s*(.*)$/m;
+import { readSkillFile, skillFolders, skillsRoot } from "./skills.js";
+import { loadProblems } from "./orchestrator.js";
+import { withSandbox } from "./sandbox.js";
 const MIN_NODE = { major: 22, minor: 18 }; // package.json engines: ">=22.18"
+const HARMLESS_PROMPT = "Reply with exactly: ok";
 function realCliInstalled(cmd) {
     const r = spawnSync(cmd, ["--version"], { stdio: "ignore", timeout: 15_000 });
     // ENOENT = not on PATH. Any other result (even a non-zero exit) means the binary exists.
@@ -51,40 +53,35 @@ export async function runDoctor(opts, deps = {}) {
     const skillsCwd = cfg ? cfg.cwd : resolve(opts.cwd);
     const runtime = cfg?.runtime ?? opts.runtime ?? "claude-code";
     // 3-6. Skills directory + each SKILL.md
-    const skillNames = new Set();
-    const skillsRoot = join(skillsCwd, ".claude", "skills");
-    if (!existsSync(skillsRoot)) {
+    const library = [];
+    const root = skillsRoot(skillsCwd);
+    if (!existsSync(root)) {
         add("fail", `no .claude/skills/ directory under ${skillsCwd}`);
     }
     else {
-        const dirs = readdirSync(skillsRoot, { withFileTypes: true }).filter((d) => d.isDirectory());
+        const dirs = skillFolders(skillsCwd);
         if (dirs.length === 0) {
             add("fail", `.claude/skills/ exists but contains no skill folders`);
         }
-        let valid = 0;
-        for (const d of dirs) {
-            const md = join(skillsRoot, d.name, "SKILL.md");
+        for (const name of dirs) {
+            const md = join(root, name, "SKILL.md");
             if (!existsSync(md)) {
-                add("fail", `skill "${d.name}" missing SKILL.md`);
+                add("fail", `skill "${name}" missing SKILL.md`);
                 continue;
             }
-            const text = readFileSync(md, "utf8");
-            const name = NAME_RE.exec(text)?.[1]?.trim() ?? "";
-            const desc = DESC_RE.exec(text)?.[1]?.trim() ?? "";
-            if (!name)
-                add("fail", `skill "${d.name}" missing name:`);
-            else if (name !== d.name)
-                add("warn", `skill "${d.name}" has name: "${name}" (doesn't match folder name)`);
-            if (!desc)
-                add("fail", `skill "${d.name}" missing description:`);
-            else if (desc === "|" || desc === ">")
-                add("warn", `skill "${d.name}" uses a multi-line description block — a single-line description triggers more reliably`);
-            skillNames.add(d.name);
-            valid++;
+            const s = readSkillFile(md, name);
+            if (!s.frontmatterName)
+                add("fail", `skill "${name}" missing name: (in a --- frontmatter block)`);
+            else if (s.frontmatterName !== name)
+                add("warn", `skill "${name}" has name: "${s.frontmatterName}" (doesn't match folder name)`);
+            if (!s.description)
+                add("fail", `skill "${name}" missing description: (in a --- frontmatter block)`);
+            library.push(s);
         }
-        if (valid > 0)
-            add("pass", `found .claude/skills/ with ${valid} skill${valid === 1 ? "" : "s"}`);
+        if (library.length > 0)
+            add("pass", `found .claude/skills/ with ${library.length} skill${library.length === 1 ? "" : "s"}`);
     }
+    const skillNames = new Set(library.map((s) => s.name));
     // 7-9. Config-dependent checks
     if (cfg) {
         const missing = cfg.cases
@@ -98,13 +95,18 @@ export async function runDoctor(opts, deps = {}) {
             add("warn", `config has only decoy cases — no skill is being verified to fire`);
         }
         // a strict threshold at a small k can be mathematically impossible to certify
-        if (wilsonInterval(cfg.k, cfg.k, cfg.conf).lo <= cfg.threshold) {
+        const need = kToCertify(cfg.threshold, cfg.conf);
+        if (need > cfg.k) {
             add("warn", `threshold ${Math.round(cfg.threshold * 100)}% with k=${cfg.k} cannot certify a pass ` +
-                `even on a perfect run — need ~k=${kToCertify(cfg.threshold, cfg.conf)} (or lower the threshold)`);
+                `even on a perfect run — need k≥${need} (or lower the threshold)`);
+        }
+        if (cfg.userConfig) {
+            add("warn", `userConfig is on: probes load your personal runtime config, so results won't reproduce in CI`);
         }
     }
     // 10-11. Runtime + a harmless live probe (verifies the CLI is installed AND authed)
-    add("info", `runtime: ${runtime}  model: ${cfg?.model ?? opts.model ?? "(runtime default)"}`);
+    const model = cfg?.model ?? opts.model;
+    add("info", `runtime: ${runtime}  model: ${model ?? "(runtime default)"}`);
     const known = runtime === "claude-code" || runtime === "opencode";
     const bin = runtime === "claude-code" ? "claude" : runtime === "opencode" ? "opencode" : null;
     let cliOk = false;
@@ -122,14 +124,46 @@ export async function runDoctor(opts, deps = {}) {
         add("info", `probe skipped (--skip-probe)`);
     }
     else if (known && cliOk) {
-        const r = await probe(runtime, "Reply with exactly: ok", {
-            cwd: skillsCwd, timeoutMs: 60_000,
-            ...(cfg?.model ?? opts.model ? { model: (cfg?.model ?? opts.model) } : {}),
-        });
-        if (r.status === "ok")
-            add("pass", `${runtime} probe succeeded (CLI is authenticated)`);
-        else
+        const settings = {
+            ...(cfg ? probeSettings(cfg) : {}),
+            ...(model ? { model } : {}),
+            ...(opts.userConfig ? { userConfig: true } : {}),
+            timeoutMs: 60_000,
+        };
+        const r = await probe(runtime, HARMLESS_PROMPT, { cwd: skillsCwd, ...settings });
+        if (r.status !== "ok") {
             add("fail", `${runtime} probe failed — check auth/login: ${r.error ?? "unknown error"}`);
+        }
+        else {
+            add("pass", `${runtime} probe succeeded (CLI is authenticated)` +
+                (r.env?.runtimeVersion ? ` — ${bin} ${r.env.runtimeVersion}` : "") +
+                (r.env?.model && !model ? `, default model ${r.env.model}` : ""));
+            // what the runtime ACTUALLY loads (only runtimes that report it)
+            if (r.env?.loadedSkills && library.length) {
+                const { missing } = loadProblems(r.env, library.map((s) => s.name), library);
+                if (missing.length) {
+                    add("fail", `the runtime did not load: ${missing.join(", ")} — it rejected or ignored these ` +
+                        `SKILL.md files, so they can never fire`);
+                }
+                else {
+                    add("pass", `the runtime loads all ${library.length} library skill${library.length === 1 ? "" : "s"}`);
+                }
+                // built-ins load in every probe; a library skill sharing a built-in's name is ambiguous
+                const empty = await withSandbox(skillsCwd, [], (sb) => probe(runtime, HARMLESS_PROMPT, { cwd: sb.cwd, ...settings }));
+                const builtins = empty.status === "ok" ? empty.env?.loadedSkills ?? [] : [];
+                const clash = library.filter((s) => builtins.includes(s.name) ||
+                    (s.frontmatterName !== undefined && builtins.includes(s.frontmatterName)));
+                for (const s of clash) {
+                    add("warn", `skill "${s.name}" has the same name as a ${runtime} built-in skill — the two compete ` +
+                        `for the same trigger; consider renaming it`);
+                }
+                if (builtins.length) {
+                    add("info", `${builtins.length} ${runtime} built-in skill(s) load alongside your library in every probe ` +
+                        `(${builtins.slice(0, 8).join(", ")}${builtins.length > 8 ? ", …" : ""}) — they are part of the real ` +
+                        `environment and can steal triggers`);
+                }
+            }
+        }
     }
     const hasFail = checks.some((c) => c.status === "fail");
     const hasWarn = checks.some((c) => c.status === "warn");

package/dist/eval.d.ts

@@ -1,8 +1,9 @@
 /** The audit: run every case co-loaded, compute reliability (with CI), detect trigger-theft,
  * and keep infrastructure failure distinct from behavioral failure (finding #1). */
-import type { Config } from "./types.ts";
+import { type Config } from "./types.ts";
 import { reliability } from "./stats.ts";
-import { type RunStats } from "./orchestrator.ts";
+import { type Budget, type RunStats } from "./orchestrator.ts";
+import { type RunManifest } from "./manifest.ts";
 /** Four honest states. `error` = the runtime failed enough that the measurement is untrustworthy. */
 export type Verdict = "pass" | "fail" | "inconclusive" | "error";
 export interface CaseResult {
@@ -10,10 +11,14 @@ export interface CaseResult {
     expected: string | null;
     stats: RunStats;
     reliability: ReturnType<typeof reliability>;
-    /** non-expected, non-None skills that fired = interference / trigger-theft */
+    /** non-expected, non-None skills that fired = interference / trigger-theft. A warning unless
+     * failOnTheft is set: one stray firing out of k is not statistical evidence of a problem. */
     theft: string[];
     decision: "above" | "below" | "undecided";
     verdict: Verdict;
+    /** the runtime reported its loaded skills and the expected skill wasn't among them — it can
+     * never fire (usually a SKILL.md the runtime rejected) */
+    notLoaded?: boolean;
 }
 export interface AuditResult {
     runtime: string;
@@ -27,8 +32,12 @@ export interface AuditResult {
         inconclusive: number;
         error: number;
     };
-    /** 0 = all pass; 1 = a behavioral failure/theft; 2 = inconclusive or infrastructure error */
+    /** 0 = all pass; 1 = a behavioral failure; 2 = inconclusive or infrastructure error */
     exitCode: 0 | 1 | 2;
+    /** who/what/when of this run — makes reports citable and baselines comparable */
+    manifest: RunManifest;
+    /** run-level problems worth reading before the verdicts (unloaded skills, personal config…) */
+    warnings: string[];
 }
 export interface ProgressEvent {
     caseIndex: number;
@@ -37,4 +46,10 @@ export interface ProgressEvent {
     validN: number;
     maxK: number;
 }
-export declare function runAudit(cfg: Config, onProgress?: (e: ProgressEvent) => void): Promise<AuditResult>;
+export interface RunOpts {
+    onProgress?: (e: ProgressEvent) => void;
+    /** cases probed in parallel (default 1) */
+    concurrency?: number;
+    budget?: Budget;
+}
+export declare function runAudit(cfg: Config, optsOrProgress?: RunOpts | ((e: ProgressEvent) => void)): Promise<AuditResult>;

package/dist/eval.js

@@ -1,12 +1,17 @@
-import { reliability, sequentialDecision } from "./stats.js";
-import { measure } from "./orchestrator.js";
+/** The audit: run every case co-loaded, compute reliability (with CI), detect trigger-theft,
+ * and keep infrastructure failure distinct from behavioral failure (finding #1). */
+import { probeSettings } from "./types.js";
+import { reliability, planDecision, sequentialPlan } from "./stats.js";
+import { measure, mapPool, infraUntrustworthy, loadProblems, extraSkills, } from "./orchestrator.js";
 import { getAdapter } from "./adapters/index.js";
-function verdictFor(stats, decision, theft) {
+import { buildManifest, buildEnvironment, capabilityWarnings } from "./manifest.js";
+import { listSkills } from "./skills.js";
+function verdictFor(stats, decision, theft, failOnTheft) {
     // `error` means INFRASTRUCTURE failures dominate — untrustworthy. It is NOT about small k:
     // a small-but-all-valid run is `inconclusive` (wide CI), which the report explains.
-    if (stats.n === 0 || (stats.errors > 0 && stats.errors >= stats.n))
+    if (infraUntrustworthy(stats))
         return "error";
-    if (theft.length > 0)
+    if (failOnTheft && theft.length > 0)
         return "fail";
     if (decision === "above")
         return "pass";
@@ -14,34 +19,48 @@ function verdictFor(stats, decision, theft) {
         return "fail";
     return "inconclusive"; // includes "k too small to decide" — report says how many runs are needed
 }
-export async function runAudit(cfg, onProgress) {
+/** The library at cfg.cwd, or null when there's no readable skills dir (load checks are skipped). */
+function tryListSkills(cwd) {
+    try {
+        return listSkills(cwd);
+    }
+    catch {
+        return null;
+    }
+}
+export async function runAudit(cfg, optsOrProgress = {}) {
+    const opts = typeof optsOrProgress === "function" ? { onProgress: optsOrProgress } : optsOrProgress;
     const adapter = getAdapter(cfg.runtime);
-    const cases = [];
-    let totalCost = 0;
-    for (let ci = 0; ci < cfg.cases.length; ci++) {
-        const c = cfg.cases[ci];
+    const library = tryListSkills(cfg.cwd);
+    const plan = sequentialPlan(cfg.k, cfg.threshold, cfg.conf);
+    const settings = probeSettings(cfg);
+    const onProgress = opts.onProgress;
+    const cases = await mapPool(cfg.cases, opts.concurrency ?? 1, async (c, ci) => {
         const stats = await measure(adapter, c.prompt, c.expected, {
-            cwd: cfg.cwd, maxK: cfg.k, threshold: cfg.threshold, conf: cfg.conf,
-            ...(cfg.model ? { model: cfg.model } : {}),
+            cwd: cfg.cwd, maxK: cfg.k, threshold: cfg.threshold, conf: cfg.conf, ...settings,
+            ...(opts.budget ? { budget: opts.budget } : {}),
             ...(onProgress ? {
                 onProbe: (validN, maxK) => onProgress({
                     caseIndex: ci + 1, caseTotal: cfg.cases.length, prompt: c.prompt, validN, maxK,
                 }),
             } : {}),
         });
-        totalCost += stats.totalCost;
         const rel = reliability(stats.hits, stats.n, cfg.conf);
         const want = c.expected ?? "None";
         const theft = Object.keys(stats.dist).filter((s) => s !== want && s !== "None");
-        const decision = sequentialDecision(stats.hits, stats.n, cfg.threshold, cfg.conf);
-        cases.push({
+        const decision = planDecision(plan, stats.hits, stats.n);
+        const notLoaded = library !== null && c.expected !== null &&
+            loadProblems(stats.env, [c.expected], library.filter((s) => s.name === c.expected)).missing.length > 0;
+        return {
             prompt: c.prompt, expected: c.expected, stats, reliability: rel,
-            theft, decision, verdict: verdictFor(stats, decision, theft),
-        });
-    }
-    // NOTE: Benjamini-Hochberg multiplicity control over per-case exact-test p-values lands in M2,
-    // once each case carries a real p-value rather than a CI decision. The functions are tested
-    // in stats.ts but are NOT yet wired into this audit path.
+            theft, decision, verdict: verdictFor(stats, decision, theft, Boolean(cfg.failOnTheft)),
+            ...(notLoaded ? { notLoaded } : {}),
+        };
+    });
+    // NOTE: per-case verdicts here are sequential tests vs the threshold, each with its own exact
+    // error bound — not a family of p-values to BH-correct. Where families of Fisher tests exist
+    // (context, ablation, baseline comparison) the correction is applied.
+    const totalCost = cases.reduce((s, c) => s + c.stats.totalCost, 0);
     const counts = {
         pass: cases.filter((c) => c.verdict === "pass").length,
         fail: cases.filter((c) => c.verdict === "fail").length,
@@ -49,8 +68,24 @@ export async function runAudit(cfg, onProgress) {
         error: cases.filter((c) => c.verdict === "error").length,
     };
     const exitCode = counts.fail > 0 ? 1 : (counts.error > 0 || counts.inconclusive > 0) ? 2 : 0;
+    const env = cases.find((c) => c.stats.env)?.stats.env;
+    const warnings = capabilityWarnings(cfg, adapter);
+    const unloaded = [...new Set(cases.filter((c) => c.notLoaded).map((c) => c.expected))];
+    if (unloaded.length) {
+        warnings.push(`the runtime did not load: ${unloaded.join(", ")} — these can never fire ` +
+            `(check each SKILL.md's frontmatter; run \`skill-probe doctor\`)`);
+    }
+    if (cfg.userConfig) {
+        warnings.push(`probes loaded your personal runtime config (--user-config): results reflect this ` +
+            `machine and won't reproduce in CI`);
+    }
+    if (cases.some((c) => c.stats.budgetExhausted)) {
+        warnings.push(`--max-cost budget reached — cases cut short are inconclusive (error if they never started), never a pass`);
+    }
     return {
         runtime: cfg.runtime, model: cfg.model ?? "(runtime default)",
         threshold: cfg.threshold, cases, totalCost, counts, exitCode,
+        manifest: buildManifest("audit", cfg, buildEnvironment(cfg, env, library ? extraSkills(env, library) : undefined, adapter)),
+        warnings,
     };
 }

package/dist/fix.d.ts

@@ -1,9 +1,6 @@
-import type { Config } from "./types.ts";
+import { type Config } from "./types.ts";
 import { reliability } from "./stats.ts";
-export declare function readDesc(file: string): string;
-/** Replace only the single-line description; backups are handled explicitly in runFix (not here,
- * because the interleaved loop calls this many times and must not snapshot intermediate states). */
-export declare function writeDesc(file: string, desc: string): void;
+import { type Budget } from "./orchestrator.ts";
 export interface Sibling {
     name: string;
     desc: string;
@@ -22,14 +19,20 @@ export interface FixResult {
     errors: number;
     applied: boolean;
     applyBar: number;
-    /** path of the kept backup (only when applied); null when reverted (no change made) */
+    /** path of the kept backup (only when applied); null when nothing was written */
     backupPath: string | null;
+    /** why the rewrite wasn't tested or applied, beyond the statistics */
+    note?: string;
+    totalCost: number;
 }
 export interface FixOpts {
     skill: string;
     k: number;
     applyBar: number;
     model?: string;
+    /** paired runs in flight at once (each pair is 2 probes; default 1) */
+    concurrency?: number;
+    budget?: Budget;
 }
 export interface FixDeps {
     rewrite?: (skill: string, cur: string, prompts: string[], siblings: Sibling[], model?: string) => Promise<string>;

package/dist/fix.js

@@ -1,31 +1,21 @@
 /** `skill-probe fix` — rewrite a failing skill's description, then PROVE the lift is real with
  * the Bayesian Beta-Binomial (not "it looks better"). Upgrades on the Python prototype:
- *  - INTERLEAVED before/after (old desc -> probe -> new desc -> probe, paired) controls for
- *    session drift at the same probe cost as block-wise.
+ *  - PAIRED before/after: two identical sandboxes of the library, one with the old description and
+ *    one with the new; each pair of probes runs at the same moment, so session/model drift hits
+ *    both arms equally.
  *  - COLLISION-AWARE rewrite: the LLM is told sibling descriptions so it won't steal triggers.
- *  - Applies the new description only if P(improvement) >= bar AND the effect is positive;
- *    otherwise reverts. Original is always backed up to SKILL.md.bak. */
-import { readFileSync, writeFileSync, existsSync, readdirSync, copyFileSync, unlinkSync } from "node:fs";
-import { join } from "node:path";
+ *  - The real SKILL.md is never touched during the experiment (an interrupted run can't leave it
+ *    half-edited). The new description is written once, at the end, only if P(improvement) >= bar
+ *    AND the effect is positive AND the file hasn't changed on disk since the run started — with
+ *    the exact original snapshotted to a timestamped backup first. */
+import { readFileSync, writeFileSync, existsSync, copyFileSync, rmSync } from "node:fs";
+import { probeSettings } from "./types.js";
 import { getAdapter } from "./adapters/index.js";
 import { probImprovement, reliability } from "./stats.js";
 import { complete } from "./llm.js";
-const DESC_RE = /^description:\s*(.*)$/m;
-function skillFile(cfg, name) {
-    return join(cfg.cwd, ".claude", "skills", name, "SKILL.md");
-}
-export function readDesc(file) {
-    const m = DESC_RE.exec(readFileSync(file, "utf8"));
-    if (!m)
-        throw new Error(`no single-line 'description:' found in ${file} (multi-line block scalars not yet supported)`);
-    return m[1].trim();
-}
-/** Replace only the single-line description; backups are handled explicitly in runFix (not here,
- * because the interleaved loop calls this many times and must not snapshot intermediate states). */
-export function writeDesc(file, desc) {
-    const txt = readFileSync(file, "utf8");
-    writeFileSync(file, txt.replace(DESC_RE, `description: ${desc}`));
-}
+import { mapPool } from "./orchestrator.js";
+import { listSkills, parseFrontmatter, replaceDescription, normalizeDescription, skillFile } from "./skills.js";
+import { buildSandbox } from "./sandbox.js";
 /** Unique per-run snapshot path, never clobbering a prior run's backup. */
 function backupPathFor(file) {
     const base = `${file}.bak.${Date.now()}`;
@@ -49,63 +39,85 @@ export async function defaultRewrite(skill, cur, prompts, siblings, model) {
 }
 export async function runFix(cfg, opts, deps = {}) {
     const adapter = getAdapter(cfg.runtime);
-    const file = skillFile(cfg, opts.skill);
+    const file = skillFile(cfg.cwd, opts.skill);
     if (!existsSync(file))
         throw new Error(`skill not found: ${file}`);
-    const root = join(cfg.cwd, ".claude", "skills");
-    const siblings = readdirSync(root)
-        .filter((n) => n !== opts.skill && existsSync(join(root, n, "SKILL.md")))
-        .map((n) => ({ name: n, desc: readDesc(join(root, n, "SKILL.md")) }));
+    const library = listSkills(cfg.cwd);
+    const siblings = library
+        .filter((s) => s.name !== opts.skill)
+        .map((s) => ({ name: s.name, desc: s.description }));
     const prompts = cfg.cases.filter((c) => c.expected === opts.skill).map((c) => c.prompt);
     if (prompts.length === 0)
         throw new Error(`no cases in the config expect skill '${opts.skill}'`);
-    const oldDesc = readDesc(file);
+    const original = readFileSync(file, "utf8");
+    const fm = parseFrontmatter(original);
+    if (!fm.ok)
+        throw new Error(`${file} has no \`---\` frontmatter block — can't rewrite its description`);
+    const oldDesc = normalizeDescription(fm.fields["description"] ?? "");
+    replaceDescription(original, oldDesc || "x"); // throws now (before paying for a rewrite) if it can't be rewritten
     const rewrite = deps.rewrite ?? defaultRewrite;
-    const newDesc = await rewrite(opts.skill, oldDesc, prompts, siblings, opts.model);
-    // Snapshot the EXACT pre-fix file once, to a unique path (never clobber a prior run's backup).
-    const backupPath = backupPathFor(file);
-    copyFileSync(file, backupPath);
-    const probeWith = async (desc, prompt) => {
-        writeDesc(file, desc);
-        return adapter.probe(prompt, { cwd: cfg.cwd, ...(cfg.model ? { model: cfg.model } : {}) });
+    const newDesc = normalizeDescription(await rewrite(opts.skill, oldDesc, prompts, siblings, opts.model));
+    const base = {
+        skill: opts.skill, oldDesc, newDesc, applyBar: opts.applyBar,
+        before: reliability(0, 0), after: reliability(0, 0),
+        pImprove: 0, deltaMean: 0, deltaCi: [0, 0],
+        n: 0, errors: 0, applied: false, backupPath: null, totalCost: 0,
     };
-    let beforeHits = 0, afterHits = 0, n = 0, errors = 0;
+    if (!newDesc)
+        return { ...base, note: "the rewrite came back empty — nothing to test" };
+    if (newDesc === oldDesc)
+        return { ...base, note: "the rewrite is identical to the current description — nothing to test" };
+    const names = library.map((s) => s.name);
+    const beforeBox = buildSandbox(cfg.cwd, names); // holds the exact original SKILL.md
+    const afterBox = buildSandbox(cfg.cwd, names);
+    let beforeHits = 0, afterHits = 0, n = 0, errors = 0, totalCost = 0;
     try {
-        for (let i = 0; i < opts.k; i++) {
-            for (const p of prompts) {
-                const rb = await probeWith(oldDesc, p);
-                const ra = await probeWith(newDesc, p);
-                if (rb.status === "error" || ra.status === "error") {
-                    errors++;
-                    continue;
-                } // drop incomplete pair
-                if (rb.skillFired === opts.skill)
-                    beforeHits++;
-                if (ra.skillFired === opts.skill)
-                    afterHits++;
-                n++;
+        const afterFile = skillFile(afterBox.cwd, opts.skill);
+        rmSync(afterFile, { force: true }); // a fresh file — never write through a link into the real library
+        writeFileSync(afterFile, replaceDescription(original, newDesc));
+        const settings = probeSettings(cfg);
+        const jobs = Array.from({ length: opts.k }, () => prompts).flat();
+        await mapPool(jobs, opts.concurrency ?? 1, async (p) => {
+            if (opts.budget?.exhausted) {
+                errors++;
+                return;
             }
-        }
-        const imp = probImprovement(beforeHits, n, afterHits, n);
-        const applied = n > 0 && imp.pImprove >= opts.applyBar && imp.deltaMean > 0;
-        if (applied) {
-            writeDesc(file, newDesc); // commit; keep the timestamped backup
-        }
-        else {
-            copyFileSync(backupPath, file); // restore exact original
-            unlinkSync(backupPath); // no change made -> don't leave a redundant backup
-        }
-        return {
-            skill: opts.skill, oldDesc, newDesc,
-            before: reliability(beforeHits, n), after: reliability(afterHits, n),
-            pImprove: imp.pImprove, deltaMean: imp.deltaMean, deltaCi: imp.deltaCi,
-            n, errors, applied, applyBar: opts.applyBar,
-            backupPath: applied ? backupPath : null,
-        };
+            const [rb, ra] = await Promise.all([
+                adapter.probe(p, { cwd: beforeBox.cwd, ...settings }),
+                adapter.probe(p, { cwd: afterBox.cwd, ...settings }),
+            ]);
+            totalCost += rb.cost + ra.cost;
+            opts.budget?.add(rb.cost + ra.cost);
+            if (rb.status === "error" || ra.status === "error") {
+                errors++;
+                return;
+            } // drop incomplete pair
+            if (rb.skillFired === opts.skill)
+                beforeHits++;
+            if (ra.skillFired === opts.skill)
+                afterHits++;
+            n++;
+        });
     }
-    catch (e) {
-        copyFileSync(backupPath, file); // restore exact pre-fix file on any error
-        unlinkSync(backupPath);
-        throw e;
+    finally {
+        beforeBox.cleanup();
+        afterBox.cleanup();
     }
+    const imp = probImprovement(beforeHits, n, afterHits, n);
+    const result = {
+        ...base,
+        before: reliability(beforeHits, n), after: reliability(afterHits, n),
+        pImprove: imp.pImprove, deltaMean: imp.deltaMean, deltaCi: imp.deltaCi,
+        n, errors, totalCost,
+    };
+    if (!(n > 0 && imp.pImprove >= opts.applyBar && imp.deltaMean > 0))
+        return result;
+    // commit: only if the file is exactly what we measured against (someone may have edited it)
+    if (readFileSync(file, "utf8") !== original) {
+        return { ...result, note: `${file} changed on disk during the run — not overwriting it; re-run fix` };
+    }
+    const backupPath = backupPathFor(file);
+    copyFileSync(file, backupPath);
+    writeFileSync(file, replaceDescription(original, newDesc));
+    return { ...result, applied: true, backupPath };
 }

package/dist/gen.d.ts

@@ -1,9 +1,10 @@
+/** `skill-probe gen` — draft a test config from the skills in a project. The skill descriptions
+ * already say what each skill is for, so an LLM can propose realistic should-fire prompts,
+ * cross-skill near-misses (to surface mis-routing), and decoys. ALWAYS human-reviewed before use
+ * (the generator tends toward obvious keyword-matches; you add the messy real-world phrasings). */
 import type { Config, EvalCase } from "./types.ts";
-export interface SkillInfo {
-    name: string;
-    description: string;
-}
-export declare function listSkills(cwd: string): SkillInfo[];
+import type { SkillInfo } from "./skills.ts";
+export { listSkills, type SkillInfo } from "./skills.ts";
 export interface GenOpts {
     perSkill: number;
     decoys: number;

package/dist/gen.js

@@ -1,25 +1,5 @@
-/** `skill-probe gen` — draft a test config from the skills in a project. The skill descriptions
- * already say what each skill is for, so an LLM can propose realistic should-fire prompts,
- * cross-skill near-misses (to surface mis-routing), and decoys. ALWAYS human-reviewed before use
- * (the generator tends toward obvious keyword-matches; you add the messy real-world phrasings). */
-import { readFileSync, existsSync, readdirSync } from "node:fs";
-import { join } from "node:path";
 import { complete } from "./llm.js";
-const DESC_RE = /^description:\s*(.*)$/m;
-export function listSkills(cwd) {
-    const root = join(cwd, ".claude", "skills");
-    if (!existsSync(root))
-        throw new Error(`no .claude/skills/ directory under ${cwd}`);
-    const skills = [];
-    for (const name of readdirSync(root)) {
-        const file = join(root, name, "SKILL.md");
-        if (!existsSync(file))
-            continue;
-        const m = DESC_RE.exec(readFileSync(file, "utf8"));
-        skills.push({ name, description: m ? m[1].trim() : "" });
-    }
-    return skills;
-}
+export { listSkills } from "./skills.js";
 export async function defaultGenerate(skills, opts) {
     const list = skills.map((s) => `- ${s.name}: ${s.description}`).join("\n");
     const sys = "You build a TEST SUITE that audits whether an AI coding agent loads the correct "

package/dist/llm.d.ts

@@ -13,3 +13,5 @@ export declare function interpretClaudeJson(code: number | null, out: string, er
     text?: string;
     error?: string;
 };
+/** CLI arguments for one completion. Exported for testing. */
+export declare function claudeCliArgs(system: string, user: string, opts: CompleteOpts): string[];