skill-probe 0.8.0 → 0.10.0

package/dist/report.js

@@ -1,24 +1,47 @@
 /** Human table + machine JSON rendering. Every number ships with its CI and k; infrastructure
  * errors are shown distinctly from behavioral results. */
-import { wilsonInterval } from "./stats.js";
+import { kToCertify } from "./stats.js";
+export { kToCertify };
 const pct = (x) => `${Math.round(x * 100)}%`;
 const pval = (p) => (p < 0.001 ? "<0.001" : p.toFixed(3));
 const LABEL = {
     pass: "PASS        ", fail: "FAIL        ",
     inconclusive: "INCONCLUSIVE", error: "ERROR       ",
 };
-/** Smallest k at which an all-success run could certify reliability >= threshold. */
-export function kToCertify(threshold, conf) {
-    for (let k = 1; k <= 500; k++) {
-        if (wilsonInterval(k, k, conf).lo > threshold)
-            return k;
+/** One line describing what the runtime actually ran with, when it told us. */
+function environmentLine(m) {
+    const e = m?.environment;
+    if (!e)
+        return null;
+    const bits = [];
+    if (e.runtimeVersion)
+        bits.push(`runtime ${e.runtimeVersion}`);
+    if (e.resolvedModel && m.model === "(runtime default)")
+        bits.push(`model resolved to ${e.resolvedModel}`);
+    if (e.extraSkills) {
+        bits.push(e.extraSkills.length
+            ? `${e.extraSkills.length} skill(s) outside the library also loaded (${e.extraSkills.slice(0, 6).join(", ")}${e.extraSkills.length > 6 ? ", …" : ""})`
+            : "no skills outside the library loaded");
     }
-    return 500;
+    bits.push(e.userConfig ? "personal runtime config INCLUDED"
+        : e.hermetic ? "hermetic (personal runtime config excluded)"
+            : "personal runtime config NOT isolated (this runtime can't exclude it)");
+    return `  ${bits.join(" · ")}`;
+}
+function theftLine(c) {
+    const counts = c.theft.map((s) => `${s} ${c.stats.dist[s]}/${c.stats.n}`).join(", ");
+    return `        ⚠ trigger-theft by: ${counts}` +
+        (c.verdict === "pass" ? " (reliability still clears the bar; --fail-on-theft makes this fail)" : "");
 }
 export function renderTable(a, opts = {}) {
     const showCost = opts.showCost ?? true;
     const out = [];
     out.push(`skill-probe — runtime: ${a.runtime}  model: ${a.model}  threshold: ${pct(a.threshold)}`);
+    const env = environmentLine(a.manifest);
+    if (env)
+        out.push(env);
+    for (const w of a.warnings ?? [])
+        out.push(`  ⚠ ${w}`);
     out.push("");
     for (const c of a.cases) {
         const r = c.reliability;
@@ -35,11 +58,17 @@ export function renderTable(a, opts = {}) {
         const dist = Object.entries(c.stats.dist)
             .sort((p, q) => q[1] - p[1]).map(([s, n]) => `${s}×${n}`).join(", ");
         out.push(`        outcomes: ${dist}`);
+        if (c.notLoaded)
+            out.push(`        ⛔ the runtime never loaded "${c.expected}" — it cannot fire (check its SKILL.md)`);
         if (c.theft.length)
-            out.push(`        ⚠ trigger-theft by: ${c.theft.join(", ")}`);
+            out.push(theftLine(c));
+        if (c.expected !== null && c.stats.preempted > 0 && c.verdict !== "pass") {
+            out.push(`        ⓘ ${c.stats.preempted}/${c.stats.n} probes called another tool first and loaded no skill — ` +
+                `activation may be under-counted (try --max-turns 2)`);
+        }
         if (c.verdict === "inconclusive") {
-            out.push(`        ↳ CI straddles threshold — need ~k=${kToCertify(a.threshold, r.conf)} ` +
-                `all-success runs to certify ≥${pct(a.threshold)} (currently k=${r.k})`);
+            out.push(`        ↳ not statistically decided at k=${r.k} — certifying ≥${pct(a.threshold)} needs ` +
+                `k≥${kToCertify(a.threshold, r.conf)} with every run firing (more if some miss)`);
         }
     }
     out.push("");
@@ -57,6 +86,11 @@ export function renderMarkdown(a, opts = {}) {
     const out = [];
     const conf = a.cases[0]?.reliability.conf ?? 0.95;
     out.push(`### skill-probe — \`${a.runtime}\` · model \`${a.model}\` · threshold ${pct(a.threshold)}`);
+    const env = environmentLine(a.manifest);
+    if (env)
+        out.push("", `_${env.trim()}_`);
+    for (const w of a.warnings ?? [])
+        out.push("", `> ⚠ ${w}`);
     out.push("");
     out.push(`| Verdict | Skill | Reliability (${pct(conf)} CI) | k | Notes |`);
     out.push("|---|---|---|---|---|");
@@ -66,10 +100,12 @@ export function renderMarkdown(a, opts = {}) {
         let notes = "";
         if (c.verdict === "error")
             notes = `infra error: ${c.stats.lastError ?? "untrustworthy"}`;
+        else if (c.notLoaded)
+            notes = `⛔ runtime never loaded \`${c.expected}\``;
         else if (c.theft.length)
-            notes = `⚠ trigger-theft by ${c.theft.join(", ")}`;
+            notes = `⚠ trigger-theft by ${c.theft.map((s) => `${s} ${c.stats.dist[s]}/${c.stats.n}`).join(", ")}`;
         else if (c.verdict === "inconclusive")
-            notes = `need ~k=${kToCertify(a.threshold, r.conf)} to certify`;
+            notes = `need k≥${kToCertify(a.threshold, r.conf)} to certify`;
         const rel = c.verdict === "error" ? "—" : `${pct(r.pHat)} [${pct(r.ciLow)}, ${pct(r.ciHigh)}]`;
         out.push(`| ${mark[c.verdict]} | \`${expect}\` | ${rel} | ${r.k} | ${notes} |`);
     }
@@ -91,6 +127,11 @@ export function renderDiagnose(d, opts = {}) {
     const out = [];
     out.push(`skill-probe diagnose — runtime: ${d.runtime}  model: ${d.model}  threshold: ${pct(d.threshold)}`);
     out.push(`  actual = real activation · intended = forced-choice routing (does the model even pick it?)`);
+    const env = environmentLine(d.manifest);
+    if (env)
+        out.push(env);
+    for (const w of d.warnings ?? [])
+        out.push(`  ⚠ ${w}`);
     out.push("");
     const dist = (m) => Object.entries(m).sort((p, q) => q[1] - p[1]).map(([s, n]) => `${s}×${n}`).join(", ") || "(none)";
     for (const c of d.cases) {
@@ -131,6 +172,12 @@ export function renderContext(a, opts = {}) {
     out.push(`skill-probe context — runtime: ${a.runtime}  model: ${a.model}  ` +
         `threshold: ${pct(a.threshold)}  library: ${a.librarySize} skills`);
     out.push(`  isolation = only that skill loaded · co-loaded = the full library (${a.librarySize}) loaded`);
+    out.push(`  every condition runs in an identical sandbox (skills + project CLAUDE.md/AGENTS.md) — only the skill set differs`);
+    const env = environmentLine(a.manifest);
+    if (env)
+        out.push(env);
+    for (const w of a.warnings ?? [])
+        out.push(`  ⚠ ${w}`);
     const familyN = a.cases.filter((c) => !c.untrustworthy).length;
     if (familyN > 1) {
         out.push(`  ${familyN} comparisons — interference decided on Benjamini-Hochberg adjusted p<0.05` +
@@ -141,6 +188,11 @@ export function renderContext(a, opts = {}) {
     for (const c of a.cases) {
         if (c.untrustworthy) {
             out.push(`  [⛔ ERROR]  ${c.expected}  | ${c.prompt}`);
+            if (c.loadIssue) {
+                out.push(`        the runtime loaded the wrong skill set — comparison NOT trustworthy`);
+                out.push(`        ${c.loadIssue}`);
+                continue;
+            }
             out.push(`        infrastructure errors dominated — comparison NOT trustworthy ` +
                 `(isolation ${c.isolation.stats.errors} err/${c.isolation.stats.n} valid, ` +
                 `co-loaded ${c.coLoaded.stats.errors} err/${c.coLoaded.stats.n} valid)`);
@@ -156,12 +208,18 @@ export function renderContext(a, opts = {}) {
         out.push(`        Δ ${c.deltaP >= 0 ? "+" : ""}${pct(c.deltaP)} under load   Fisher p=${pval(c.fisherP)}` +
             (familyN > 1 ? ` · BH-adj p=${pval(c.fisherPAdj)}` : ""));
         if (c.interference) {
+            const external = new Set(c.externalThieves ?? []);
             const thieves = Object.keys(c.coLoaded.stats.dist)
-                .filter((s) => s !== c.expected && s !== "None");
+                .filter((s) => s !== c.expected && s !== "None")
+                .map((s) => (external.has(s) ? `${s} (outside the library)` : s));
             out.push(`        ↳ fires reliably alone but is suppressed when the library is co-loaded` +
                 (thieves.length ? ` (stolen by: ${thieves.join(", ")})` : ""));
             if (c.ablation) {
-                if (c.ablation.length === 0) {
+                if (c.ablation.length === 0 && external.size) {
+                    out.push(`        leave-one-out: only skills outside the library stole these probes ` +
+                        `(${[...external].join(", ")}) — they can't be removed; narrow or rename "${c.expected}" to stop colliding`);
+                }
+                else if (c.ablation.length === 0) {
                     out.push(`        leave-one-out: no sibling stole these probes (suppressed to None) — ` +
                         `dilution, not theft; nothing to ablate`);
                 }
@@ -169,7 +227,7 @@ export function renderContext(a, opts = {}) {
                     out.push(`        leave-one-out (each suspect removed, recovery vs co-loaded):`);
                     for (const abl of c.ablation) {
                         if (abl.untrustworthy) {
-                            out.push(`          − ${abl.removed}: infra errors dominated — run not trustworthy`);
+                            out.push(`          − ${abl.removed}: ${abl.loadIssue ?? "infra errors dominated"} — run not trustworthy`);
                             continue;
                         }
                         const tag = abl.culprit
@@ -201,6 +259,39 @@ export function renderContext(a, opts = {}) {
         (showCost ? `  |  cost $${a.totalCost.toFixed(4)}` : ""));
     return out.join("\n");
 }
+/** regression gate vs a saved baseline: only significant (BH-corrected) drops fail the build. */
+export function renderBaseline(cmp) {
+    const out = [];
+    out.push(`Baseline gate — vs baseline saved ${cmp.baselineDate} (skill-probe ${cmp.baselineVersion})`);
+    for (const m of cmp.mismatches)
+        out.push(`  ⚠ ${m}`);
+    for (const r of cmp.cases) {
+        if (r.untrustworthy) {
+            out.push(`  [⛔ NOT COMPARABLE]  ${r.expected ?? "(decoy)"}  | ${r.prompt} — a side has no valid probes`);
+            continue;
+        }
+        if (!r.regressed && !r.improved)
+            continue; // stable cases are summarized, not listed
+        const flag = r.regressed ? "▼ REGRESSED" : "▲ improved";
+        out.push(`  [${flag}]  ${r.expected ?? "(decoy)"}  | ${r.prompt}`);
+        out.push(`        ${pct(r.before.pHat)} (${r.before.hits}/${r.before.n}) → ${pct(r.after.pHat)} ` +
+            `(${r.after.hits}/${r.after.n})   Δ${r.delta >= 0 ? "+" : ""}${pct(r.delta)}   ` +
+            `p=${pval(r.fisherP)} · adj ${pval(r.fisherPAdj)}`);
+    }
+    const extras = [];
+    if (cmp.newCases.length)
+        extras.push(`${cmp.newCases.length} new case(s) not in baseline (measured, not gated)`);
+    if (cmp.missingCases.length)
+        extras.push(`${cmp.missingCases.length} baseline case(s) missing from this run`);
+    if (extras.length)
+        out.push(`  ${extras.join(" · ")}`);
+    out.push(cmp.regressions
+        ? `Gate: ▼ FAIL — ${cmp.regressions} significant regression(s)` +
+            (cmp.improvements ? `, ${cmp.improvements} improvement(s)` : "")
+        : `Gate: ✅ no significant regressions (${cmp.cases.length} gated` +
+            (cmp.improvements ? `, ${cmp.improvements} improved — consider re-saving the baseline` : "") + `)`);
+    return out.join("\n");
+}
 export function renderFix(f) {
     const out = [];
     out.push(`skill-probe fix — skill: ${f.skill}`);
@@ -208,19 +299,25 @@ export function renderFix(f) {
     out.push(`  old: ${f.oldDesc}`);
     out.push(`  new: ${f.newDesc}`);
     out.push("");
-    out.push(`  before: ${pct(f.before.pHat)} [${pct(f.before.ciLow)}, ${pct(f.before.ciHigh)}]  ` +
-        `after: ${pct(f.after.pHat)} [${pct(f.after.ciLow)}, ${pct(f.after.ciHigh)}]  ` +
-        `(${f.n} paired runs${f.errors ? `, ${f.errors} dropped to infra errors` : ""})`);
-    out.push(`  P(rewrite improved reliability) = ${pct(f.pImprove)}   ` +
-        `Δ = ${f.deltaMean >= 0 ? "+" : ""}${pct(f.deltaMean)} [${pct(f.deltaCi[0])}, ${pct(f.deltaCi[1])}]`);
-    out.push("");
-    if (f.applied) {
+    if (f.n > 0 || f.errors > 0) {
+        out.push(`  before: ${pct(f.before.pHat)} [${pct(f.before.ciLow)}, ${pct(f.before.ciHigh)}]  ` +
+            `after: ${pct(f.after.pHat)} [${pct(f.after.ciLow)}, ${pct(f.after.ciHigh)}]  ` +
+            `(${f.n} paired runs${f.errors ? `, ${f.errors} dropped to infra errors` : ""})` +
+            (f.totalCost ? `  cost $${f.totalCost.toFixed(4)}` : ""));
+        out.push(`  P(rewrite improved reliability) = ${pct(f.pImprove)}   ` +
+            `Δ = ${f.deltaMean >= 0 ? "+" : ""}${pct(f.deltaMean)} [${pct(f.deltaCi[0])}, ${pct(f.deltaCi[1])}]`);
+        out.push("");
+    }
+    if (f.note) {
+        out.push(`  ↩ NOT APPLIED — ${f.note}. Description unchanged.`);
+    }
+    else if (f.applied) {
         out.push(`  ✅ APPLIED — P(improvement) ${pct(f.pImprove)} ≥ bar ${pct(f.applyBar)} and Δ>0.`);
         out.push(`     original backed up to: ${f.backupPath}`);
     }
     else {
-        out.push(`  ↩ REVERTED — not enough evidence the rewrite helps ` +
-            `(need P(improvement) ≥ ${pct(f.applyBar)} and Δ>0). Description unchanged, no backup kept.`);
+        out.push(`  ↩ NOT APPLIED — not enough evidence the rewrite helps ` +
+            `(need P(improvement) ≥ ${pct(f.applyBar)} and Δ>0). Description unchanged.`);
     }
     return out.join("\n");
 }

package/dist/sandbox.d.ts

@@ -0,0 +1,18 @@
+/** Project instruction files copied into every sandbox, relative to the project root. */
+export declare const INSTRUCTION_FILES: string[];
+export interface Sandbox {
+    cwd: string;
+    skills: string[];
+    cleanup: () => void;
+}
+/** Claude Code creates `<config>/projects/<cwd with non-alphanumerics as "-">/memory/` for every
+ * working directory it starts in, even with --no-session-persistence. Each sandbox is a fresh path,
+ * so without this a big context/ablation run would leave dozens of empty dirs per run. Removes the
+ * dir for THIS sandbox path only, and only if it contains nothing but empty directories — it never
+ * deletes a file. Exported for testing. */
+export declare function removeRuntimeStateFor(sandboxPath: string, configDir?: string): void;
+/** Build a sandbox whose .claude/skills/ holds exactly `skillNames` (copied from srcCwd, with any
+ * supporting files) plus srcCwd's instruction files. Removed on cleanup() or on process exit. */
+export declare function buildSandbox(srcCwd: string, skillNames: string[]): Sandbox;
+/** Run `fn` inside a sandbox, always cleaning up. */
+export declare function withSandbox<T>(srcCwd: string, skillNames: string[], fn: (sb: Sandbox) => Promise<T>): Promise<T>;

package/dist/sandbox.js

@@ -0,0 +1,98 @@
+/** Probe sandboxes: throwaway project dirs holding a chosen subset of the library.
+ *
+ * Every condition that is COMPARED against another (context's isolation / co-loaded / ablation runs,
+ * fix's before / after) must be built the same way, differing ONLY in which skills are present.
+ * Comparing a sandbox against the real project dir confounds the skill set with everything else in
+ * the project (CLAUDE.md, settings, files): a drop caused by the project's CLAUDE.md would read as
+ * "interference", and every leave-one-out run would "recover" and name an innocent sibling.
+ *
+ * Each sandbox gets the project's instruction files (they shape routing, so they're held constant)
+ * but not its settings/hooks/MCP config (side effects; can't run safely from another directory). */
+import { mkdtempSync, mkdirSync, cpSync, rmSync, existsSync, readdirSync, realpathSync, statSync } from "node:fs";
+import { homedir, tmpdir } from "node:os";
+import { dirname, join } from "node:path";
+import { skillsRoot } from "./skills.js";
+import { trackTempDir, untrackTempDir } from "./proc.js";
+/** Project instruction files copied into every sandbox, relative to the project root. */
+export const INSTRUCTION_FILES = ["CLAUDE.md", "AGENTS.md", join(".claude", "CLAUDE.md")];
+/** True if `dir` holds nothing but (recursively) empty directories. */
+function onlyEmptyDirs(dir) {
+    for (const e of readdirSync(dir, { withFileTypes: true })) {
+        if (!e.isDirectory() || !onlyEmptyDirs(join(dir, e.name)))
+            return false;
+    }
+    return true;
+}
+/** Claude Code creates `<config>/projects/<cwd with non-alphanumerics as "-">/memory/` for every
+ * working directory it starts in, even with --no-session-persistence. Each sandbox is a fresh path,
+ * so without this a big context/ablation run would leave dozens of empty dirs per run. Removes the
+ * dir for THIS sandbox path only, and only if it contains nothing but empty directories — it never
+ * deletes a file. Exported for testing. */
+export function removeRuntimeStateFor(sandboxPath, configDir) {
+    const root = join(configDir ?? process.env["CLAUDE_CONFIG_DIR"] ?? join(homedir(), ".claude"), "projects");
+    const paths = new Set([sandboxPath]);
+    try {
+        paths.add(realpathSync(sandboxPath));
+    }
+    catch { /* already removed */ }
+    for (const p of paths) {
+        const dir = join(root, p.replace(/[^a-zA-Z0-9]/g, "-"));
+        try {
+            if (statSync(dir).isDirectory() && onlyEmptyDirs(dir))
+                rmSync(dir, { recursive: true });
+        }
+        catch { /* absent or unreadable — leave it */ }
+    }
+}
+/** Build a sandbox whose .claude/skills/ holds exactly `skillNames` (copied from srcCwd, with any
+ * supporting files) plus srcCwd's instruction files. Removed on cleanup() or on process exit. */
+export function buildSandbox(srcCwd, skillNames) {
+    const root = mkdtempSync(join(tmpdir(), "skill-probe-"));
+    let real = root;
+    try {
+        real = realpathSync(root);
+    }
+    catch { /* keep root */ }
+    const forgetState = () => { removeRuntimeStateFor(root); if (real !== root)
+        removeRuntimeStateFor(real); };
+    trackTempDir(root, forgetState);
+    const cleanup = () => {
+        rmSync(root, { recursive: true, force: true });
+        forgetState();
+        untrackTempDir(root);
+    };
+    try {
+        const destSkills = skillsRoot(root);
+        mkdirSync(destSkills, { recursive: true });
+        for (const name of skillNames) {
+            const src = join(skillsRoot(srcCwd), name);
+            if (!existsSync(src))
+                throw new Error(`skill not found in ${srcCwd}: ${name}`);
+            // dereference: a symlinked SKILL.md (or skill folder) must become a real copy, or a write
+            // into the sandbox would land in the user's real file
+            cpSync(src, join(destSkills, name), { recursive: true, dereference: true });
+        }
+        for (const rel of INSTRUCTION_FILES) {
+            const src = join(srcCwd, rel);
+            if (!existsSync(src))
+                continue;
+            mkdirSync(dirname(join(root, rel)), { recursive: true });
+            cpSync(src, join(root, rel), { dereference: true });
+        }
+    }
+    catch (e) {
+        cleanup();
+        throw e;
+    }
+    return { cwd: root, skills: [...skillNames], cleanup };
+}
+/** Run `fn` inside a sandbox, always cleaning up. */
+export async function withSandbox(srcCwd, skillNames, fn) {
+    const sb = buildSandbox(srcCwd, skillNames);
+    try {
+        return await fn(sb);
+    }
+    finally {
+        sb.cleanup();
+    }
+}

package/dist/skills.d.ts

@@ -0,0 +1,48 @@
+export interface SkillInfo {
+    /** the skill's folder name under .claude/skills/ — its identity everywhere in skill-probe */
+    name: string;
+    /** description, whitespace-normalized to one line ("" if missing) */
+    description: string;
+    /** `name:` from the frontmatter, if present (may differ from the folder name) */
+    frontmatterName?: string;
+}
+interface FieldSpan {
+    key: string;
+    value: string;
+    start: number;
+    end: number;
+    /** an unterminated quoted scalar — the span's extent is unknowable, so it must not be rewritten */
+    broken?: boolean;
+}
+export interface Frontmatter {
+    /** false when the file doesn't open with a `---` frontmatter block */
+    ok: boolean;
+    fields: Record<string, string>;
+    /** line spans of each top-level key's entry (for in-place rewriting) */
+    spans: FieldSpan[];
+    lines: string[];
+    /** index of the closing `---` line */
+    close: number;
+    eol: string;
+    /** offsets[i] = where line i starts in the BOM-stripped text; offsets[lines.length] = its length */
+    offsets: number[];
+}
+export declare function parseFrontmatter(text: string): Frontmatter;
+/** One-line form of a description, as a router would read it. */
+export declare function normalizeDescription(s: string): string;
+/** Format a string as a YAML scalar for a single `key: value` line: plain when unambiguous,
+ * otherwise double-quoted (JSON string syntax is valid YAML). */
+export declare function yamlScalar(s: string): string;
+/** Return `text` with its frontmatter `description` replaced by `desc` (written on one line).
+ * Every other byte of the file — including each line's own line ending — is preserved. Throws if
+ * there is no frontmatter block or the existing description is an unterminated quoted string. */
+export declare function replaceDescription(text: string, desc: string): string;
+export declare function readSkillFile(file: string, folder: string): SkillInfo;
+export declare function skillsRoot(cwd: string): string;
+export declare function skillFile(cwd: string, name: string): string;
+/** Every folder under <cwd>/.claude/skills/, following symlinks (a symlinked skill folder is a
+ * skill like any other), sorted. Throws if there is no skills dir. */
+export declare function skillFolders(cwd: string): string[];
+/** Every skill folder under <cwd>/.claude/skills/ that has a SKILL.md, sorted by name. */
+export declare function listSkills(cwd: string): SkillInfo[];
+export {};