skill-probe 0.9.0 → 0.10.0

package/dist/orchestrator.d.ts

@@ -1,7 +1,9 @@
 /** Runs one prompt up to k VALID times with sequential early-stopping vs the threshold.
  * Infra errors (status="error") are retried up to a budget and never counted as a behavioral
- * outcome (finding #1) — they're tracked separately so the audit can flag untrustworthy cases. */
-import type { RuntimeAdapter } from "./types.ts";
+ * outcome (finding #1) — they're tracked separately so the audit can flag untrustworthy cases.
+ * Early stopping follows an exact group-sequential plan (stats.ts), so peeking after every probe
+ * doesn't inflate the verdict error rate. */
+import type { ProbeEnv, ProbeOpts, RuntimeAdapter } from "./types.ts";
 export interface RunStats {
     /** valid probes whose outcome matched the expected behavior */
     hits: number;
@@ -14,16 +16,56 @@ export interface RunStats {
     /** distribution of outcomes over valid probes: skill name (or "None") -> count */
     dist: Record<string, number>;
     totalCost: number;
+    /** valid probes where the model called some other tool and no skill fired — with a low
+     * --max-turns the skill may have been about to load, so activation can be under-counted */
+    preempted: number;
+    /** what the runtime reported loading (first probe that reported it) */
+    env?: ProbeEnv;
+    /** stopped because the shared --max-cost budget ran out (the measurement is incomplete) */
+    budgetExhausted?: boolean;
+}
+/** A shared spend cap across every probe of a run (`--max-cost`). Checked before each probe, so a
+ * run can overshoot by at most the probes already in flight. */
+export declare class Budget {
+    spent: number;
+    readonly maxUsd: number;
+    constructor(maxUsd: number);
+    get exhausted(): boolean;
+    add(usd: number): void;
 }
 /** A measurement is untrustworthy when infrastructure failures dominate — no valid probes, or
  * errors at least equal the valid count. Such a run must NOT read as a behavioral result (a runtime
  * outage is not a clean pass). Shared by the audit, context, and diagnose paths. */
 export declare function infraUntrustworthy(s: RunStats): boolean;
-export declare function measure(adapter: RuntimeAdapter, prompt: string, expected: string | null, opts: {
+export interface MeasureOpts extends Omit<ProbeOpts, "cwd"> {
     cwd: string;
     maxK: number;
     threshold: number;
     conf: number;
-    model?: string;
+    budget?: Budget;
     onProbe?: (validN: number, maxK: number) => void;
-}): Promise<RunStats>;
+}
+export declare function measure(adapter: RuntimeAdapter, prompt: string, expected: string | null, opts: MeasureOpts): Promise<RunStats>;
+/** Bounded-concurrency map preserving input order. concurrency 1 = strictly sequential. */
+export declare function mapPool<T, R>(items: readonly T[], concurrency: number, fn: (item: T, index: number) => Promise<R>): Promise<R[]>;
+/** Problems with what the runtime actually loaded for a condition that was meant to contain
+ * exactly `intended` out of `library`. Matches either folder names or frontmatter names. Skills in
+ * `ambiguous` (library skills sharing a name with a runtime built-in, which is "loaded" whether or
+ * not the library copy is) can't be judged from names alone and are skipped. */
+export declare function loadProblems(env: ProbeEnv | undefined, intended: readonly string[], library: ReadonlyArray<{
+    name: string;
+    frontmatterName?: string;
+}>, ambiguous?: ReadonlySet<string>): {
+    missing: string[];
+    unexpected: string[];
+};
+/** Library skills whose name (folder or frontmatter) is also a runtime built-in's name. */
+export declare function builtinClashes(builtins: readonly string[], library: ReadonlyArray<{
+    name: string;
+    frontmatterName?: string;
+}>): string[];
+/** Skills the runtime loaded that are NOT part of the library (runtime built-ins, user config…). */
+export declare function extraSkills(env: ProbeEnv | undefined, library: ReadonlyArray<{
+    name: string;
+    frontmatterName?: string;
+}>): string[] | undefined;

package/dist/orchestrator.js

@@ -1,4 +1,13 @@
-import { sequentialDecision } from "./stats.js";
+import { planDecision, sequentialPlan } from "./stats.js";
+/** A shared spend cap across every probe of a run (`--max-cost`). Checked before each probe, so a
+ * run can overshoot by at most the probes already in flight. */
+export class Budget {
+    spent = 0;
+    maxUsd;
+    constructor(maxUsd) { this.maxUsd = maxUsd; }
+    get exhausted() { return this.spent >= this.maxUsd; }
+    add(usd) { this.spent += usd; }
+}
 /** A measurement is untrustworthy when infrastructure failures dominate — no valid probes, or
  * errors at least equal the valid count. Such a run must NOT read as a behavioral result (a runtime
  * outage is not a clean pass). Shared by the audit, context, and diagnose paths. */
@@ -6,32 +15,92 @@ export function infraUntrustworthy(s) {
     return s.n === 0 || (s.errors > 0 && s.errors >= s.n);
 }
 export async function measure(adapter, prompt, expected, opts) {
+    const { maxK, threshold, conf, budget, onProbe, ...probeOpts } = opts;
     const want = expected ?? "None";
+    const plan = sequentialPlan(maxK, threshold, conf);
     const dist = {};
-    let hits = 0, n = 0, errors = 0, totalCost = 0;
+    let hits = 0, n = 0, errors = 0, totalCost = 0, preempted = 0;
     let lastError;
+    let env;
+    let budgetExhausted = false;
     // allow up to 50% extra attempts to absorb transient infra errors without inflating k
-    const maxAttempts = opts.maxK + Math.ceil(opts.maxK / 2);
-    for (let attempt = 0; attempt < maxAttempts && n < opts.maxK; attempt++) {
-        const r = await adapter.probe(prompt, { cwd: opts.cwd, ...(opts.model ? { model: opts.model } : {}) });
+    const maxAttempts = maxK + Math.ceil(maxK / 2);
+    for (let attempt = 0; attempt < maxAttempts && n < maxK; attempt++) {
+        if (budget?.exhausted) {
+            budgetExhausted = true;
+            lastError = `--max-cost budget ($${budget.maxUsd}) reached`;
+            break;
+        }
+        const r = await adapter.probe(prompt, probeOpts);
         totalCost += r.cost;
+        budget?.add(r.cost);
+        if (!env && r.env?.loadedSkills)
+            env = r.env;
         if (r.status === "error") {
             errors++;
             lastError = r.error;
-            opts.onProbe?.(n, opts.maxK);
+            onProbe?.(n, maxK);
             continue; // do NOT record as a behavioral outcome
         }
         const key = r.skillFired ?? "None";
         dist[key] = (dist[key] ?? 0) + 1;
         if (key === want)
             hits++;
+        if (r.skillFired === null && r.trajectory.length > 0)
+            preempted++;
         n++;
-        opts.onProbe?.(n, opts.maxK);
-        if (n >= 3) {
-            const d = sequentialDecision(hits, n, opts.threshold, opts.conf);
-            if (d !== "undecided")
-                break;
-        }
+        onProbe?.(n, maxK);
+        if (planDecision(plan, hits, n) !== "undecided")
+            break;
     }
-    return { hits, n, errors, dist, totalCost, ...(lastError ? { lastError } : {}) };
+    return {
+        hits, n, errors, dist, totalCost, preempted,
+        ...(lastError ? { lastError } : {}),
+        ...(env ? { env } : {}),
+        ...(budgetExhausted ? { budgetExhausted } : {}),
+    };
+}
+/** Bounded-concurrency map preserving input order. concurrency 1 = strictly sequential. */
+export async function mapPool(items, concurrency, fn) {
+    const out = new Array(items.length);
+    let next = 0;
+    const worker = async () => {
+        while (next < items.length) {
+            const i = next++;
+            out[i] = await fn(items[i], i);
+        }
+    };
+    const width = Math.max(1, Math.min(Math.floor(concurrency) || 1, items.length));
+    await Promise.all(Array.from({ length: width }, worker));
+    return out;
+}
+/** Problems with what the runtime actually loaded for a condition that was meant to contain
+ * exactly `intended` out of `library`. Matches either folder names or frontmatter names. Skills in
+ * `ambiguous` (library skills sharing a name with a runtime built-in, which is "loaded" whether or
+ * not the library copy is) can't be judged from names alone and are skipped. */
+export function loadProblems(env, intended, library, ambiguous = new Set()) {
+    if (!env?.loadedSkills)
+        return { missing: [], unexpected: [] };
+    const loaded = new Set(env.loadedSkills);
+    const isLoaded = (s) => loaded.has(s.name) || (s.frontmatterName !== undefined && loaded.has(s.frontmatterName));
+    const want = new Set(intended);
+    const judged = library.filter((s) => !ambiguous.has(s.name));
+    return {
+        missing: judged.filter((s) => want.has(s.name) && !isLoaded(s)).map((s) => s.name),
+        unexpected: judged.filter((s) => !want.has(s.name) && isLoaded(s)).map((s) => s.name),
+    };
+}
+/** Library skills whose name (folder or frontmatter) is also a runtime built-in's name. */
+export function builtinClashes(builtins, library) {
+    const b = new Set(builtins);
+    return library
+        .filter((s) => b.has(s.name) || (s.frontmatterName !== undefined && b.has(s.frontmatterName)))
+        .map((s) => s.name);
+}
+/** Skills the runtime loaded that are NOT part of the library (runtime built-ins, user config…). */
+export function extraSkills(env, library) {
+    if (!env?.loadedSkills)
+        return undefined;
+    const mine = new Set(library.flatMap((s) => [s.name, ...(s.frontmatterName ? [s.frontmatterName] : [])]));
+    return env.loadedSkills.filter((s) => !mine.has(s)).sort();
 }

package/dist/proc.d.ts

@@ -0,0 +1,24 @@
+export interface ProcResult {
+    /** exit code; null when killed by a signal or when the spawn itself failed */
+    code: number | null;
+    stdout: string;
+    stderr: string;
+    timedOut: boolean;
+    /** set when the binary couldn't be started at all (e.g. ENOENT: not on PATH) */
+    spawnError?: string;
+}
+/** Run `cmd args` in `cwd` with stdin closed (some CLIs hang waiting on an open stdin). Never
+ * rejects: failures come back as data so callers can classify them as infrastructure errors.
+ * Resolves within timeoutMs + PIPE_GRACE_MS no matter what the child's descendants do. */
+export declare function runProcess(cmd: string, args: string[], opts: {
+    cwd: string;
+    timeoutMs: number;
+    env?: NodeJS.ProcessEnv;
+}): Promise<ProcResult>;
+/** Track a temp dir so it's removed (plus `also`) even if the run is interrupted. */
+export declare function trackTempDir(dir: string, also?: () => void): void;
+export declare function untrackTempDir(dir: string): void;
+/** Kill live children and remove tracked temp dirs. Synchronous: safe inside an 'exit' handler. */
+export declare function cleanupAll(): void;
+/** For the CLI entry point only (library callers keep default signal behavior). */
+export declare function installSignalHandlers(): void;

package/dist/proc.js

@@ -0,0 +1,101 @@
+/** Child-process plumbing shared by the runtime adapters and the LLM CLI backend: one spawn with a
+ * hard timeout, plus a registry of live children and temp dirs so Ctrl-C / SIGTERM never leaves
+ * orphaned `claude` processes or sandbox directories behind. */
+import { spawn } from "node:child_process";
+import { rmSync } from "node:fs";
+const liveChildren = new Set();
+/** temp dir -> extra cleanup to run alongside removing it */
+const liveDirs = new Map();
+/** How long to wait for stdout/stderr to close after the child exits or is killed. A descendant
+ * that inherited our pipes can hold them open forever; the timeout must still bound wall time. */
+const PIPE_GRACE_MS = 1500;
+const OWN_GROUP = process.platform !== "win32";
+/** Kill the child and everything it spawned. Each child leads its own process group (detached),
+ * so one signal to -pid reaches grandchildren too. Only while the leader is alive — once it has
+ * exited, its group id could in principle be recycled, and we never signal a group we don't own. */
+function killTree(child) {
+    if (child.pid === undefined || child.exitCode !== null || child.signalCode !== null)
+        return;
+    try {
+        if (OWN_GROUP)
+            process.kill(-child.pid, "SIGKILL");
+        else
+            child.kill("SIGKILL");
+    }
+    catch {
+        try {
+            child.kill("SIGKILL");
+        }
+        catch { /* already gone */ }
+    }
+}
+/** Run `cmd args` in `cwd` with stdin closed (some CLIs hang waiting on an open stdin). Never
+ * rejects: failures come back as data so callers can classify them as infrastructure errors.
+ * Resolves within timeoutMs + PIPE_GRACE_MS no matter what the child's descendants do. */
+export function runProcess(cmd, args, opts) {
+    return new Promise((resolve) => {
+        let stdout = "", stderr = "", timedOut = false, settled = false;
+        let exitCode = null;
+        let grace;
+        const child = spawn(cmd, args, {
+            cwd: opts.cwd, stdio: ["ignore", "pipe", "pipe"], detached: OWN_GROUP,
+            ...(opts.env ? { env: opts.env } : {}),
+        });
+        liveChildren.add(child);
+        const finish = (r) => {
+            if (settled)
+                return;
+            settled = true;
+            clearTimeout(timer);
+            clearTimeout(grace);
+            liveChildren.delete(child);
+            child.stdout?.destroy();
+            child.stderr?.destroy();
+            resolve(r);
+        };
+        const finishSoon = () => {
+            grace ??= setTimeout(() => finish({ code: exitCode, stdout, stderr, timedOut }), PIPE_GRACE_MS);
+        };
+        const timer = setTimeout(() => { timedOut = true; killTree(child); finishSoon(); }, opts.timeoutMs);
+        child.stdout.on("data", (d) => { stdout += d.toString(); });
+        child.stderr.on("data", (d) => { stderr += d.toString(); });
+        child.on("error", (e) => finish({ code: null, stdout, stderr, timedOut, spawnError: e.message }));
+        child.on("exit", (code) => { exitCode = code; finishSoon(); });
+        child.on("close", (code) => finish({ code, stdout, stderr, timedOut }));
+    });
+}
+/** Track a temp dir so it's removed (plus `also`) even if the run is interrupted. */
+export function trackTempDir(dir, also) { liveDirs.set(dir, also); }
+export function untrackTempDir(dir) { liveDirs.delete(dir); }
+/** Kill live children and remove tracked temp dirs. Synchronous: safe inside an 'exit' handler. */
+export function cleanupAll() {
+    for (const c of liveChildren)
+        killTree(c);
+    liveChildren.clear();
+    for (const [d, also] of liveDirs) {
+        try {
+            rmSync(d, { recursive: true, force: true });
+        }
+        catch { /* best effort */ }
+        try {
+            also?.();
+        }
+        catch { /* best effort */ }
+    }
+    liveDirs.clear();
+}
+let installed = false;
+/** For the CLI entry point only (library callers keep default signal behavior). */
+export function installSignalHandlers() {
+    if (installed)
+        return;
+    installed = true;
+    process.on("exit", cleanupAll);
+    for (const [sig, code] of [["SIGINT", 130], ["SIGTERM", 143]]) {
+        process.on(sig, () => {
+            process.stderr.write(`\nskill-probe: interrupted (${sig}) — stopping probes and cleaning up\n`);
+            cleanupAll();
+            process.exit(code);
+        });
+    }
+}

package/dist/report.d.ts

@@ -1,11 +1,13 @@
+/** Human table + machine JSON rendering. Every number ships with its CI and k; infrastructure
+ * errors are shown distinctly from behavioral results. */
+import { kToCertify } from "./stats.ts";
 import type { AuditResult } from "./eval.ts";
 import type { FixResult } from "./fix.ts";
 import type { ContextAuditResult } from "./context.ts";
 import type { DoctorResult } from "./doctor.ts";
 import type { DiagnoseResult } from "./diagnose.ts";
 import type { BaselineComparison } from "./baseline.ts";
-/** Smallest k at which an all-success run could certify reliability >= threshold. */
-export declare function kToCertify(threshold: number, conf: number): number;
+export { kToCertify };
 export interface RenderOpts {
     showCost?: boolean;
 }

package/dist/report.js

@@ -1,24 +1,47 @@
 /** Human table + machine JSON rendering. Every number ships with its CI and k; infrastructure
  * errors are shown distinctly from behavioral results. */
-import { wilsonInterval } from "./stats.js";
+import { kToCertify } from "./stats.js";
+export { kToCertify };
 const pct = (x) => `${Math.round(x * 100)}%`;
 const pval = (p) => (p < 0.001 ? "<0.001" : p.toFixed(3));
 const LABEL = {
     pass: "PASS        ", fail: "FAIL        ",
     inconclusive: "INCONCLUSIVE", error: "ERROR       ",
 };
-/** Smallest k at which an all-success run could certify reliability >= threshold. */
-export function kToCertify(threshold, conf) {
-    for (let k = 1; k <= 500; k++) {
-        if (wilsonInterval(k, k, conf).lo > threshold)
-            return k;
+/** One line describing what the runtime actually ran with, when it told us. */
+function environmentLine(m) {
+    const e = m?.environment;
+    if (!e)
+        return null;
+    const bits = [];
+    if (e.runtimeVersion)
+        bits.push(`runtime ${e.runtimeVersion}`);
+    if (e.resolvedModel && m.model === "(runtime default)")
+        bits.push(`model resolved to ${e.resolvedModel}`);
+    if (e.extraSkills) {
+        bits.push(e.extraSkills.length
+            ? `${e.extraSkills.length} skill(s) outside the library also loaded (${e.extraSkills.slice(0, 6).join(", ")}${e.extraSkills.length > 6 ? ", …" : ""})`
+            : "no skills outside the library loaded");
     }
-    return 500;
+    bits.push(e.userConfig ? "personal runtime config INCLUDED"
+        : e.hermetic ? "hermetic (personal runtime config excluded)"
+            : "personal runtime config NOT isolated (this runtime can't exclude it)");
+    return `  ${bits.join(" · ")}`;
+}
+function theftLine(c) {
+    const counts = c.theft.map((s) => `${s} ${c.stats.dist[s]}/${c.stats.n}`).join(", ");
+    return `        ⚠ trigger-theft by: ${counts}` +
+        (c.verdict === "pass" ? " (reliability still clears the bar; --fail-on-theft makes this fail)" : "");
 }
 export function renderTable(a, opts = {}) {
     const showCost = opts.showCost ?? true;
     const out = [];
     out.push(`skill-probe — runtime: ${a.runtime}  model: ${a.model}  threshold: ${pct(a.threshold)}`);
+    const env = environmentLine(a.manifest);
+    if (env)
+        out.push(env);
+    for (const w of a.warnings ?? [])
+        out.push(`  ⚠ ${w}`);
     out.push("");
     for (const c of a.cases) {
         const r = c.reliability;
@@ -35,11 +58,17 @@ export function renderTable(a, opts = {}) {
         const dist = Object.entries(c.stats.dist)
             .sort((p, q) => q[1] - p[1]).map(([s, n]) => `${s}×${n}`).join(", ");
         out.push(`        outcomes: ${dist}`);
+        if (c.notLoaded)
+            out.push(`        ⛔ the runtime never loaded "${c.expected}" — it cannot fire (check its SKILL.md)`);
         if (c.theft.length)
-            out.push(`        ⚠ trigger-theft by: ${c.theft.join(", ")}`);
+            out.push(theftLine(c));
+        if (c.expected !== null && c.stats.preempted > 0 && c.verdict !== "pass") {
+            out.push(`        ⓘ ${c.stats.preempted}/${c.stats.n} probes called another tool first and loaded no skill — ` +
+                `activation may be under-counted (try --max-turns 2)`);
+        }
         if (c.verdict === "inconclusive") {
-            out.push(`        ↳ CI straddles threshold — need ~k=${kToCertify(a.threshold, r.conf)} ` +
-                `all-success runs to certify ≥${pct(a.threshold)} (currently k=${r.k})`);
+            out.push(`        ↳ not statistically decided at k=${r.k} — certifying ≥${pct(a.threshold)} needs ` +
+                `k≥${kToCertify(a.threshold, r.conf)} with every run firing (more if some miss)`);
         }
     }
     out.push("");
@@ -57,6 +86,11 @@ export function renderMarkdown(a, opts = {}) {
     const out = [];
     const conf = a.cases[0]?.reliability.conf ?? 0.95;
     out.push(`### skill-probe — \`${a.runtime}\` · model \`${a.model}\` · threshold ${pct(a.threshold)}`);
+    const env = environmentLine(a.manifest);
+    if (env)
+        out.push("", `_${env.trim()}_`);
+    for (const w of a.warnings ?? [])
+        out.push("", `> ⚠ ${w}`);
     out.push("");
     out.push(`| Verdict | Skill | Reliability (${pct(conf)} CI) | k | Notes |`);
     out.push("|---|---|---|---|---|");
@@ -66,10 +100,12 @@ export function renderMarkdown(a, opts = {}) {
         let notes = "";
         if (c.verdict === "error")
             notes = `infra error: ${c.stats.lastError ?? "untrustworthy"}`;
+        else if (c.notLoaded)
+            notes = `⛔ runtime never loaded \`${c.expected}\``;
         else if (c.theft.length)
-            notes = `⚠ trigger-theft by ${c.theft.join(", ")}`;
+            notes = `⚠ trigger-theft by ${c.theft.map((s) => `${s} ${c.stats.dist[s]}/${c.stats.n}`).join(", ")}`;
         else if (c.verdict === "inconclusive")
-            notes = `need ~k=${kToCertify(a.threshold, r.conf)} to certify`;
+            notes = `need k≥${kToCertify(a.threshold, r.conf)} to certify`;
         const rel = c.verdict === "error" ? "—" : `${pct(r.pHat)} [${pct(r.ciLow)}, ${pct(r.ciHigh)}]`;
         out.push(`| ${mark[c.verdict]} | \`${expect}\` | ${rel} | ${r.k} | ${notes} |`);
     }
@@ -91,6 +127,11 @@ export function renderDiagnose(d, opts = {}) {
     const out = [];
     out.push(`skill-probe diagnose — runtime: ${d.runtime}  model: ${d.model}  threshold: ${pct(d.threshold)}`);
     out.push(`  actual = real activation · intended = forced-choice routing (does the model even pick it?)`);
+    const env = environmentLine(d.manifest);
+    if (env)
+        out.push(env);
+    for (const w of d.warnings ?? [])
+        out.push(`  ⚠ ${w}`);
     out.push("");
     const dist = (m) => Object.entries(m).sort((p, q) => q[1] - p[1]).map(([s, n]) => `${s}×${n}`).join(", ") || "(none)";
     for (const c of d.cases) {
@@ -131,6 +172,12 @@ export function renderContext(a, opts = {}) {
     out.push(`skill-probe context — runtime: ${a.runtime}  model: ${a.model}  ` +
         `threshold: ${pct(a.threshold)}  library: ${a.librarySize} skills`);
     out.push(`  isolation = only that skill loaded · co-loaded = the full library (${a.librarySize}) loaded`);
+    out.push(`  every condition runs in an identical sandbox (skills + project CLAUDE.md/AGENTS.md) — only the skill set differs`);
+    const env = environmentLine(a.manifest);
+    if (env)
+        out.push(env);
+    for (const w of a.warnings ?? [])
+        out.push(`  ⚠ ${w}`);
     const familyN = a.cases.filter((c) => !c.untrustworthy).length;
     if (familyN > 1) {
         out.push(`  ${familyN} comparisons — interference decided on Benjamini-Hochberg adjusted p<0.05` +
@@ -141,6 +188,11 @@ export function renderContext(a, opts = {}) {
     for (const c of a.cases) {
         if (c.untrustworthy) {
             out.push(`  [⛔ ERROR]  ${c.expected}  | ${c.prompt}`);
+            if (c.loadIssue) {
+                out.push(`        the runtime loaded the wrong skill set — comparison NOT trustworthy`);
+                out.push(`        ${c.loadIssue}`);
+                continue;
+            }
             out.push(`        infrastructure errors dominated — comparison NOT trustworthy ` +
                 `(isolation ${c.isolation.stats.errors} err/${c.isolation.stats.n} valid, ` +
                 `co-loaded ${c.coLoaded.stats.errors} err/${c.coLoaded.stats.n} valid)`);
@@ -156,12 +208,18 @@ export function renderContext(a, opts = {}) {
         out.push(`        Δ ${c.deltaP >= 0 ? "+" : ""}${pct(c.deltaP)} under load   Fisher p=${pval(c.fisherP)}` +
             (familyN > 1 ? ` · BH-adj p=${pval(c.fisherPAdj)}` : ""));
         if (c.interference) {
+            const external = new Set(c.externalThieves ?? []);
             const thieves = Object.keys(c.coLoaded.stats.dist)
-                .filter((s) => s !== c.expected && s !== "None");
+                .filter((s) => s !== c.expected && s !== "None")
+                .map((s) => (external.has(s) ? `${s} (outside the library)` : s));
             out.push(`        ↳ fires reliably alone but is suppressed when the library is co-loaded` +
                 (thieves.length ? ` (stolen by: ${thieves.join(", ")})` : ""));
             if (c.ablation) {
-                if (c.ablation.length === 0) {
+                if (c.ablation.length === 0 && external.size) {
+                    out.push(`        leave-one-out: only skills outside the library stole these probes ` +
+                        `(${[...external].join(", ")}) — they can't be removed; narrow or rename "${c.expected}" to stop colliding`);
+                }
+                else if (c.ablation.length === 0) {
                     out.push(`        leave-one-out: no sibling stole these probes (suppressed to None) — ` +
                         `dilution, not theft; nothing to ablate`);
                 }
@@ -169,7 +227,7 @@ export function renderContext(a, opts = {}) {
                     out.push(`        leave-one-out (each suspect removed, recovery vs co-loaded):`);
                     for (const abl of c.ablation) {
                         if (abl.untrustworthy) {
-                            out.push(`          − ${abl.removed}: infra errors dominated — run not trustworthy`);
+                            out.push(`          − ${abl.removed}: ${abl.loadIssue ?? "infra errors dominated"} — run not trustworthy`);
                             continue;
                         }
                         const tag = abl.culprit
@@ -241,19 +299,25 @@ export function renderFix(f) {
     out.push(`  old: ${f.oldDesc}`);
     out.push(`  new: ${f.newDesc}`);
     out.push("");
-    out.push(`  before: ${pct(f.before.pHat)} [${pct(f.before.ciLow)}, ${pct(f.before.ciHigh)}]  ` +
-        `after: ${pct(f.after.pHat)} [${pct(f.after.ciLow)}, ${pct(f.after.ciHigh)}]  ` +
-        `(${f.n} paired runs${f.errors ? `, ${f.errors} dropped to infra errors` : ""})`);
-    out.push(`  P(rewrite improved reliability) = ${pct(f.pImprove)}   ` +
-        `Δ = ${f.deltaMean >= 0 ? "+" : ""}${pct(f.deltaMean)} [${pct(f.deltaCi[0])}, ${pct(f.deltaCi[1])}]`);
-    out.push("");
-    if (f.applied) {
+    if (f.n > 0 || f.errors > 0) {
+        out.push(`  before: ${pct(f.before.pHat)} [${pct(f.before.ciLow)}, ${pct(f.before.ciHigh)}]  ` +
+            `after: ${pct(f.after.pHat)} [${pct(f.after.ciLow)}, ${pct(f.after.ciHigh)}]  ` +
+            `(${f.n} paired runs${f.errors ? `, ${f.errors} dropped to infra errors` : ""})` +
+            (f.totalCost ? `  cost $${f.totalCost.toFixed(4)}` : ""));
+        out.push(`  P(rewrite improved reliability) = ${pct(f.pImprove)}   ` +
+            `Δ = ${f.deltaMean >= 0 ? "+" : ""}${pct(f.deltaMean)} [${pct(f.deltaCi[0])}, ${pct(f.deltaCi[1])}]`);
+        out.push("");
+    }
+    if (f.note) {
+        out.push(`  ↩ NOT APPLIED — ${f.note}. Description unchanged.`);
+    }
+    else if (f.applied) {
         out.push(`  ✅ APPLIED — P(improvement) ${pct(f.pImprove)} ≥ bar ${pct(f.applyBar)} and Δ>0.`);
         out.push(`     original backed up to: ${f.backupPath}`);
     }
     else {
-        out.push(`  ↩ REVERTED — not enough evidence the rewrite helps ` +
-            `(need P(improvement) ≥ ${pct(f.applyBar)} and Δ>0). Description unchanged, no backup kept.`);
+        out.push(`  ↩ NOT APPLIED — not enough evidence the rewrite helps ` +
+            `(need P(improvement) ≥ ${pct(f.applyBar)} and Δ>0). Description unchanged.`);
     }
     return out.join("\n");
 }

package/dist/sandbox.d.ts

@@ -0,0 +1,18 @@
+/** Project instruction files copied into every sandbox, relative to the project root. */
+export declare const INSTRUCTION_FILES: string[];
+export interface Sandbox {
+    cwd: string;
+    skills: string[];
+    cleanup: () => void;
+}
+/** Claude Code creates `<config>/projects/<cwd with non-alphanumerics as "-">/memory/` for every
+ * working directory it starts in, even with --no-session-persistence. Each sandbox is a fresh path,
+ * so without this a big context/ablation run would leave dozens of empty dirs per run. Removes the
+ * dir for THIS sandbox path only, and only if it contains nothing but empty directories — it never
+ * deletes a file. Exported for testing. */
+export declare function removeRuntimeStateFor(sandboxPath: string, configDir?: string): void;
+/** Build a sandbox whose .claude/skills/ holds exactly `skillNames` (copied from srcCwd, with any
+ * supporting files) plus srcCwd's instruction files. Removed on cleanup() or on process exit. */
+export declare function buildSandbox(srcCwd: string, skillNames: string[]): Sandbox;
+/** Run `fn` inside a sandbox, always cleaning up. */
+export declare function withSandbox<T>(srcCwd: string, skillNames: string[], fn: (sb: Sandbox) => Promise<T>): Promise<T>;

package/dist/sandbox.js

@@ -0,0 +1,98 @@
+/** Probe sandboxes: throwaway project dirs holding a chosen subset of the library.
+ *
+ * Every condition that is COMPARED against another (context's isolation / co-loaded / ablation runs,
+ * fix's before / after) must be built the same way, differing ONLY in which skills are present.
+ * Comparing a sandbox against the real project dir confounds the skill set with everything else in
+ * the project (CLAUDE.md, settings, files): a drop caused by the project's CLAUDE.md would read as
+ * "interference", and every leave-one-out run would "recover" and name an innocent sibling.
+ *
+ * Each sandbox gets the project's instruction files (they shape routing, so they're held constant)
+ * but not its settings/hooks/MCP config (side effects; can't run safely from another directory). */
+import { mkdtempSync, mkdirSync, cpSync, rmSync, existsSync, readdirSync, realpathSync, statSync } from "node:fs";
+import { homedir, tmpdir } from "node:os";
+import { dirname, join } from "node:path";
+import { skillsRoot } from "./skills.js";
+import { trackTempDir, untrackTempDir } from "./proc.js";
+/** Project instruction files copied into every sandbox, relative to the project root. */
+export const INSTRUCTION_FILES = ["CLAUDE.md", "AGENTS.md", join(".claude", "CLAUDE.md")];
+/** True if `dir` holds nothing but (recursively) empty directories. */
+function onlyEmptyDirs(dir) {
+    for (const e of readdirSync(dir, { withFileTypes: true })) {
+        if (!e.isDirectory() || !onlyEmptyDirs(join(dir, e.name)))
+            return false;
+    }
+    return true;
+}
+/** Claude Code creates `<config>/projects/<cwd with non-alphanumerics as "-">/memory/` for every
+ * working directory it starts in, even with --no-session-persistence. Each sandbox is a fresh path,
+ * so without this a big context/ablation run would leave dozens of empty dirs per run. Removes the
+ * dir for THIS sandbox path only, and only if it contains nothing but empty directories — it never
+ * deletes a file. Exported for testing. */
+export function removeRuntimeStateFor(sandboxPath, configDir) {
+    const root = join(configDir ?? process.env["CLAUDE_CONFIG_DIR"] ?? join(homedir(), ".claude"), "projects");
+    const paths = new Set([sandboxPath]);
+    try {
+        paths.add(realpathSync(sandboxPath));
+    }
+    catch { /* already removed */ }
+    for (const p of paths) {
+        const dir = join(root, p.replace(/[^a-zA-Z0-9]/g, "-"));
+        try {
+            if (statSync(dir).isDirectory() && onlyEmptyDirs(dir))
+                rmSync(dir, { recursive: true });
+        }
+        catch { /* absent or unreadable — leave it */ }
+    }
+}
+/** Build a sandbox whose .claude/skills/ holds exactly `skillNames` (copied from srcCwd, with any
+ * supporting files) plus srcCwd's instruction files. Removed on cleanup() or on process exit. */
+export function buildSandbox(srcCwd, skillNames) {
+    const root = mkdtempSync(join(tmpdir(), "skill-probe-"));
+    let real = root;
+    try {
+        real = realpathSync(root);
+    }
+    catch { /* keep root */ }
+    const forgetState = () => { removeRuntimeStateFor(root); if (real !== root)
+        removeRuntimeStateFor(real); };
+    trackTempDir(root, forgetState);
+    const cleanup = () => {
+        rmSync(root, { recursive: true, force: true });
+        forgetState();
+        untrackTempDir(root);
+    };
+    try {
+        const destSkills = skillsRoot(root);
+        mkdirSync(destSkills, { recursive: true });
+        for (const name of skillNames) {
+            const src = join(skillsRoot(srcCwd), name);
+            if (!existsSync(src))
+                throw new Error(`skill not found in ${srcCwd}: ${name}`);
+            // dereference: a symlinked SKILL.md (or skill folder) must become a real copy, or a write
+            // into the sandbox would land in the user's real file
+            cpSync(src, join(destSkills, name), { recursive: true, dereference: true });
+        }
+        for (const rel of INSTRUCTION_FILES) {
+            const src = join(srcCwd, rel);
+            if (!existsSync(src))
+                continue;
+            mkdirSync(dirname(join(root, rel)), { recursive: true });
+            cpSync(src, join(root, rel), { dereference: true });
+        }
+    }
+    catch (e) {
+        cleanup();
+        throw e;
+    }
+    return { cwd: root, skills: [...skillNames], cleanup };
+}
+/** Run `fn` inside a sandbox, always cleaning up. */
+export async function withSandbox(srcCwd, skillNames, fn) {
+    const sb = buildSandbox(srcCwd, skillNames);
+    try {
+        return await fn(sb);
+    }
+    finally {
+        sb.cleanup();
+    }
+}