skill-probe 0.9.0 → 0.10.0

package/dist/context.js

@@ -2,39 +2,41 @@
  * (the real, hard condition). This command additionally measures each skill in ISOLATION (only that
  * skill present) and compares the two: it catches the "fires fine alone, fails under load" case that
  * a single aggregate rate hides. The drop is tested with Fisher's exact test, so interference is
- * reported as a real effect (p-value), not eyeballed. */
-import { mkdtempSync, mkdirSync, cpSync, rmSync, existsSync } from "node:fs";
-import { tmpdir } from "node:os";
-import { join } from "node:path";
+ * reported as a real effect (p-value), not eyeballed.
+ *
+ * Every condition (isolation, co-loaded, each leave-one-out ablation) runs in a sandbox built the
+ * same way — see sandbox.ts — so the ONLY difference between conditions is which skills are
+ * present. When the runtime reports what it loaded, each condition is checked against what it was
+ * meant to contain; a mismatch makes the comparison untrustworthy instead of silently wrong. */
+import { probeSettings } from "./types.js";
 import { reliability, fisherExact2x2, bhAdjust } from "./stats.js";
-import { measure, infraUntrustworthy } from "./orchestrator.js";
+import { measure, mapPool, infraUntrustworthy, loadProblems, extraSkills, builtinClashes, } from "./orchestrator.js";
 import { getAdapter } from "./adapters/index.js";
-import { listSkills } from "./gen.js";
+import { listSkills } from "./skills.js";
+import { buildSandbox, withSandbox } from "./sandbox.js";
 import { ConfigError } from "./config.js";
-import { buildManifest } from "./manifest.js";
-/** Build a throwaway project dir whose .claude/skills/ holds ONLY the named skills (copied from
- * srcCwd). Returns its path plus a cleanup(). The probe runs against this dir to load a subset. */
-export function buildContextDir(srcCwd, skillNames) {
-    const root = mkdtempSync(join(tmpdir(), "skill-probe-ctx-"));
-    const destSkills = join(root, ".claude", "skills");
-    mkdirSync(destSkills, { recursive: true });
-    for (const name of skillNames) {
-        const src = join(srcCwd, ".claude", "skills", name);
-        if (!existsSync(src))
-            throw new Error(`skill not found in ${srcCwd}: ${name}`);
-        cpSync(src, join(destSkills, name), { recursive: true });
-    }
-    return { cwd: root, cleanup: () => rmSync(root, { recursive: true, force: true }) };
+import { buildManifest, buildEnvironment, capabilityWarnings } from "./manifest.js";
+/** Kept for compatibility: a sandbox holding only the named skills (plus instruction files). */
+export const buildContextDir = buildSandbox;
+/** "" when the condition loaded exactly what it should have; otherwise a human-readable reason. */
+function describeLoadIssue(label, stats, intended, library, ambiguous) {
+    const { missing, unexpected } = loadProblems(stats.env, intended, library, ambiguous);
+    const parts = [];
+    if (missing.length)
+        parts.push(`did not load ${missing.join(", ")}`);
+    if (unexpected.length)
+        parts.push(`also loaded ${unexpected.join(", ")}`);
+    return parts.length ? `${label}: the runtime ${parts.join(" and ")}` : "";
 }
+const fisherVs = (a, b) => (a.n > 0 && b.n > 0) ? fisherExact2x2(a.hits, a.n - a.hits, b.hits, b.n - b.hits) : 1;
 export async function runContextAudit(cfg, optsOrProgress = {}) {
     const opts = typeof optsOrProgress === "function" ? { onProgress: optsOrProgress } : optsOrProgress;
     const onProgress = opts.onProgress;
     const adapter = getAdapter(cfg.runtime);
     const library = listSkills(cfg.cwd);
-    const present = new Set(library.map((s) => s.name));
-    const measured = [];
+    const libraryNames = library.map((s) => s.name);
+    const present = new Set(libraryNames);
     const skipped = [];
-    let totalCost = 0;
     // decoys (null expected) have nothing to isolate → skipped. An expected skill that ISN'T in the
     // library is almost always a config typo: fail loud BEFORE spending probes, never silently skip.
     const eligible = [];
@@ -45,54 +47,73 @@ export async function runContextAudit(cfg, optsOrProgress = {}) {
         else if (!present.has(c.expected))
             unknown.push(c.expected);
         else
-            eligible.push(c);
+            eligible.push({ prompt: c.prompt, expected: c.expected });
     }
     if (unknown.length) {
         throw new ConfigError([
             `expected skill(s) not in ${cfg.cwd}/.claude/skills/: ${[...new Set(unknown)].join(", ")} ` +
-                `(typo?). Available: ${library.map((s) => s.name).join(", ") || "(none)"}.`,
+                `(typo?). Available: ${libraryNames.join(", ") || "(none)"}.`,
         ]);
     }
-    const common = { maxK: cfg.k, threshold: cfg.threshold, conf: cfg.conf, ...(cfg.model ? { model: cfg.model } : {}) };
-    for (let i = 0; i < eligible.length; i++) {
-        const c = eligible[i];
-        const expected = c.expected;
-        // isolation: a temp project containing only this skill
-        const ctx = buildContextDir(cfg.cwd, [expected]);
-        let iso;
-        try {
-            iso = await measure(adapter, c.prompt, expected, {
-                cwd: ctx.cwd, ...common,
-                ...(onProgress ? { onProbe: (validN, maxK) => onProgress({ caseIndex: i + 1, caseTotal: eligible.length, prompt: c.prompt, condition: "isolation", validN, maxK }) } : {}),
-            });
-        }
-        finally {
-            ctx.cleanup();
-        }
-        // co-loaded: the real library at cfg.cwd
-        const co = await measure(adapter, c.prompt, expected, {
-            cwd: cfg.cwd, ...common,
-            ...(onProgress ? { onProbe: (validN, maxK) => onProgress({ caseIndex: i + 1, caseTotal: eligible.length, prompt: c.prompt, condition: "co-loaded", validN, maxK }) } : {}),
+    const common = {
+        maxK: cfg.k, threshold: cfg.threshold, conf: cfg.conf, ...probeSettings(cfg),
+        ...(opts.budget ? { budget: opts.budget } : {}),
+    };
+    const progress = (caseIndex, caseTotal, prompt, condition, removed) => onProgress
+        ? { onProbe: (validN, maxK) => onProgress({ caseIndex, caseTotal, prompt, condition, validN, maxK, ...(removed ? { removed } : {}) }) }
+        : {};
+    // the full library, shared read-only by every co-loaded run
+    const measured = await withSandbox(cfg.cwd, libraryNames, (full) => mapPool(eligible, opts.concurrency ?? 1, async (c, i) => {
+        const iso = await withSandbox(cfg.cwd, [c.expected], (sb) => measure(adapter, c.prompt, c.expected, {
+            cwd: sb.cwd, ...common, ...progress(i + 1, eligible.length, c.prompt, "isolation"),
+        }));
+        const co = await measure(adapter, c.prompt, c.expected, {
+            cwd: full.cwd, ...common, ...progress(i + 1, eligible.length, c.prompt, "co-loaded"),
         });
-        totalCost += iso.totalCost + co.totalCost;
         const isoRel = reliability(iso.hits, iso.n, cfg.conf);
         const coRel = reliability(co.hits, co.n, cfg.conf);
-        // 2x2: rows = isolation/co-loaded, cols = hit/miss
-        const fisherP = (iso.n > 0 && co.n > 0)
-            ? fisherExact2x2(iso.hits, iso.n - iso.hits, co.hits, co.n - co.hits)
-            : 1;
-        const deltaP = coRel.pHat - isoRel.pHat;
-        // if infra failures dominated either side, the comparison isn't a behavioral result at all —
-        // it must never read as "no interference / pass" just because fisherP defaulted to 1.
+        // load-set integrity is judged after all cases (it may need the runtime's built-in names)
         const untrustworthy = infraUntrustworthy(iso) || infraUntrustworthy(co);
-        // interference is decided AFTER the loop, once the whole family of comparisons is known
-        // (Benjamini-Hochberg) — placeholders here.
-        measured.push({
-            prompt: c.prompt, expected,
+        const externalThieves = Object.keys(co.dist).filter((s) => s !== c.expected && s !== "None" && !present.has(s));
+        // interference is decided AFTER all cases, once the whole family is known (BH) — placeholders
+        return {
+            prompt: c.prompt, expected: c.expected,
             isolation: { stats: iso, rel: isoRel },
             coLoaded: { stats: co, rel: coRel },
-            deltaP, fisherP, fisherPAdj: 1, untrustworthy, interference: false,
-        });
+            deltaP: coRel.pHat - isoRel.pHat, fisherP: fisherVs(iso, co), fisherPAdj: 1,
+            untrustworthy, interference: false,
+            ...(externalThieves.length ? { externalThieves } : {}),
+        };
+    }));
+    // Did each condition load exactly the skills it should? A library skill that shares a name with a
+    // runtime built-in looks "loaded" everywhere, so when anything looks wrong, learn the built-in
+    // names once (an empty sandbox) and leave those ambiguous names out of the check.
+    let ambiguous = new Set();
+    let extraCost = 0;
+    let builtinsProbe;
+    // memoized promise (not a flag): concurrent ablation runs must all wait for the same answer
+    const learnBuiltins = () => (builtinsProbe ??= (async () => {
+        const r = await withSandbox(cfg.cwd, [], (sb) => adapter.probe("Reply with exactly: ok", { cwd: sb.cwd, ...probeSettings(cfg) }));
+        extraCost += r.cost;
+        opts.budget?.add(r.cost);
+        if (r.status === "ok" && r.env?.loadedSkills)
+            ambiguous = new Set(builtinClashes(r.env.loadedSkills, library));
+    })());
+    const looksWrong = (st, intended) => loadProblems(st.env, intended, library).unexpected.length > 0;
+    if (measured.some((m) => looksWrong(m.isolation.stats, [m.expected]) || looksWrong(m.coLoaded.stats, libraryNames))) {
+        await learnBuiltins();
+    }
+    for (const m of measured) {
+        const loadIssue = [
+            describeLoadIssue("isolation", m.isolation.stats, [m.expected], library, ambiguous),
+            describeLoadIssue("co-loaded", m.coLoaded.stats, libraryNames, library, ambiguous),
+        ].filter(Boolean).join("; ");
+        // a condition that didn't contain what it should isn't a behavioral result — it must never
+        // read as "no interference / pass".
+        if (loadIssue) {
+            m.loadIssue = loadIssue;
+            m.untrustworthy = true;
+        }
     }
     // One Fisher test per case = a family of comparisons: correct across the trustworthy ones so a
     // big library doesn't buy false interference flags by chance (raw p<0.05 × 40 cases ≈ 2 flukes).
@@ -112,46 +133,39 @@ export async function runContextAudit(cfg, optsOrProgress = {}) {
     // BH-corrected across all ablation runs (their own family). A significant recovery NAMES the
     // thief; no suspects in the outcome distribution means the drop was dilution, not theft.
     if (opts.ablate) {
-        const libraryNames = library.map((s) => s.name);
         const flagged = measured.filter((m) => m.interference);
         const cap = opts.suspects ?? 3;
-        const allRuns = [];
-        for (let fi = 0; fi < flagged.length; fi++) {
-            const m = flagged[fi];
-            const suspects = Object.entries(m.coLoaded.stats.dist)
-                .filter(([name]) => name !== m.expected && name !== "None")
-                .sort((x, y) => y[1] - x[1])
-                .slice(0, cap);
+        const jobs = [];
+        flagged.forEach((m, fi) => {
             m.ablation = [];
-            for (const [suspect, stolen] of suspects) {
-                const ctx = buildContextDir(cfg.cwd, libraryNames.filter((n) => n !== suspect));
-                let st;
-                try {
-                    st = await measure(adapter, m.prompt, m.expected, {
-                        cwd: ctx.cwd, ...common,
-                        ...(onProgress ? { onProbe: (validN, maxK) => onProgress({ caseIndex: fi + 1, caseTotal: flagged.length, prompt: m.prompt, condition: "ablation", removed: suspect, validN, maxK }) } : {}),
-                    });
-                }
-                finally {
-                    ctx.cleanup();
-                }
-                totalCost += st.totalCost;
-                const rel = reliability(st.hits, st.n, cfg.conf);
-                const co = m.coLoaded.stats;
-                const fisherP = (st.n > 0 && co.n > 0)
-                    ? fisherExact2x2(st.hits, st.n - st.hits, co.hits, co.n - co.hits)
-                    : 1;
-                const abl = {
-                    removed: suspect, stolen, stats: st, rel,
-                    deltaVsCoLoaded: rel.pHat - m.coLoaded.rel.pHat,
-                    fisherP, fisherPAdj: 1,
-                    untrustworthy: infraUntrustworthy(st),
-                    culprit: false,
-                };
-                m.ablation.push(abl);
-                allRuns.push(abl);
-            }
-        }
+            Object.entries(m.coLoaded.stats.dist)
+                .filter(([name]) => name !== m.expected && name !== "None" && present.has(name))
+                .sort((x, y) => y[1] - x[1])
+                .slice(0, cap)
+                .forEach(([suspect, stolen]) => jobs.push({ m, fi, suspect, stolen }));
+        });
+        const runs = await mapPool(jobs, opts.concurrency ?? 1, async ({ m, fi, suspect, stolen }) => {
+            const kept = libraryNames.filter((n) => n !== suspect);
+            const st = await withSandbox(cfg.cwd, kept, (sb) => measure(adapter, m.prompt, m.expected, {
+                cwd: sb.cwd, ...common, ...progress(fi + 1, flagged.length, m.prompt, "ablation", suspect),
+            }));
+            const rel = reliability(st.hits, st.n, cfg.conf);
+            if (looksWrong(st, kept))
+                await learnBuiltins();
+            const loadIssue = describeLoadIssue(`without ${suspect}`, st, kept, library, ambiguous);
+            const abl = {
+                removed: suspect, stolen, stats: st, rel,
+                deltaVsCoLoaded: rel.pHat - m.coLoaded.rel.pHat,
+                fisherP: fisherVs(st, m.coLoaded.stats), fisherPAdj: 1,
+                untrustworthy: infraUntrustworthy(st) || Boolean(loadIssue),
+                ...(loadIssue ? { loadIssue } : {}),
+                culprit: false,
+            };
+            return { m, abl };
+        });
+        for (const { m, abl } of runs)
+            m.ablation.push(abl);
+        const allRuns = runs.map((r) => r.abl);
         const ablFamily = allRuns.filter((r) => !r.untrustworthy);
         const ablAdj = bhAdjust(ablFamily.map((r) => r.fisherP));
         ablFamily.forEach((r, j) => { r.fisherPAdj = ablAdj[j]; });
@@ -164,10 +178,23 @@ export async function runContextAudit(cfg, optsOrProgress = {}) {
     const exitCode = measured.some((m) => m.interference) ? 1
         : measured.some((m) => m.untrustworthy) ? 2
             : 0;
+    const totalCost = extraCost + measured.reduce((s, m) => s + m.isolation.stats.totalCost + m.coLoaded.stats.totalCost +
+        (m.ablation ?? []).reduce((t, a) => t + a.stats.totalCost, 0), 0);
+    const env = measured.find((m) => m.coLoaded.stats.env)?.coLoaded.stats.env;
+    const warnings = capabilityWarnings(cfg, adapter);
+    if (ambiguous.size) {
+        warnings.push(`${[...ambiguous].join(", ")} share${ambiguous.size === 1 ? "s" : ""} a name with a ${cfg.runtime} ` +
+            `built-in skill — the two compete for the same trigger, and load checks can't tell them apart (rename recommended)`);
+    }
+    const allStats = measured.flatMap((m) => [m.isolation.stats, m.coLoaded.stats, ...(m.ablation ?? []).map((a) => a.stats)]);
+    if (allStats.some((st) => st.budgetExhausted)) {
+        warnings.push(`--max-cost budget reached — some conditions were cut short (inconclusive or error, never a clean result)`);
+    }
     return {
         runtime: cfg.runtime, model: cfg.model ?? "(runtime default)",
         threshold: cfg.threshold, conf: cfg.conf, librarySize: library.length,
         cases: measured, skipped, totalCost, exitCode,
-        manifest: buildManifest("context", cfg),
+        manifest: buildManifest("context", cfg, buildEnvironment(cfg, env, extraSkills(env, library), adapter)),
+        warnings,
     };
 }

package/dist/diagnose.d.ts

@@ -11,10 +11,10 @@
  * We measure ACTUAL activation (the real runtime probe) and INTENDED routing (a forced-choice: the
  * model is shown the skill descriptions and asked which one applies, with no auto-execution), then
  * classify the gap between them. */
-import type { Config } from "./types.ts";
+import { type Config } from "./types.ts";
 import { reliability } from "./stats.ts";
-import { type RunStats } from "./orchestrator.ts";
-import { type SkillInfo } from "./gen.ts";
+import { type Budget, type RunStats } from "./orchestrator.ts";
+import { type SkillInfo } from "./skills.ts";
 import { type RunManifest } from "./manifest.ts";
 export type DiagVerdict = "routes-ok" | "routing-miss" | "description-problem" | "inconclusive" | "error";
 /** Returns the chosen skill name, "none", or null when the answer is unusable (LLM error/garbage). */
@@ -53,6 +53,8 @@ export interface DiagnoseResult {
     exitCode: 0 | 1 | 2;
     /** who/what/when of this run — makes reports citable and comparable */
     manifest: RunManifest;
+    /** run-level caveats (runtime capabilities, budget cut-offs) */
+    warnings: string[];
 }
 export interface DiagProgress {
     caseIndex: number;
@@ -64,8 +66,12 @@ export interface DiagProgress {
 }
 export declare function normalizeChoice(text: string, skills: SkillInfo[]): string | null;
 export declare const defaultChoice: ChoiceFn;
-export declare function runDiagnose(cfg: Config, deps?: {
+export interface DiagnoseOpts {
     choice?: ChoiceFn;
     onProgress?: (e: DiagProgress) => void;
-}): Promise<DiagnoseResult>;
+    /** cases diagnosed in parallel (default 1) */
+    concurrency?: number;
+    budget?: Budget;
+}
+export declare function runDiagnose(cfg: Config, deps?: DiagnoseOpts): Promise<DiagnoseResult>;
 export {};

package/dist/diagnose.js

@@ -1,10 +1,24 @@
-import { reliability, sequentialDecision } from "./stats.js";
-import { measure, infraUntrustworthy } from "./orchestrator.js";
+/** `skill-probe diagnose` — when a skill fails to fire, WHY? Two very different root causes need
+ * two very different fixes, and you can't tell them apart from the activation rate alone:
+ *
+ *   - DESCRIPTION PROBLEM: the model, asked point-blank "which skill handles this?", doesn't even
+ *     pick the expected skill. Its description fails to communicate what it's for. Remedy: rewrite
+ *     the description (`skill-probe fix`).
+ *   - ROUTING MISS: the model DOES pick the expected skill when asked — it understands the
+ *     description — but a sibling wins at actual activation time. Remedy: deconflict the sibling /
+ *     inspect with `skill-probe context`. Rewriting this skill's description won't help.
+ *
+ * We measure ACTUAL activation (the real runtime probe) and INTENDED routing (a forced-choice: the
+ * model is shown the skill descriptions and asked which one applies, with no auto-execution), then
+ * classify the gap between them. */
+import { probeSettings } from "./types.js";
+import { reliability, planDecision, sequentialPlan } from "./stats.js";
+import { measure, mapPool, infraUntrustworthy, extraSkills } from "./orchestrator.js";
 import { getAdapter } from "./adapters/index.js";
-import { listSkills } from "./gen.js";
+import { listSkills } from "./skills.js";
 import { complete } from "./llm.js";
 import { ConfigError } from "./config.js";
-import { buildManifest } from "./manifest.js";
+import { buildManifest, buildEnvironment, capabilityWarnings } from "./manifest.js";
 const ROUTER_SYSTEM = "You are a skill router. Given a user request and a list of available skills (name + " +
     "description), decide which single skill should handle the request. Reply with EXACTLY one skill " +
     "name from the list, or the word none if no skill fits. Output only that one word — no punctuation, " +
@@ -57,12 +71,12 @@ async function measureIntended(skills, prompt, expected, o) {
             hits++;
         n++;
         o.onProbe?.(n, o.maxK);
-        if (n >= 3 && sequentialDecision(hits, n, o.threshold, o.conf) !== "undecided")
+        if (planDecision(o.plan, hits, n) !== "undecided")
             break;
     }
     return { dist, hits, n, errors };
 }
-function classify(actual, actualRel, intended, expected, threshold, conf) {
+function classify(actual, actualRel, intended, expected, plan) {
     const competitor = Object.entries(actual.dist)
         .filter(([s]) => s !== expected && s !== "None")
         .sort((p, q) => q[1] - p[1])[0]?.[0] ?? null;
@@ -70,11 +84,11 @@ function classify(actual, actualRel, intended, expected, threshold, conf) {
         return { verdict: "error", competitor,
             remedy: "infrastructure/LLM errors dominated — rerun; the result isn't trustworthy." };
     }
-    const actualDecision = sequentialDecision(actual.hits, actual.n, threshold, conf);
+    const actualDecision = planDecision(plan, actual.hits, actual.n);
     if (actualDecision === "above") {
         // actual is the ground truth. But if the forced-choice router rarely picks it, the description
         // reads ambiguously in isolation — a leading indicator that activation may be fragile.
-        const ambiguous = sequentialDecision(intended.hits, intended.n, threshold, conf) === "below";
+        const ambiguous = planDecision(plan, intended.hits, intended.n) === "below";
         const note = ambiguous
             ? ` — but the description reads ambiguously in isolation (intended only ${Math.round((intended.hits / intended.n) * 100)}%), ` +
                 `so activation may be fragile across models/contexts`
@@ -82,7 +96,7 @@ function classify(actual, actualRel, intended, expected, threshold, conf) {
         return { verdict: "routes-ok", competitor: null,
             remedy: `fires reliably (${Math.round(actualRel.pHat * 100)}%) — no action needed${note}.` };
     }
-    const intendedDecision = sequentialDecision(intended.hits, intended.n, threshold, conf);
+    const intendedDecision = planDecision(plan, intended.hits, intended.n);
     if (intendedDecision === "above") {
         return { verdict: "routing-miss", competitor,
             remedy: `the model picks "${expected}" when asked, but ${competitor ? `"${competitor}"` : "another skill"} ` +
@@ -102,6 +116,7 @@ export async function runDiagnose(cfg, deps = {}) {
     const choice = deps.choice ?? defaultChoice;
     const skills = listSkills(cfg.cwd);
     const present = new Set(skills.map((s) => s.name));
+    const plan = sequentialPlan(cfg.k, cfg.threshold, cfg.conf);
     const skipped = [];
     const unknown = [];
     const eligible = [];
@@ -111,7 +126,7 @@ export async function runDiagnose(cfg, deps = {}) {
         else if (!present.has(c.expected))
             unknown.push(c.expected);
         else
-            eligible.push(c);
+            eligible.push({ prompt: c.prompt, expected: c.expected });
     }
     if (unknown.length) {
         throw new ConfigError([
@@ -119,38 +134,42 @@ export async function runDiagnose(cfg, deps = {}) {
                 `(typo?). Available: ${skills.map((s) => s.name).join(", ") || "(none)"}.`,
         ]);
     }
-    const cases = [];
-    let totalCost = 0;
-    for (let i = 0; i < eligible.length; i++) {
-        const c = eligible[i];
-        const expected = c.expected;
+    const onProgress = deps.onProgress;
+    const cases = await mapPool(eligible, deps.concurrency ?? 1, async (c, i) => {
+        const { expected } = c;
         const actual = await measure(adapter, c.prompt, expected, {
-            cwd: cfg.cwd, maxK: cfg.k, threshold: cfg.threshold, conf: cfg.conf,
-            ...(cfg.model ? { model: cfg.model } : {}),
-            ...(deps.onProgress ? { onProbe: (validN, maxK) => deps.onProgress({ caseIndex: i + 1, caseTotal: eligible.length, prompt: c.prompt, phase: "actual", validN, maxK }) } : {}),
+            cwd: cfg.cwd, maxK: cfg.k, threshold: cfg.threshold, conf: cfg.conf, ...probeSettings(cfg),
+            ...(deps.budget ? { budget: deps.budget } : {}),
+            ...(onProgress ? { onProbe: (validN, maxK) => onProgress({ caseIndex: i + 1, caseTotal: eligible.length, prompt: c.prompt, phase: "actual", validN, maxK }) } : {}),
         });
         const intended = await measureIntended(skills, c.prompt, expected, {
-            maxK: cfg.k, threshold: cfg.threshold, conf: cfg.conf, choice,
+            maxK: cfg.k, plan, choice,
             ...(cfg.model ? { model: cfg.model } : {}),
-            ...(deps.onProgress ? { onProbe: (validN, maxK) => deps.onProgress({ caseIndex: i + 1, caseTotal: eligible.length, prompt: c.prompt, phase: "intended", validN, maxK }) } : {}),
+            ...(onProgress ? { onProbe: (validN, maxK) => onProgress({ caseIndex: i + 1, caseTotal: eligible.length, prompt: c.prompt, phase: "intended", validN, maxK }) } : {}),
         });
-        totalCost += actual.totalCost;
         const actualRel = reliability(actual.hits, actual.n, cfg.conf);
         const intendedRel = reliability(intended.hits, intended.n, cfg.conf);
-        const { verdict, competitor, remedy } = classify(actual, actualRel, intended, expected, cfg.threshold, cfg.conf);
-        cases.push({
+        const { verdict, competitor, remedy } = classify(actual, actualRel, intended, expected, plan);
+        return {
             prompt: c.prompt, expected,
             actual: { stats: actual, rel: actualRel },
             intended: { stats: intended, rel: intendedRel },
             verdict, competitor, remedy,
-        });
-    }
+        };
+    });
+    const totalCost = cases.reduce((s, c) => s + c.actual.stats.totalCost, 0);
     const hasProblem = cases.some((c) => c.verdict === "routing-miss" || c.verdict === "description-problem");
     const hasBad = cases.some((c) => c.verdict === "error" || c.verdict === "inconclusive");
     const exitCode = hasProblem ? 1 : hasBad ? 2 : 0;
+    const env = cases.find((c) => c.actual.stats.env)?.actual.stats.env;
+    const warnings = capabilityWarnings(cfg, adapter);
+    if (cases.some((c) => c.actual.stats.budgetExhausted)) {
+        warnings.push(`--max-cost budget reached — cases cut short are inconclusive (error if they never started)`);
+    }
     return {
         runtime: cfg.runtime, model: cfg.model ?? "(runtime default)",
         threshold: cfg.threshold, conf: cfg.conf, cases, skipped, totalCost, exitCode,
-        manifest: buildManifest("diagnose", cfg),
+        manifest: buildManifest("diagnose", cfg, buildEnvironment(cfg, env, extraSkills(env, skills), adapter)),
+        warnings,
     };
 }

package/dist/doctor.d.ts

@@ -1,4 +1,4 @@
-import type { ProbeResult, ProbeOpts } from "./types.ts";
+import { type ProbeResult, type ProbeOpts } from "./types.ts";
 export type CheckStatus = "pass" | "warn" | "fail" | "info";
 export interface Check {
     status: CheckStatus;
@@ -14,6 +14,7 @@ export interface DoctorOpts {
     runtime?: string;
     model?: string;
     skipProbe?: boolean;
+    userConfig?: boolean;
 }
 /** Injection seam so tests don't shell out to a real runtime. */
 export interface DoctorDeps {

package/dist/doctor.js

@@ -4,15 +4,17 @@
  *
  * Exit: 0 = healthy, 1 = warnings only, 2 = at least one hard failure. */
 import { spawnSync } from "node:child_process";
-import { existsSync, readdirSync, readFileSync } from "node:fs";
+import { existsSync } from "node:fs";
 import { join, resolve } from "node:path";
+import { probeSettings } from "./types.js";
 import { loadConfig } from "./config.js";
-import { wilsonInterval } from "./stats.js";
-import { kToCertify } from "./report.js";
+import { kToCertify } from "./stats.js";
 import { getAdapter } from "./adapters/index.js";
-const NAME_RE = /^name:\s*(.*)$/m;
-const DESC_RE = /^description:\s*(.*)$/m;
+import { readSkillFile, skillFolders, skillsRoot } from "./skills.js";
+import { loadProblems } from "./orchestrator.js";
+import { withSandbox } from "./sandbox.js";
 const MIN_NODE = { major: 22, minor: 18 }; // package.json engines: ">=22.18"
+const HARMLESS_PROMPT = "Reply with exactly: ok";
 function realCliInstalled(cmd) {
     const r = spawnSync(cmd, ["--version"], { stdio: "ignore", timeout: 15_000 });
     // ENOENT = not on PATH. Any other result (even a non-zero exit) means the binary exists.
@@ -51,40 +53,35 @@ export async function runDoctor(opts, deps = {}) {
     const skillsCwd = cfg ? cfg.cwd : resolve(opts.cwd);
     const runtime = cfg?.runtime ?? opts.runtime ?? "claude-code";
     // 3-6. Skills directory + each SKILL.md
-    const skillNames = new Set();
-    const skillsRoot = join(skillsCwd, ".claude", "skills");
-    if (!existsSync(skillsRoot)) {
+    const library = [];
+    const root = skillsRoot(skillsCwd);
+    if (!existsSync(root)) {
         add("fail", `no .claude/skills/ directory under ${skillsCwd}`);
     }
     else {
-        const dirs = readdirSync(skillsRoot, { withFileTypes: true }).filter((d) => d.isDirectory());
+        const dirs = skillFolders(skillsCwd);
         if (dirs.length === 0) {
             add("fail", `.claude/skills/ exists but contains no skill folders`);
         }
-        let valid = 0;
-        for (const d of dirs) {
-            const md = join(skillsRoot, d.name, "SKILL.md");
+        for (const name of dirs) {
+            const md = join(root, name, "SKILL.md");
             if (!existsSync(md)) {
-                add("fail", `skill "${d.name}" missing SKILL.md`);
+                add("fail", `skill "${name}" missing SKILL.md`);
                 continue;
             }
-            const text = readFileSync(md, "utf8");
-            const name = NAME_RE.exec(text)?.[1]?.trim() ?? "";
-            const desc = DESC_RE.exec(text)?.[1]?.trim() ?? "";
-            if (!name)
-                add("fail", `skill "${d.name}" missing name:`);
-            else if (name !== d.name)
-                add("warn", `skill "${d.name}" has name: "${name}" (doesn't match folder name)`);
-            if (!desc)
-                add("fail", `skill "${d.name}" missing description:`);
-            else if (desc === "|" || desc === ">")
-                add("warn", `skill "${d.name}" uses a multi-line description block — a single-line description triggers more reliably`);
-            skillNames.add(d.name);
-            valid++;
+            const s = readSkillFile(md, name);
+            if (!s.frontmatterName)
+                add("fail", `skill "${name}" missing name: (in a --- frontmatter block)`);
+            else if (s.frontmatterName !== name)
+                add("warn", `skill "${name}" has name: "${s.frontmatterName}" (doesn't match folder name)`);
+            if (!s.description)
+                add("fail", `skill "${name}" missing description: (in a --- frontmatter block)`);
+            library.push(s);
         }
-        if (valid > 0)
-            add("pass", `found .claude/skills/ with ${valid} skill${valid === 1 ? "" : "s"}`);
+        if (library.length > 0)
+            add("pass", `found .claude/skills/ with ${library.length} skill${library.length === 1 ? "" : "s"}`);
     }
+    const skillNames = new Set(library.map((s) => s.name));
     // 7-9. Config-dependent checks
     if (cfg) {
         const missing = cfg.cases
@@ -98,13 +95,18 @@ export async function runDoctor(opts, deps = {}) {
             add("warn", `config has only decoy cases — no skill is being verified to fire`);
         }
         // a strict threshold at a small k can be mathematically impossible to certify
-        if (wilsonInterval(cfg.k, cfg.k, cfg.conf).lo <= cfg.threshold) {
+        const need = kToCertify(cfg.threshold, cfg.conf);
+        if (need > cfg.k) {
             add("warn", `threshold ${Math.round(cfg.threshold * 100)}% with k=${cfg.k} cannot certify a pass ` +
-                `even on a perfect run — need ~k=${kToCertify(cfg.threshold, cfg.conf)} (or lower the threshold)`);
+                `even on a perfect run — need k≥${need} (or lower the threshold)`);
+        }
+        if (cfg.userConfig) {
+            add("warn", `userConfig is on: probes load your personal runtime config, so results won't reproduce in CI`);
         }
     }
     // 10-11. Runtime + a harmless live probe (verifies the CLI is installed AND authed)
-    add("info", `runtime: ${runtime}  model: ${cfg?.model ?? opts.model ?? "(runtime default)"}`);
+    const model = cfg?.model ?? opts.model;
+    add("info", `runtime: ${runtime}  model: ${model ?? "(runtime default)"}`);
     const known = runtime === "claude-code" || runtime === "opencode";
     const bin = runtime === "claude-code" ? "claude" : runtime === "opencode" ? "opencode" : null;
     let cliOk = false;
@@ -122,14 +124,46 @@ export async function runDoctor(opts, deps = {}) {
         add("info", `probe skipped (--skip-probe)`);
     }
     else if (known && cliOk) {
-        const r = await probe(runtime, "Reply with exactly: ok", {
-            cwd: skillsCwd, timeoutMs: 60_000,
-            ...(cfg?.model ?? opts.model ? { model: (cfg?.model ?? opts.model) } : {}),
-        });
-        if (r.status === "ok")
-            add("pass", `${runtime} probe succeeded (CLI is authenticated)`);
-        else
+        const settings = {
+            ...(cfg ? probeSettings(cfg) : {}),
+            ...(model ? { model } : {}),
+            ...(opts.userConfig ? { userConfig: true } : {}),
+            timeoutMs: 60_000,
+        };
+        const r = await probe(runtime, HARMLESS_PROMPT, { cwd: skillsCwd, ...settings });
+        if (r.status !== "ok") {
             add("fail", `${runtime} probe failed — check auth/login: ${r.error ?? "unknown error"}`);
+        }
+        else {
+            add("pass", `${runtime} probe succeeded (CLI is authenticated)` +
+                (r.env?.runtimeVersion ? ` — ${bin} ${r.env.runtimeVersion}` : "") +
+                (r.env?.model && !model ? `, default model ${r.env.model}` : ""));
+            // what the runtime ACTUALLY loads (only runtimes that report it)
+            if (r.env?.loadedSkills && library.length) {
+                const { missing } = loadProblems(r.env, library.map((s) => s.name), library);
+                if (missing.length) {
+                    add("fail", `the runtime did not load: ${missing.join(", ")} — it rejected or ignored these ` +
+                        `SKILL.md files, so they can never fire`);
+                }
+                else {
+                    add("pass", `the runtime loads all ${library.length} library skill${library.length === 1 ? "" : "s"}`);
+                }
+                // built-ins load in every probe; a library skill sharing a built-in's name is ambiguous
+                const empty = await withSandbox(skillsCwd, [], (sb) => probe(runtime, HARMLESS_PROMPT, { cwd: sb.cwd, ...settings }));
+                const builtins = empty.status === "ok" ? empty.env?.loadedSkills ?? [] : [];
+                const clash = library.filter((s) => builtins.includes(s.name) ||
+                    (s.frontmatterName !== undefined && builtins.includes(s.frontmatterName)));
+                for (const s of clash) {
+                    add("warn", `skill "${s.name}" has the same name as a ${runtime} built-in skill — the two compete ` +
+                        `for the same trigger; consider renaming it`);
+                }
+                if (builtins.length) {
+                    add("info", `${builtins.length} ${runtime} built-in skill(s) load alongside your library in every probe ` +
+                        `(${builtins.slice(0, 8).join(", ")}${builtins.length > 8 ? ", …" : ""}) — they are part of the real ` +
+                        `environment and can steal triggers`);
+                }
+            }
+        }
     }
     const hasFail = checks.some((c) => c.status === "fail");
     const hasWarn = checks.some((c) => c.status === "warn");