skill-probe 0.8.0 → 0.10.0

package/dist/context.d.ts

@@ -1,12 +1,20 @@
-import type { Config } from "./types.ts";
+/** `skill-probe context` — activation-rate-BY-CONTEXT. The audit measures every skill co-loaded
+ * (the real, hard condition). This command additionally measures each skill in ISOLATION (only that
+ * skill present) and compares the two: it catches the "fires fine alone, fails under load" case that
+ * a single aggregate rate hides. The drop is tested with Fisher's exact test, so interference is
+ * reported as a real effect (p-value), not eyeballed.
+ *
+ * Every condition (isolation, co-loaded, each leave-one-out ablation) runs in a sandbox built the
+ * same way — see sandbox.ts — so the ONLY difference between conditions is which skills are
+ * present. When the runtime reports what it loaded, each condition is checked against what it was
+ * meant to contain; a mismatch makes the comparison untrustworthy instead of silently wrong. */
+import { type Config } from "./types.ts";
 import { reliability } from "./stats.ts";
-import { type RunStats } from "./orchestrator.ts";
-/** Build a throwaway project dir whose .claude/skills/ holds ONLY the named skills (copied from
- * srcCwd). Returns its path plus a cleanup(). The probe runs against this dir to load a subset. */
-export declare function buildContextDir(srcCwd: string, skillNames: string[]): {
-    cwd: string;
-    cleanup: () => void;
-};
+import { type Budget, type RunStats } from "./orchestrator.ts";
+import { buildSandbox } from "./sandbox.ts";
+import { type RunManifest } from "./manifest.ts";
+/** Kept for compatibility: a sandbox holding only the named skills (plus instruction files). */
+export declare const buildContextDir: typeof buildSandbox;
 type Rel = ReturnType<typeof reliability>;
 interface Condition {
     stats: RunStats;
@@ -24,6 +32,8 @@ export interface AblationResult {
     fisherP: number;
     fisherPAdj: number;
     untrustworthy: boolean;
+    /** why the run isn't a valid comparison, when the runtime loaded the wrong skill set */
+    loadIssue?: string;
     culprit: boolean;
 }
 export interface ContextCaseResult {
@@ -37,8 +47,14 @@ export interface ContextCaseResult {
      * test per case is a FAMILY, and raw p<0.05 over a 40-case library expects ~2 false flags by
      * chance alone. Interference is decided on THIS value. (1 for untrustworthy cases.) */
     fisherPAdj: number;
-    /** infra failures dominated one side — the comparison is not trustworthy (NOT a behavioral result) */
+    /** infra failures dominated one side, or a condition loaded the wrong skills — the comparison is
+     * not trustworthy (NOT a behavioral result) */
     untrustworthy: boolean;
+    /** why a condition isn't valid, when the runtime loaded the wrong skill set */
+    loadIssue?: string;
+    /** skills OUTSIDE the library (e.g. runtime built-ins) that fired in the co-loaded run — they
+     * can't be ablated, only avoided (narrow or rename the expected skill) */
+    externalThieves?: string[];
     /** fires reliably alone but drops significantly (BH-corrected) when the library is co-loaded */
     interference: boolean;
     /** leave-one-out runs (only with { ablate: true }, only for interference cases). Empty array =
@@ -56,6 +72,10 @@ export interface ContextAuditResult {
     totalCost: number;
     /** 0 = no interference; 1 = a skill is suppressed under load; 2 = an untrustworthy (infra) case */
     exitCode: 0 | 1 | 2;
+    /** who/what/when of this run — makes reports citable and comparable */
+    manifest: RunManifest;
+    /** run-level caveats (built-in name clashes, budget cut-offs) */
+    warnings: string[];
 }
 export interface ContextProgress {
     caseIndex: number;
@@ -74,6 +94,9 @@ export interface ContextOpts {
     /** max suspect siblings to ablate per flagged case, ranked by probes stolen (default 3) */
     suspects?: number;
     onProgress?: (e: ContextProgress) => void;
+    /** cases (and ablation runs) probed in parallel (default 1) */
+    concurrency?: number;
+    budget?: Budget;
 }
 export declare function runContextAudit(cfg: Config, optsOrProgress?: ContextOpts | ((e: ContextProgress) => void)): Promise<ContextAuditResult>;
 export {};

package/dist/context.js

@@ -2,38 +2,41 @@
  * (the real, hard condition). This command additionally measures each skill in ISOLATION (only that
  * skill present) and compares the two: it catches the "fires fine alone, fails under load" case that
  * a single aggregate rate hides. The drop is tested with Fisher's exact test, so interference is
- * reported as a real effect (p-value), not eyeballed. */
-import { mkdtempSync, mkdirSync, cpSync, rmSync, existsSync } from "node:fs";
-import { tmpdir } from "node:os";
-import { join } from "node:path";
+ * reported as a real effect (p-value), not eyeballed.
+ *
+ * Every condition (isolation, co-loaded, each leave-one-out ablation) runs in a sandbox built the
+ * same way — see sandbox.ts — so the ONLY difference between conditions is which skills are
+ * present. When the runtime reports what it loaded, each condition is checked against what it was
+ * meant to contain; a mismatch makes the comparison untrustworthy instead of silently wrong. */
+import { probeSettings } from "./types.js";
 import { reliability, fisherExact2x2, bhAdjust } from "./stats.js";
-import { measure, infraUntrustworthy } from "./orchestrator.js";
+import { measure, mapPool, infraUntrustworthy, loadProblems, extraSkills, builtinClashes, } from "./orchestrator.js";
 import { getAdapter } from "./adapters/index.js";
-import { listSkills } from "./gen.js";
+import { listSkills } from "./skills.js";
+import { buildSandbox, withSandbox } from "./sandbox.js";
 import { ConfigError } from "./config.js";
-/** Build a throwaway project dir whose .claude/skills/ holds ONLY the named skills (copied from
- * srcCwd). Returns its path plus a cleanup(). The probe runs against this dir to load a subset. */
-export function buildContextDir(srcCwd, skillNames) {
-    const root = mkdtempSync(join(tmpdir(), "skill-probe-ctx-"));
-    const destSkills = join(root, ".claude", "skills");
-    mkdirSync(destSkills, { recursive: true });
-    for (const name of skillNames) {
-        const src = join(srcCwd, ".claude", "skills", name);
-        if (!existsSync(src))
-            throw new Error(`skill not found in ${srcCwd}: ${name}`);
-        cpSync(src, join(destSkills, name), { recursive: true });
-    }
-    return { cwd: root, cleanup: () => rmSync(root, { recursive: true, force: true }) };
+import { buildManifest, buildEnvironment, capabilityWarnings } from "./manifest.js";
+/** Kept for compatibility: a sandbox holding only the named skills (plus instruction files). */
+export const buildContextDir = buildSandbox;
+/** "" when the condition loaded exactly what it should have; otherwise a human-readable reason. */
+function describeLoadIssue(label, stats, intended, library, ambiguous) {
+    const { missing, unexpected } = loadProblems(stats.env, intended, library, ambiguous);
+    const parts = [];
+    if (missing.length)
+        parts.push(`did not load ${missing.join(", ")}`);
+    if (unexpected.length)
+        parts.push(`also loaded ${unexpected.join(", ")}`);
+    return parts.length ? `${label}: the runtime ${parts.join(" and ")}` : "";
 }
+const fisherVs = (a, b) => (a.n > 0 && b.n > 0) ? fisherExact2x2(a.hits, a.n - a.hits, b.hits, b.n - b.hits) : 1;
 export async function runContextAudit(cfg, optsOrProgress = {}) {
     const opts = typeof optsOrProgress === "function" ? { onProgress: optsOrProgress } : optsOrProgress;
     const onProgress = opts.onProgress;
     const adapter = getAdapter(cfg.runtime);
     const library = listSkills(cfg.cwd);
-    const present = new Set(library.map((s) => s.name));
-    const measured = [];
+    const libraryNames = library.map((s) => s.name);
+    const present = new Set(libraryNames);
     const skipped = [];
-    let totalCost = 0;
     // decoys (null expected) have nothing to isolate → skipped. An expected skill that ISN'T in the
     // library is almost always a config typo: fail loud BEFORE spending probes, never silently skip.
     const eligible = [];
@@ -44,54 +47,73 @@ export async function runContextAudit(cfg, optsOrProgress = {}) {
         else if (!present.has(c.expected))
             unknown.push(c.expected);
         else
-            eligible.push(c);
+            eligible.push({ prompt: c.prompt, expected: c.expected });
     }
     if (unknown.length) {
         throw new ConfigError([
             `expected skill(s) not in ${cfg.cwd}/.claude/skills/: ${[...new Set(unknown)].join(", ")} ` +
-                `(typo?). Available: ${library.map((s) => s.name).join(", ") || "(none)"}.`,
+                `(typo?). Available: ${libraryNames.join(", ") || "(none)"}.`,
         ]);
     }
-    const common = { maxK: cfg.k, threshold: cfg.threshold, conf: cfg.conf, ...(cfg.model ? { model: cfg.model } : {}) };
-    for (let i = 0; i < eligible.length; i++) {
-        const c = eligible[i];
-        const expected = c.expected;
-        // isolation: a temp project containing only this skill
-        const ctx = buildContextDir(cfg.cwd, [expected]);
-        let iso;
-        try {
-            iso = await measure(adapter, c.prompt, expected, {
-                cwd: ctx.cwd, ...common,
-                ...(onProgress ? { onProbe: (validN, maxK) => onProgress({ caseIndex: i + 1, caseTotal: eligible.length, prompt: c.prompt, condition: "isolation", validN, maxK }) } : {}),
-            });
-        }
-        finally {
-            ctx.cleanup();
-        }
-        // co-loaded: the real library at cfg.cwd
-        const co = await measure(adapter, c.prompt, expected, {
-            cwd: cfg.cwd, ...common,
-            ...(onProgress ? { onProbe: (validN, maxK) => onProgress({ caseIndex: i + 1, caseTotal: eligible.length, prompt: c.prompt, condition: "co-loaded", validN, maxK }) } : {}),
+    const common = {
+        maxK: cfg.k, threshold: cfg.threshold, conf: cfg.conf, ...probeSettings(cfg),
+        ...(opts.budget ? { budget: opts.budget } : {}),
+    };
+    const progress = (caseIndex, caseTotal, prompt, condition, removed) => onProgress
+        ? { onProbe: (validN, maxK) => onProgress({ caseIndex, caseTotal, prompt, condition, validN, maxK, ...(removed ? { removed } : {}) }) }
+        : {};
+    // the full library, shared read-only by every co-loaded run
+    const measured = await withSandbox(cfg.cwd, libraryNames, (full) => mapPool(eligible, opts.concurrency ?? 1, async (c, i) => {
+        const iso = await withSandbox(cfg.cwd, [c.expected], (sb) => measure(adapter, c.prompt, c.expected, {
+            cwd: sb.cwd, ...common, ...progress(i + 1, eligible.length, c.prompt, "isolation"),
+        }));
+        const co = await measure(adapter, c.prompt, c.expected, {
+            cwd: full.cwd, ...common, ...progress(i + 1, eligible.length, c.prompt, "co-loaded"),
         });
-        totalCost += iso.totalCost + co.totalCost;
         const isoRel = reliability(iso.hits, iso.n, cfg.conf);
         const coRel = reliability(co.hits, co.n, cfg.conf);
-        // 2x2: rows = isolation/co-loaded, cols = hit/miss
-        const fisherP = (iso.n > 0 && co.n > 0)
-            ? fisherExact2x2(iso.hits, iso.n - iso.hits, co.hits, co.n - co.hits)
-            : 1;
-        const deltaP = coRel.pHat - isoRel.pHat;
-        // if infra failures dominated either side, the comparison isn't a behavioral result at all —
-        // it must never read as "no interference / pass" just because fisherP defaulted to 1.
+        // load-set integrity is judged after all cases (it may need the runtime's built-in names)
         const untrustworthy = infraUntrustworthy(iso) || infraUntrustworthy(co);
-        // interference is decided AFTER the loop, once the whole family of comparisons is known
-        // (Benjamini-Hochberg) — placeholders here.
-        measured.push({
-            prompt: c.prompt, expected,
+        const externalThieves = Object.keys(co.dist).filter((s) => s !== c.expected && s !== "None" && !present.has(s));
+        // interference is decided AFTER all cases, once the whole family is known (BH) — placeholders
+        return {
+            prompt: c.prompt, expected: c.expected,
             isolation: { stats: iso, rel: isoRel },
             coLoaded: { stats: co, rel: coRel },
-            deltaP, fisherP, fisherPAdj: 1, untrustworthy, interference: false,
-        });
+            deltaP: coRel.pHat - isoRel.pHat, fisherP: fisherVs(iso, co), fisherPAdj: 1,
+            untrustworthy, interference: false,
+            ...(externalThieves.length ? { externalThieves } : {}),
+        };
+    }));
+    // Did each condition load exactly the skills it should? A library skill that shares a name with a
+    // runtime built-in looks "loaded" everywhere, so when anything looks wrong, learn the built-in
+    // names once (an empty sandbox) and leave those ambiguous names out of the check.
+    let ambiguous = new Set();
+    let extraCost = 0;
+    let builtinsProbe;
+    // memoized promise (not a flag): concurrent ablation runs must all wait for the same answer
+    const learnBuiltins = () => (builtinsProbe ??= (async () => {
+        const r = await withSandbox(cfg.cwd, [], (sb) => adapter.probe("Reply with exactly: ok", { cwd: sb.cwd, ...probeSettings(cfg) }));
+        extraCost += r.cost;
+        opts.budget?.add(r.cost);
+        if (r.status === "ok" && r.env?.loadedSkills)
+            ambiguous = new Set(builtinClashes(r.env.loadedSkills, library));
+    })());
+    const looksWrong = (st, intended) => loadProblems(st.env, intended, library).unexpected.length > 0;
+    if (measured.some((m) => looksWrong(m.isolation.stats, [m.expected]) || looksWrong(m.coLoaded.stats, libraryNames))) {
+        await learnBuiltins();
+    }
+    for (const m of measured) {
+        const loadIssue = [
+            describeLoadIssue("isolation", m.isolation.stats, [m.expected], library, ambiguous),
+            describeLoadIssue("co-loaded", m.coLoaded.stats, libraryNames, library, ambiguous),
+        ].filter(Boolean).join("; ");
+        // a condition that didn't contain what it should isn't a behavioral result — it must never
+        // read as "no interference / pass".
+        if (loadIssue) {
+            m.loadIssue = loadIssue;
+            m.untrustworthy = true;
+        }
     }
     // One Fisher test per case = a family of comparisons: correct across the trustworthy ones so a
     // big library doesn't buy false interference flags by chance (raw p<0.05 × 40 cases ≈ 2 flukes).
@@ -111,46 +133,39 @@ export async function runContextAudit(cfg, optsOrProgress = {}) {
     // BH-corrected across all ablation runs (their own family). A significant recovery NAMES the
     // thief; no suspects in the outcome distribution means the drop was dilution, not theft.
     if (opts.ablate) {
-        const libraryNames = library.map((s) => s.name);
         const flagged = measured.filter((m) => m.interference);
         const cap = opts.suspects ?? 3;
-        const allRuns = [];
-        for (let fi = 0; fi < flagged.length; fi++) {
-            const m = flagged[fi];
-            const suspects = Object.entries(m.coLoaded.stats.dist)
-                .filter(([name]) => name !== m.expected && name !== "None")
-                .sort((x, y) => y[1] - x[1])
-                .slice(0, cap);
+        const jobs = [];
+        flagged.forEach((m, fi) => {
             m.ablation = [];
-            for (const [suspect, stolen] of suspects) {
-                const ctx = buildContextDir(cfg.cwd, libraryNames.filter((n) => n !== suspect));
-                let st;
-                try {
-                    st = await measure(adapter, m.prompt, m.expected, {
-                        cwd: ctx.cwd, ...common,
-                        ...(onProgress ? { onProbe: (validN, maxK) => onProgress({ caseIndex: fi + 1, caseTotal: flagged.length, prompt: m.prompt, condition: "ablation", removed: suspect, validN, maxK }) } : {}),
-                    });
-                }
-                finally {
-                    ctx.cleanup();
-                }
-                totalCost += st.totalCost;
-                const rel = reliability(st.hits, st.n, cfg.conf);
-                const co = m.coLoaded.stats;
-                const fisherP = (st.n > 0 && co.n > 0)
-                    ? fisherExact2x2(st.hits, st.n - st.hits, co.hits, co.n - co.hits)
-                    : 1;
-                const abl = {
-                    removed: suspect, stolen, stats: st, rel,
-                    deltaVsCoLoaded: rel.pHat - m.coLoaded.rel.pHat,
-                    fisherP, fisherPAdj: 1,
-                    untrustworthy: infraUntrustworthy(st),
-                    culprit: false,
-                };
-                m.ablation.push(abl);
-                allRuns.push(abl);
-            }
-        }
+            Object.entries(m.coLoaded.stats.dist)
+                .filter(([name]) => name !== m.expected && name !== "None" && present.has(name))
+                .sort((x, y) => y[1] - x[1])
+                .slice(0, cap)
+                .forEach(([suspect, stolen]) => jobs.push({ m, fi, suspect, stolen }));
+        });
+        const runs = await mapPool(jobs, opts.concurrency ?? 1, async ({ m, fi, suspect, stolen }) => {
+            const kept = libraryNames.filter((n) => n !== suspect);
+            const st = await withSandbox(cfg.cwd, kept, (sb) => measure(adapter, m.prompt, m.expected, {
+                cwd: sb.cwd, ...common, ...progress(fi + 1, flagged.length, m.prompt, "ablation", suspect),
+            }));
+            const rel = reliability(st.hits, st.n, cfg.conf);
+            if (looksWrong(st, kept))
+                await learnBuiltins();
+            const loadIssue = describeLoadIssue(`without ${suspect}`, st, kept, library, ambiguous);
+            const abl = {
+                removed: suspect, stolen, stats: st, rel,
+                deltaVsCoLoaded: rel.pHat - m.coLoaded.rel.pHat,
+                fisherP: fisherVs(st, m.coLoaded.stats), fisherPAdj: 1,
+                untrustworthy: infraUntrustworthy(st) || Boolean(loadIssue),
+                ...(loadIssue ? { loadIssue } : {}),
+                culprit: false,
+            };
+            return { m, abl };
+        });
+        for (const { m, abl } of runs)
+            m.ablation.push(abl);
+        const allRuns = runs.map((r) => r.abl);
         const ablFamily = allRuns.filter((r) => !r.untrustworthy);
         const ablAdj = bhAdjust(ablFamily.map((r) => r.fisherP));
         ablFamily.forEach((r, j) => { r.fisherPAdj = ablAdj[j]; });
@@ -163,9 +178,23 @@ export async function runContextAudit(cfg, optsOrProgress = {}) {
     const exitCode = measured.some((m) => m.interference) ? 1
         : measured.some((m) => m.untrustworthy) ? 2
             : 0;
+    const totalCost = extraCost + measured.reduce((s, m) => s + m.isolation.stats.totalCost + m.coLoaded.stats.totalCost +
+        (m.ablation ?? []).reduce((t, a) => t + a.stats.totalCost, 0), 0);
+    const env = measured.find((m) => m.coLoaded.stats.env)?.coLoaded.stats.env;
+    const warnings = capabilityWarnings(cfg, adapter);
+    if (ambiguous.size) {
+        warnings.push(`${[...ambiguous].join(", ")} share${ambiguous.size === 1 ? "s" : ""} a name with a ${cfg.runtime} ` +
+            `built-in skill — the two compete for the same trigger, and load checks can't tell them apart (rename recommended)`);
+    }
+    const allStats = measured.flatMap((m) => [m.isolation.stats, m.coLoaded.stats, ...(m.ablation ?? []).map((a) => a.stats)]);
+    if (allStats.some((st) => st.budgetExhausted)) {
+        warnings.push(`--max-cost budget reached — some conditions were cut short (inconclusive or error, never a clean result)`);
+    }
     return {
         runtime: cfg.runtime, model: cfg.model ?? "(runtime default)",
         threshold: cfg.threshold, conf: cfg.conf, librarySize: library.length,
         cases: measured, skipped, totalCost, exitCode,
+        manifest: buildManifest("context", cfg, buildEnvironment(cfg, env, extraSkills(env, library), adapter)),
+        warnings,
     };
 }

package/dist/diagnose.d.ts

@@ -11,10 +11,11 @@
  * We measure ACTUAL activation (the real runtime probe) and INTENDED routing (a forced-choice: the
  * model is shown the skill descriptions and asked which one applies, with no auto-execution), then
  * classify the gap between them. */
-import type { Config } from "./types.ts";
+import { type Config } from "./types.ts";
 import { reliability } from "./stats.ts";
-import { type RunStats } from "./orchestrator.ts";
-import { type SkillInfo } from "./gen.ts";
+import { type Budget, type RunStats } from "./orchestrator.ts";
+import { type SkillInfo } from "./skills.ts";
+import { type RunManifest } from "./manifest.ts";
 export type DiagVerdict = "routes-ok" | "routing-miss" | "description-problem" | "inconclusive" | "error";
 /** Returns the chosen skill name, "none", or null when the answer is unusable (LLM error/garbage). */
 export type ChoiceFn = (skills: SkillInfo[], prompt: string, model?: string) => Promise<string | null>;
@@ -50,6 +51,10 @@ export interface DiagnoseResult {
     skipped: string[];
     totalCost: number;
     exitCode: 0 | 1 | 2;
+    /** who/what/when of this run — makes reports citable and comparable */
+    manifest: RunManifest;
+    /** run-level caveats (runtime capabilities, budget cut-offs) */
+    warnings: string[];
 }
 export interface DiagProgress {
     caseIndex: number;
@@ -61,8 +66,12 @@ export interface DiagProgress {
 }
 export declare function normalizeChoice(text: string, skills: SkillInfo[]): string | null;
 export declare const defaultChoice: ChoiceFn;
-export declare function runDiagnose(cfg: Config, deps?: {
+export interface DiagnoseOpts {
     choice?: ChoiceFn;
     onProgress?: (e: DiagProgress) => void;
-}): Promise<DiagnoseResult>;
+    /** cases diagnosed in parallel (default 1) */
+    concurrency?: number;
+    budget?: Budget;
+}
+export declare function runDiagnose(cfg: Config, deps?: DiagnoseOpts): Promise<DiagnoseResult>;
 export {};

package/dist/diagnose.js

@@ -1,9 +1,24 @@
-import { reliability, sequentialDecision } from "./stats.js";
-import { measure, infraUntrustworthy } from "./orchestrator.js";
+/** `skill-probe diagnose` — when a skill fails to fire, WHY? Two very different root causes need
+ * two very different fixes, and you can't tell them apart from the activation rate alone:
+ *
+ *   - DESCRIPTION PROBLEM: the model, asked point-blank "which skill handles this?", doesn't even
+ *     pick the expected skill. Its description fails to communicate what it's for. Remedy: rewrite
+ *     the description (`skill-probe fix`).
+ *   - ROUTING MISS: the model DOES pick the expected skill when asked — it understands the
+ *     description — but a sibling wins at actual activation time. Remedy: deconflict the sibling /
+ *     inspect with `skill-probe context`. Rewriting this skill's description won't help.
+ *
+ * We measure ACTUAL activation (the real runtime probe) and INTENDED routing (a forced-choice: the
+ * model is shown the skill descriptions and asked which one applies, with no auto-execution), then
+ * classify the gap between them. */
+import { probeSettings } from "./types.js";
+import { reliability, planDecision, sequentialPlan } from "./stats.js";
+import { measure, mapPool, infraUntrustworthy, extraSkills } from "./orchestrator.js";
 import { getAdapter } from "./adapters/index.js";
-import { listSkills } from "./gen.js";
+import { listSkills } from "./skills.js";
 import { complete } from "./llm.js";
 import { ConfigError } from "./config.js";
+import { buildManifest, buildEnvironment, capabilityWarnings } from "./manifest.js";
 const ROUTER_SYSTEM = "You are a skill router. Given a user request and a list of available skills (name + " +
     "description), decide which single skill should handle the request. Reply with EXACTLY one skill " +
     "name from the list, or the word none if no skill fits. Output only that one word — no punctuation, " +
@@ -56,12 +71,12 @@ async function measureIntended(skills, prompt, expected, o) {
             hits++;
         n++;
         o.onProbe?.(n, o.maxK);
-        if (n >= 3 && sequentialDecision(hits, n, o.threshold, o.conf) !== "undecided")
+        if (planDecision(o.plan, hits, n) !== "undecided")
             break;
     }
     return { dist, hits, n, errors };
 }
-function classify(actual, actualRel, intended, expected, threshold, conf) {
+function classify(actual, actualRel, intended, expected, plan) {
     const competitor = Object.entries(actual.dist)
         .filter(([s]) => s !== expected && s !== "None")
         .sort((p, q) => q[1] - p[1])[0]?.[0] ?? null;
@@ -69,11 +84,11 @@ function classify(actual, actualRel, intended, expected, threshold, conf) {
         return { verdict: "error", competitor,
             remedy: "infrastructure/LLM errors dominated — rerun; the result isn't trustworthy." };
     }
-    const actualDecision = sequentialDecision(actual.hits, actual.n, threshold, conf);
+    const actualDecision = planDecision(plan, actual.hits, actual.n);
     if (actualDecision === "above") {
         // actual is the ground truth. But if the forced-choice router rarely picks it, the description
         // reads ambiguously in isolation — a leading indicator that activation may be fragile.
-        const ambiguous = sequentialDecision(intended.hits, intended.n, threshold, conf) === "below";
+        const ambiguous = planDecision(plan, intended.hits, intended.n) === "below";
         const note = ambiguous
             ? ` — but the description reads ambiguously in isolation (intended only ${Math.round((intended.hits / intended.n) * 100)}%), ` +
                 `so activation may be fragile across models/contexts`
@@ -81,7 +96,7 @@ function classify(actual, actualRel, intended, expected, threshold, conf) {
         return { verdict: "routes-ok", competitor: null,
             remedy: `fires reliably (${Math.round(actualRel.pHat * 100)}%) — no action needed${note}.` };
     }
-    const intendedDecision = sequentialDecision(intended.hits, intended.n, threshold, conf);
+    const intendedDecision = planDecision(plan, intended.hits, intended.n);
     if (intendedDecision === "above") {
         return { verdict: "routing-miss", competitor,
             remedy: `the model picks "${expected}" when asked, but ${competitor ? `"${competitor}"` : "another skill"} ` +
@@ -101,6 +116,7 @@ export async function runDiagnose(cfg, deps = {}) {
     const choice = deps.choice ?? defaultChoice;
     const skills = listSkills(cfg.cwd);
     const present = new Set(skills.map((s) => s.name));
+    const plan = sequentialPlan(cfg.k, cfg.threshold, cfg.conf);
     const skipped = [];
     const unknown = [];
     const eligible = [];
@@ -110,7 +126,7 @@ export async function runDiagnose(cfg, deps = {}) {
         else if (!present.has(c.expected))
             unknown.push(c.expected);
         else
-            eligible.push(c);
+            eligible.push({ prompt: c.prompt, expected: c.expected });
     }
     if (unknown.length) {
         throw new ConfigError([
@@ -118,37 +134,42 @@ export async function runDiagnose(cfg, deps = {}) {
                 `(typo?). Available: ${skills.map((s) => s.name).join(", ") || "(none)"}.`,
         ]);
     }
-    const cases = [];
-    let totalCost = 0;
-    for (let i = 0; i < eligible.length; i++) {
-        const c = eligible[i];
-        const expected = c.expected;
+    const onProgress = deps.onProgress;
+    const cases = await mapPool(eligible, deps.concurrency ?? 1, async (c, i) => {
+        const { expected } = c;
         const actual = await measure(adapter, c.prompt, expected, {
-            cwd: cfg.cwd, maxK: cfg.k, threshold: cfg.threshold, conf: cfg.conf,
-            ...(cfg.model ? { model: cfg.model } : {}),
-            ...(deps.onProgress ? { onProbe: (validN, maxK) => deps.onProgress({ caseIndex: i + 1, caseTotal: eligible.length, prompt: c.prompt, phase: "actual", validN, maxK }) } : {}),
+            cwd: cfg.cwd, maxK: cfg.k, threshold: cfg.threshold, conf: cfg.conf, ...probeSettings(cfg),
+            ...(deps.budget ? { budget: deps.budget } : {}),
+            ...(onProgress ? { onProbe: (validN, maxK) => onProgress({ caseIndex: i + 1, caseTotal: eligible.length, prompt: c.prompt, phase: "actual", validN, maxK }) } : {}),
         });
         const intended = await measureIntended(skills, c.prompt, expected, {
-            maxK: cfg.k, threshold: cfg.threshold, conf: cfg.conf, choice,
+            maxK: cfg.k, plan, choice,
             ...(cfg.model ? { model: cfg.model } : {}),
-            ...(deps.onProgress ? { onProbe: (validN, maxK) => deps.onProgress({ caseIndex: i + 1, caseTotal: eligible.length, prompt: c.prompt, phase: "intended", validN, maxK }) } : {}),
+            ...(onProgress ? { onProbe: (validN, maxK) => onProgress({ caseIndex: i + 1, caseTotal: eligible.length, prompt: c.prompt, phase: "intended", validN, maxK }) } : {}),
         });
-        totalCost += actual.totalCost;
         const actualRel = reliability(actual.hits, actual.n, cfg.conf);
         const intendedRel = reliability(intended.hits, intended.n, cfg.conf);
-        const { verdict, competitor, remedy } = classify(actual, actualRel, intended, expected, cfg.threshold, cfg.conf);
-        cases.push({
+        const { verdict, competitor, remedy } = classify(actual, actualRel, intended, expected, plan);
+        return {
             prompt: c.prompt, expected,
             actual: { stats: actual, rel: actualRel },
             intended: { stats: intended, rel: intendedRel },
             verdict, competitor, remedy,
-        });
-    }
+        };
+    });
+    const totalCost = cases.reduce((s, c) => s + c.actual.stats.totalCost, 0);
     const hasProblem = cases.some((c) => c.verdict === "routing-miss" || c.verdict === "description-problem");
     const hasBad = cases.some((c) => c.verdict === "error" || c.verdict === "inconclusive");
     const exitCode = hasProblem ? 1 : hasBad ? 2 : 0;
+    const env = cases.find((c) => c.actual.stats.env)?.actual.stats.env;
+    const warnings = capabilityWarnings(cfg, adapter);
+    if (cases.some((c) => c.actual.stats.budgetExhausted)) {
+        warnings.push(`--max-cost budget reached — cases cut short are inconclusive (error if they never started)`);
+    }
     return {
         runtime: cfg.runtime, model: cfg.model ?? "(runtime default)",
         threshold: cfg.threshold, conf: cfg.conf, cases, skipped, totalCost, exitCode,
+        manifest: buildManifest("diagnose", cfg, buildEnvironment(cfg, env, extraSkills(env, skills), adapter)),
+        warnings,
     };
 }

package/dist/doctor.d.ts

@@ -1,4 +1,4 @@
-import type { ProbeResult, ProbeOpts } from "./types.ts";
+import { type ProbeResult, type ProbeOpts } from "./types.ts";
 export type CheckStatus = "pass" | "warn" | "fail" | "info";
 export interface Check {
     status: CheckStatus;
@@ -14,6 +14,7 @@ export interface DoctorOpts {
     runtime?: string;
     model?: string;
     skipProbe?: boolean;
+    userConfig?: boolean;
 }
 /** Injection seam so tests don't shell out to a real runtime. */
 export interface DoctorDeps {