skill-probe 0.9.0 → 0.10.0

package/dist/eval.d.ts

@@ -1,8 +1,8 @@
 /** The audit: run every case co-loaded, compute reliability (with CI), detect trigger-theft,
  * and keep infrastructure failure distinct from behavioral failure (finding #1). */
-import type { Config } from "./types.ts";
+import { type Config } from "./types.ts";
 import { reliability } from "./stats.ts";
-import { type RunStats } from "./orchestrator.ts";
+import { type Budget, type RunStats } from "./orchestrator.ts";
 import { type RunManifest } from "./manifest.ts";
 /** Four honest states. `error` = the runtime failed enough that the measurement is untrustworthy. */
 export type Verdict = "pass" | "fail" | "inconclusive" | "error";
@@ -11,10 +11,14 @@ export interface CaseResult {
     expected: string | null;
     stats: RunStats;
     reliability: ReturnType<typeof reliability>;
-    /** non-expected, non-None skills that fired = interference / trigger-theft */
+    /** non-expected, non-None skills that fired = interference / trigger-theft. A warning unless
+     * failOnTheft is set: one stray firing out of k is not statistical evidence of a problem. */
     theft: string[];
     decision: "above" | "below" | "undecided";
     verdict: Verdict;
+    /** the runtime reported its loaded skills and the expected skill wasn't among them — it can
+     * never fire (usually a SKILL.md the runtime rejected) */
+    notLoaded?: boolean;
 }
 export interface AuditResult {
     runtime: string;
@@ -28,10 +32,12 @@ export interface AuditResult {
         inconclusive: number;
         error: number;
     };
-    /** 0 = all pass; 1 = a behavioral failure/theft; 2 = inconclusive or infrastructure error */
+    /** 0 = all pass; 1 = a behavioral failure; 2 = inconclusive or infrastructure error */
     exitCode: 0 | 1 | 2;
     /** who/what/when of this run — makes reports citable and baselines comparable */
     manifest: RunManifest;
+    /** run-level problems worth reading before the verdicts (unloaded skills, personal config…) */
+    warnings: string[];
 }
 export interface ProgressEvent {
     caseIndex: number;
@@ -40,4 +46,10 @@ export interface ProgressEvent {
     validN: number;
     maxK: number;
 }
-export declare function runAudit(cfg: Config, onProgress?: (e: ProgressEvent) => void): Promise<AuditResult>;
+export interface RunOpts {
+    onProgress?: (e: ProgressEvent) => void;
+    /** cases probed in parallel (default 1) */
+    concurrency?: number;
+    budget?: Budget;
+}
+export declare function runAudit(cfg: Config, optsOrProgress?: RunOpts | ((e: ProgressEvent) => void)): Promise<AuditResult>;

package/dist/eval.js

@@ -1,13 +1,17 @@
-import { reliability, sequentialDecision } from "./stats.js";
-import { measure } from "./orchestrator.js";
+/** The audit: run every case co-loaded, compute reliability (with CI), detect trigger-theft,
+ * and keep infrastructure failure distinct from behavioral failure (finding #1). */
+import { probeSettings } from "./types.js";
+import { reliability, planDecision, sequentialPlan } from "./stats.js";
+import { measure, mapPool, infraUntrustworthy, loadProblems, extraSkills, } from "./orchestrator.js";
 import { getAdapter } from "./adapters/index.js";
-import { buildManifest } from "./manifest.js";
-function verdictFor(stats, decision, theft) {
+import { buildManifest, buildEnvironment, capabilityWarnings } from "./manifest.js";
+import { listSkills } from "./skills.js";
+function verdictFor(stats, decision, theft, failOnTheft) {
     // `error` means INFRASTRUCTURE failures dominate — untrustworthy. It is NOT about small k:
     // a small-but-all-valid run is `inconclusive` (wide CI), which the report explains.
-    if (stats.n === 0 || (stats.errors > 0 && stats.errors >= stats.n))
+    if (infraUntrustworthy(stats))
         return "error";
-    if (theft.length > 0)
+    if (failOnTheft && theft.length > 0)
         return "fail";
     if (decision === "above")
         return "pass";
@@ -15,34 +19,48 @@ function verdictFor(stats, decision, theft) {
         return "fail";
     return "inconclusive"; // includes "k too small to decide" — report says how many runs are needed
 }
-export async function runAudit(cfg, onProgress) {
+/** The library at cfg.cwd, or null when there's no readable skills dir (load checks are skipped). */
+function tryListSkills(cwd) {
+    try {
+        return listSkills(cwd);
+    }
+    catch {
+        return null;
+    }
+}
+export async function runAudit(cfg, optsOrProgress = {}) {
+    const opts = typeof optsOrProgress === "function" ? { onProgress: optsOrProgress } : optsOrProgress;
     const adapter = getAdapter(cfg.runtime);
-    const cases = [];
-    let totalCost = 0;
-    for (let ci = 0; ci < cfg.cases.length; ci++) {
-        const c = cfg.cases[ci];
+    const library = tryListSkills(cfg.cwd);
+    const plan = sequentialPlan(cfg.k, cfg.threshold, cfg.conf);
+    const settings = probeSettings(cfg);
+    const onProgress = opts.onProgress;
+    const cases = await mapPool(cfg.cases, opts.concurrency ?? 1, async (c, ci) => {
         const stats = await measure(adapter, c.prompt, c.expected, {
-            cwd: cfg.cwd, maxK: cfg.k, threshold: cfg.threshold, conf: cfg.conf,
-            ...(cfg.model ? { model: cfg.model } : {}),
+            cwd: cfg.cwd, maxK: cfg.k, threshold: cfg.threshold, conf: cfg.conf, ...settings,
+            ...(opts.budget ? { budget: opts.budget } : {}),
             ...(onProgress ? {
                 onProbe: (validN, maxK) => onProgress({
                     caseIndex: ci + 1, caseTotal: cfg.cases.length, prompt: c.prompt, validN, maxK,
                 }),
             } : {}),
         });
-        totalCost += stats.totalCost;
         const rel = reliability(stats.hits, stats.n, cfg.conf);
         const want = c.expected ?? "None";
         const theft = Object.keys(stats.dist).filter((s) => s !== want && s !== "None");
-        const decision = sequentialDecision(stats.hits, stats.n, cfg.threshold, cfg.conf);
-        cases.push({
+        const decision = planDecision(plan, stats.hits, stats.n);
+        const notLoaded = library !== null && c.expected !== null &&
+            loadProblems(stats.env, [c.expected], library.filter((s) => s.name === c.expected)).missing.length > 0;
+        return {
             prompt: c.prompt, expected: c.expected, stats, reliability: rel,
-            theft, decision, verdict: verdictFor(stats, decision, theft),
-        });
-    }
-    // NOTE: per-case verdicts here are CI decisions vs the threshold, not p-values — so there is no
-    // family to BH-correct in this path. Where families of Fisher tests exist (context, ablation,
-    // baseline comparison) the correction is applied.
+            theft, decision, verdict: verdictFor(stats, decision, theft, Boolean(cfg.failOnTheft)),
+            ...(notLoaded ? { notLoaded } : {}),
+        };
+    });
+    // NOTE: per-case verdicts here are sequential tests vs the threshold, each with its own exact
+    // error bound — not a family of p-values to BH-correct. Where families of Fisher tests exist
+    // (context, ablation, baseline comparison) the correction is applied.
+    const totalCost = cases.reduce((s, c) => s + c.stats.totalCost, 0);
     const counts = {
         pass: cases.filter((c) => c.verdict === "pass").length,
         fail: cases.filter((c) => c.verdict === "fail").length,
@@ -50,9 +68,24 @@ export async function runAudit(cfg, onProgress) {
         error: cases.filter((c) => c.verdict === "error").length,
     };
     const exitCode = counts.fail > 0 ? 1 : (counts.error > 0 || counts.inconclusive > 0) ? 2 : 0;
+    const env = cases.find((c) => c.stats.env)?.stats.env;
+    const warnings = capabilityWarnings(cfg, adapter);
+    const unloaded = [...new Set(cases.filter((c) => c.notLoaded).map((c) => c.expected))];
+    if (unloaded.length) {
+        warnings.push(`the runtime did not load: ${unloaded.join(", ")} — these can never fire ` +
+            `(check each SKILL.md's frontmatter; run \`skill-probe doctor\`)`);
+    }
+    if (cfg.userConfig) {
+        warnings.push(`probes loaded your personal runtime config (--user-config): results reflect this ` +
+            `machine and won't reproduce in CI`);
+    }
+    if (cases.some((c) => c.stats.budgetExhausted)) {
+        warnings.push(`--max-cost budget reached — cases cut short are inconclusive (error if they never started), never a pass`);
+    }
     return {
         runtime: cfg.runtime, model: cfg.model ?? "(runtime default)",
         threshold: cfg.threshold, cases, totalCost, counts, exitCode,
-        manifest: buildManifest("audit", cfg),
+        manifest: buildManifest("audit", cfg, buildEnvironment(cfg, env, library ? extraSkills(env, library) : undefined, adapter)),
+        warnings,
     };
 }

package/dist/fix.d.ts

@@ -1,9 +1,6 @@
-import type { Config } from "./types.ts";
+import { type Config } from "./types.ts";
 import { reliability } from "./stats.ts";
-export declare function readDesc(file: string): string;
-/** Replace only the single-line description; backups are handled explicitly in runFix (not here,
- * because the interleaved loop calls this many times and must not snapshot intermediate states). */
-export declare function writeDesc(file: string, desc: string): void;
+import { type Budget } from "./orchestrator.ts";
 export interface Sibling {
     name: string;
     desc: string;
@@ -22,14 +19,20 @@ export interface FixResult {
     errors: number;
     applied: boolean;
     applyBar: number;
-    /** path of the kept backup (only when applied); null when reverted (no change made) */
+    /** path of the kept backup (only when applied); null when nothing was written */
     backupPath: string | null;
+    /** why the rewrite wasn't tested or applied, beyond the statistics */
+    note?: string;
+    totalCost: number;
 }
 export interface FixOpts {
     skill: string;
     k: number;
     applyBar: number;
     model?: string;
+    /** paired runs in flight at once (each pair is 2 probes; default 1) */
+    concurrency?: number;
+    budget?: Budget;
 }
 export interface FixDeps {
     rewrite?: (skill: string, cur: string, prompts: string[], siblings: Sibling[], model?: string) => Promise<string>;

package/dist/fix.js

@@ -1,31 +1,21 @@
 /** `skill-probe fix` — rewrite a failing skill's description, then PROVE the lift is real with
  * the Bayesian Beta-Binomial (not "it looks better"). Upgrades on the Python prototype:
- *  - INTERLEAVED before/after (old desc -> probe -> new desc -> probe, paired) controls for
- *    session drift at the same probe cost as block-wise.
+ *  - PAIRED before/after: two identical sandboxes of the library, one with the old description and
+ *    one with the new; each pair of probes runs at the same moment, so session/model drift hits
+ *    both arms equally.
  *  - COLLISION-AWARE rewrite: the LLM is told sibling descriptions so it won't steal triggers.
- *  - Applies the new description only if P(improvement) >= bar AND the effect is positive;
- *    otherwise reverts. Original is always backed up to SKILL.md.bak. */
-import { readFileSync, writeFileSync, existsSync, readdirSync, copyFileSync, unlinkSync } from "node:fs";
-import { join } from "node:path";
+ *  - The real SKILL.md is never touched during the experiment (an interrupted run can't leave it
+ *    half-edited). The new description is written once, at the end, only if P(improvement) >= bar
+ *    AND the effect is positive AND the file hasn't changed on disk since the run started — with
+ *    the exact original snapshotted to a timestamped backup first. */
+import { readFileSync, writeFileSync, existsSync, copyFileSync, rmSync } from "node:fs";
+import { probeSettings } from "./types.js";
 import { getAdapter } from "./adapters/index.js";
 import { probImprovement, reliability } from "./stats.js";
 import { complete } from "./llm.js";
-const DESC_RE = /^description:\s*(.*)$/m;
-function skillFile(cfg, name) {
-    return join(cfg.cwd, ".claude", "skills", name, "SKILL.md");
-}
-export function readDesc(file) {
-    const m = DESC_RE.exec(readFileSync(file, "utf8"));
-    if (!m)
-        throw new Error(`no single-line 'description:' found in ${file} (multi-line block scalars not yet supported)`);
-    return m[1].trim();
-}
-/** Replace only the single-line description; backups are handled explicitly in runFix (not here,
- * because the interleaved loop calls this many times and must not snapshot intermediate states). */
-export function writeDesc(file, desc) {
-    const txt = readFileSync(file, "utf8");
-    writeFileSync(file, txt.replace(DESC_RE, `description: ${desc}`));
-}
+import { mapPool } from "./orchestrator.js";
+import { listSkills, parseFrontmatter, replaceDescription, normalizeDescription, skillFile } from "./skills.js";
+import { buildSandbox } from "./sandbox.js";
 /** Unique per-run snapshot path, never clobbering a prior run's backup. */
 function backupPathFor(file) {
     const base = `${file}.bak.${Date.now()}`;
@@ -49,63 +39,85 @@ export async function defaultRewrite(skill, cur, prompts, siblings, model) {
 }
 export async function runFix(cfg, opts, deps = {}) {
     const adapter = getAdapter(cfg.runtime);
-    const file = skillFile(cfg, opts.skill);
+    const file = skillFile(cfg.cwd, opts.skill);
     if (!existsSync(file))
         throw new Error(`skill not found: ${file}`);
-    const root = join(cfg.cwd, ".claude", "skills");
-    const siblings = readdirSync(root)
-        .filter((n) => n !== opts.skill && existsSync(join(root, n, "SKILL.md")))
-        .map((n) => ({ name: n, desc: readDesc(join(root, n, "SKILL.md")) }));
+    const library = listSkills(cfg.cwd);
+    const siblings = library
+        .filter((s) => s.name !== opts.skill)
+        .map((s) => ({ name: s.name, desc: s.description }));
     const prompts = cfg.cases.filter((c) => c.expected === opts.skill).map((c) => c.prompt);
     if (prompts.length === 0)
         throw new Error(`no cases in the config expect skill '${opts.skill}'`);
-    const oldDesc = readDesc(file);
+    const original = readFileSync(file, "utf8");
+    const fm = parseFrontmatter(original);
+    if (!fm.ok)
+        throw new Error(`${file} has no \`---\` frontmatter block — can't rewrite its description`);
+    const oldDesc = normalizeDescription(fm.fields["description"] ?? "");
+    replaceDescription(original, oldDesc || "x"); // throws now (before paying for a rewrite) if it can't be rewritten
     const rewrite = deps.rewrite ?? defaultRewrite;
-    const newDesc = await rewrite(opts.skill, oldDesc, prompts, siblings, opts.model);
-    // Snapshot the EXACT pre-fix file once, to a unique path (never clobber a prior run's backup).
-    const backupPath = backupPathFor(file);
-    copyFileSync(file, backupPath);
-    const probeWith = async (desc, prompt) => {
-        writeDesc(file, desc);
-        return adapter.probe(prompt, { cwd: cfg.cwd, ...(cfg.model ? { model: cfg.model } : {}) });
+    const newDesc = normalizeDescription(await rewrite(opts.skill, oldDesc, prompts, siblings, opts.model));
+    const base = {
+        skill: opts.skill, oldDesc, newDesc, applyBar: opts.applyBar,
+        before: reliability(0, 0), after: reliability(0, 0),
+        pImprove: 0, deltaMean: 0, deltaCi: [0, 0],
+        n: 0, errors: 0, applied: false, backupPath: null, totalCost: 0,
     };
-    let beforeHits = 0, afterHits = 0, n = 0, errors = 0;
+    if (!newDesc)
+        return { ...base, note: "the rewrite came back empty — nothing to test" };
+    if (newDesc === oldDesc)
+        return { ...base, note: "the rewrite is identical to the current description — nothing to test" };
+    const names = library.map((s) => s.name);
+    const beforeBox = buildSandbox(cfg.cwd, names); // holds the exact original SKILL.md
+    const afterBox = buildSandbox(cfg.cwd, names);
+    let beforeHits = 0, afterHits = 0, n = 0, errors = 0, totalCost = 0;
     try {
-        for (let i = 0; i < opts.k; i++) {
-            for (const p of prompts) {
-                const rb = await probeWith(oldDesc, p);
-                const ra = await probeWith(newDesc, p);
-                if (rb.status === "error" || ra.status === "error") {
-                    errors++;
-                    continue;
-                } // drop incomplete pair
-                if (rb.skillFired === opts.skill)
-                    beforeHits++;
-                if (ra.skillFired === opts.skill)
-                    afterHits++;
-                n++;
+        const afterFile = skillFile(afterBox.cwd, opts.skill);
+        rmSync(afterFile, { force: true }); // a fresh file — never write through a link into the real library
+        writeFileSync(afterFile, replaceDescription(original, newDesc));
+        const settings = probeSettings(cfg);
+        const jobs = Array.from({ length: opts.k }, () => prompts).flat();
+        await mapPool(jobs, opts.concurrency ?? 1, async (p) => {
+            if (opts.budget?.exhausted) {
+                errors++;
+                return;
             }
-        }
-        const imp = probImprovement(beforeHits, n, afterHits, n);
-        const applied = n > 0 && imp.pImprove >= opts.applyBar && imp.deltaMean > 0;
-        if (applied) {
-            writeDesc(file, newDesc); // commit; keep the timestamped backup
-        }
-        else {
-            copyFileSync(backupPath, file); // restore exact original
-            unlinkSync(backupPath); // no change made -> don't leave a redundant backup
-        }
-        return {
-            skill: opts.skill, oldDesc, newDesc,
-            before: reliability(beforeHits, n), after: reliability(afterHits, n),
-            pImprove: imp.pImprove, deltaMean: imp.deltaMean, deltaCi: imp.deltaCi,
-            n, errors, applied, applyBar: opts.applyBar,
-            backupPath: applied ? backupPath : null,
-        };
+            const [rb, ra] = await Promise.all([
+                adapter.probe(p, { cwd: beforeBox.cwd, ...settings }),
+                adapter.probe(p, { cwd: afterBox.cwd, ...settings }),
+            ]);
+            totalCost += rb.cost + ra.cost;
+            opts.budget?.add(rb.cost + ra.cost);
+            if (rb.status === "error" || ra.status === "error") {
+                errors++;
+                return;
+            } // drop incomplete pair
+            if (rb.skillFired === opts.skill)
+                beforeHits++;
+            if (ra.skillFired === opts.skill)
+                afterHits++;
+            n++;
+        });
     }
-    catch (e) {
-        copyFileSync(backupPath, file); // restore exact pre-fix file on any error
-        unlinkSync(backupPath);
-        throw e;
+    finally {
+        beforeBox.cleanup();
+        afterBox.cleanup();
     }
+    const imp = probImprovement(beforeHits, n, afterHits, n);
+    const result = {
+        ...base,
+        before: reliability(beforeHits, n), after: reliability(afterHits, n),
+        pImprove: imp.pImprove, deltaMean: imp.deltaMean, deltaCi: imp.deltaCi,
+        n, errors, totalCost,
+    };
+    if (!(n > 0 && imp.pImprove >= opts.applyBar && imp.deltaMean > 0))
+        return result;
+    // commit: only if the file is exactly what we measured against (someone may have edited it)
+    if (readFileSync(file, "utf8") !== original) {
+        return { ...result, note: `${file} changed on disk during the run — not overwriting it; re-run fix` };
+    }
+    const backupPath = backupPathFor(file);
+    copyFileSync(file, backupPath);
+    writeFileSync(file, replaceDescription(original, newDesc));
+    return { ...result, applied: true, backupPath };
 }

package/dist/gen.d.ts

@@ -1,9 +1,10 @@
+/** `skill-probe gen` — draft a test config from the skills in a project. The skill descriptions
+ * already say what each skill is for, so an LLM can propose realistic should-fire prompts,
+ * cross-skill near-misses (to surface mis-routing), and decoys. ALWAYS human-reviewed before use
+ * (the generator tends toward obvious keyword-matches; you add the messy real-world phrasings). */
 import type { Config, EvalCase } from "./types.ts";
-export interface SkillInfo {
-    name: string;
-    description: string;
-}
-export declare function listSkills(cwd: string): SkillInfo[];
+import type { SkillInfo } from "./skills.ts";
+export { listSkills, type SkillInfo } from "./skills.ts";
 export interface GenOpts {
     perSkill: number;
     decoys: number;

package/dist/gen.js

@@ -1,25 +1,5 @@
-/** `skill-probe gen` — draft a test config from the skills in a project. The skill descriptions
- * already say what each skill is for, so an LLM can propose realistic should-fire prompts,
- * cross-skill near-misses (to surface mis-routing), and decoys. ALWAYS human-reviewed before use
- * (the generator tends toward obvious keyword-matches; you add the messy real-world phrasings). */
-import { readFileSync, existsSync, readdirSync } from "node:fs";
-import { join } from "node:path";
 import { complete } from "./llm.js";
-const DESC_RE = /^description:\s*(.*)$/m;
-export function listSkills(cwd) {
-    const root = join(cwd, ".claude", "skills");
-    if (!existsSync(root))
-        throw new Error(`no .claude/skills/ directory under ${cwd}`);
-    const skills = [];
-    for (const name of readdirSync(root)) {
-        const file = join(root, name, "SKILL.md");
-        if (!existsSync(file))
-            continue;
-        const m = DESC_RE.exec(readFileSync(file, "utf8"));
-        skills.push({ name, description: m ? m[1].trim() : "" });
-    }
-    return skills;
-}
+export { listSkills } from "./skills.js";
 export async function defaultGenerate(skills, opts) {
     const list = skills.map((s) => `- ${s.name}: ${s.description}`).join("\n");
     const sys = "You build a TEST SUITE that audits whether an AI coding agent loads the correct "

package/dist/llm.d.ts

@@ -13,3 +13,5 @@ export declare function interpretClaudeJson(code: number | null, out: string, er
     text?: string;
     error?: string;
 };
+/** CLI arguments for one completion. Exported for testing. */
+export declare function claudeCliArgs(system: string, user: string, opts: CompleteOpts): string[];

package/dist/llm.js

@@ -3,8 +3,8 @@
  *   - otherwise              → fall back to the `claude` CLI (works on a Claude subscription,
  *                              no API key needed).
  * So the whole tool — audit, gen, fix — runs on just a logged-in `claude`. */
-import { spawn } from "node:child_process";
 import { tmpdir } from "node:os";
+import { runProcess } from "./proc.js";
 const ANTHROPIC_URL = "https://api.anthropic.com/v1/messages";
 /** Which backend complete() will use, given the environment. Exported for testing/diagnostics. */
 export function llmBackend() {
@@ -47,26 +47,26 @@ export function interpretClaudeJson(code, out, err, timedOut) {
     text = text.trim();
     return text ? { text } : { error: `claude CLI returned no text (${(err || out).slice(0, 200)})` };
 }
-function completeViaClaudeCli(system, user, opts) {
-    return new Promise((resolve, reject) => {
-        const args = ["-p", user, "--append-system-prompt", system, "--output-format", "json"];
-        if (opts.model)
-            args.push("--model", opts.model);
-        // run in a neutral cwd so the project's own skills/AGENTS.md don't bias the generation
-        const child = spawn("claude", args, { cwd: tmpdir(), stdio: ["ignore", "pipe", "pipe"] });
-        let out = "", err = "", settled = false, timedOut = false;
-        const timer = setTimeout(() => { timedOut = true; child.kill("SIGKILL"); }, 120_000);
-        const done = (fn) => { if (settled)
-            return; settled = true; clearTimeout(timer); fn(); };
-        child.stdout.on("data", (d) => { out += d.toString(); });
-        child.stderr.on("data", (d) => { err += d.toString(); });
-        child.on("error", (e) => done(() => reject(new Error(`claude CLI failed: ${e.message} — set ANTHROPIC_API_KEY or install/login 'claude'.`))));
-        child.on("close", (code) => done(() => {
-            const r = interpretClaudeJson(code, out, err, timedOut);
-            if (r.text)
-                resolve(r.text);
-            else
-                reject(new Error(r.error ?? "claude CLI failed"));
-        }));
-    });
+/** CLI arguments for one completion. Exported for testing. */
+export function claudeCliArgs(system, user, opts) {
+    // hermetic + no session transcript: diagnose calls this k times per case, and neither the user's
+    // plugins/MCP servers nor ~/.claude/projects clutter have any business in a plain completion
+    const args = [
+        "-p", user, "--append-system-prompt", system, "--output-format", "json",
+        "--no-session-persistence", "--setting-sources", "project", "--strict-mcp-config",
+    ];
+    if (opts.model)
+        args.push("--model", opts.model);
+    return args;
+}
+async function completeViaClaudeCli(system, user, opts) {
+    // run in a neutral cwd so the project's own skills/AGENTS.md don't bias the generation
+    const r = await runProcess("claude", claudeCliArgs(system, user, opts), { cwd: tmpdir(), timeoutMs: 120_000 });
+    if (r.spawnError) {
+        throw new Error(`claude CLI failed: ${r.spawnError} — set ANTHROPIC_API_KEY or install/login 'claude'.`);
+    }
+    const out = interpretClaudeJson(r.code, r.stdout, r.stderr, r.timedOut);
+    if (out.text)
+        return out.text;
+    throw new Error(out.error ?? "claude CLI failed");
 }

package/dist/manifest.d.ts

@@ -1,4 +1,18 @@
-import type { Config } from "./types.ts";
+import type { Config, ProbeEnv, RuntimeAdapter } from "./types.ts";
+/** What the runtime actually ran with — reported by the runtime itself, not assumed. */
+export interface RunEnvironment {
+    /** runtime CLI version (built-in skills and routing change between versions) */
+    runtimeVersion?: string;
+    /** the model the runtime actually used — resolves "(runtime default)" */
+    resolvedModel?: string;
+    /** skills the runtime loaded that are not in the library (e.g. Claude Code's built-ins) */
+    extraSkills?: string[];
+    /** the user's personal runtime config was requested in probes (--user-config) */
+    userConfig: boolean;
+    /** personal runtime config was actually kept out of probes (false when the runtime can't) */
+    hermetic?: boolean;
+    maxTurns: number;
+}
 export interface RunManifest {
     tool: "skill-probe";
     version: string;
@@ -13,9 +27,14 @@ export interface RunManifest {
     /** sha256 (first 16 hex chars) of the resolved config — cases + settings, NOT cwd */
     configHash: string;
     node: string;
+    environment?: RunEnvironment;
 }
 export declare function toolVersion(): string;
 /** Hash the parts of the config that change what was measured. `cwd` is deliberately EXCLUDED:
- * the same library checked out at a different path (CI runner vs laptop) is the same experiment. */
+ * the same library checked out at a different path (CI runner vs laptop) is the same experiment.
+ * Settings added later are hashed only when non-default, so existing baselines keep their hash. */
 export declare function configHash(cfg: Config): string;
-export declare function buildManifest(command: RunManifest["command"], cfg: Config): RunManifest;
+export declare function buildEnvironment(cfg: Config, env: ProbeEnv | undefined, extraSkills: string[] | undefined, adapter: RuntimeAdapter): RunEnvironment;
+/** Caveats when the config asks for something this runtime can't do. */
+export declare function capabilityWarnings(cfg: Config, adapter: RuntimeAdapter): string[];
+export declare function buildManifest(command: RunManifest["command"], cfg: Config, environment?: RunEnvironment): RunManifest;

package/dist/manifest.js

@@ -19,7 +19,8 @@ export function toolVersion() {
     return cachedVersion;
 }
 /** Hash the parts of the config that change what was measured. `cwd` is deliberately EXCLUDED:
- * the same library checked out at a different path (CI runner vs laptop) is the same experiment. */
+ * the same library checked out at a different path (CI runner vs laptop) is the same experiment.
+ * Settings added later are hashed only when non-default, so existing baselines keep their hash. */
 export function configHash(cfg) {
     const canonical = JSON.stringify({
         runtime: cfg.runtime,
@@ -28,10 +29,33 @@ export function configHash(cfg) {
         threshold: cfg.threshold,
         conf: cfg.conf,
         cases: cfg.cases.map((c) => ({ prompt: c.prompt, expected: c.expected })),
+        ...((cfg.maxTurns ?? 1) !== 1 ? { maxTurns: cfg.maxTurns } : {}),
+        ...(cfg.userConfig ? { userConfig: true } : {}),
     });
     return createHash("sha256").update(canonical).digest("hex").slice(0, 16);
 }
-export function buildManifest(command, cfg) {
+export function buildEnvironment(cfg, env, extraSkills, adapter) {
+    return {
+        ...(env?.runtimeVersion ? { runtimeVersion: env.runtimeVersion } : {}),
+        ...(env?.model ? { resolvedModel: env.model } : {}),
+        ...(extraSkills ? { extraSkills } : {}),
+        userConfig: Boolean(cfg.userConfig),
+        hermetic: Boolean(adapter.isolatesUserConfig) && !cfg.userConfig,
+        maxTurns: cfg.maxTurns ?? 1,
+    };
+}
+/** Caveats when the config asks for something this runtime can't do. */
+export function capabilityWarnings(cfg, adapter) {
+    const w = [];
+    if (cfg.maxTurns !== undefined && cfg.maxTurns !== 1 && !adapter.supportsMaxTurns) {
+        w.push(`${adapter.name} has no turn limit — maxTurns (${cfg.maxTurns}) is ignored`);
+    }
+    if (!adapter.isolatesUserConfig && !cfg.userConfig) {
+        w.push(`${adapter.name} can't exclude your personal/global config from probes — results may differ between machines`);
+    }
+    return w;
+}
+export function buildManifest(command, cfg, environment) {
     return {
         tool: "skill-probe",
         version: toolVersion(),
@@ -44,5 +68,6 @@ export function buildManifest(command, cfg) {
         conf: cfg.conf,
         configHash: configHash(cfg),
         node: process.version,
+        ...(environment ? { environment } : {}),
     };
 }