skill-probe 0.9.0 → 0.10.0

package/dist/cli.js

@@ -2,7 +2,8 @@
 /** skill-probe CLI.
  *   skill-probe --config <f>                  audit a co-loaded skill library
  *   skill-probe fix --config <f> --skill <n>  rewrite a skill's description & prove the lift
- * Exit: 0 = pass/applied, 1 = behavioral fail/reverted, 2 = inconclusive/infra error/usage. */
+ * Exit: 0 = pass/applied, 1 = behavioral fail/not applied, 2 = inconclusive/infra error/usage,
+ *       130/143 = interrupted (children killed, sandboxes removed). */
 import { parseArgs } from "node:util";
 import { resolve } from "node:path";
 import { runAudit } from "./eval.js";
@@ -14,10 +15,14 @@ import { listSkills, generateCases, genConfig } from "./gen.js";
 import { loadConfig, parseProbability, parseIntFlag, ConfigError } from "./config.js";
 import { loadBaseline, saveBaseline, compareToBaseline } from "./baseline.js";
 import { renderTable, renderMarkdown, renderJson, renderFix, renderContext, renderDoctor, renderDiagnose, renderBaseline } from "./report.js";
+import { Budget } from "./orchestrator.js";
+import { toolVersion } from "./manifest.js";
+import { installSignalHandlers } from "./proc.js";
+const COMMANDS = new Set(["audit", "context", "diagnose", "doctor", "gen", "fix"]);
 const HELP = `skill-probe — audit a co-loaded agent skill library by real activation behavior.
 
 USAGE
-  skill-probe --config <file.json> [options]                 # audit
+  skill-probe [audit] --config <file.json> [options]         # audit
   skill-probe context --config <file.json> [--ablate]        # isolation vs co-loaded activation rates
   skill-probe diagnose --config <file.json> [options]        # why a skill fails: routing-miss vs description
   skill-probe doctor [--cwd <dir>] [--config <file.json>]    # preflight: setup/auth/config sanity
@@ -32,12 +37,24 @@ OPTIONS
                            Recorded in the report — REQUIRED for a fair cross-runtime comparison.
       --k <n>              max VALID repeats per prompt (positive integer; default 10)
       --threshold <p>      min acceptable reliability, strictly 0..1 (default 0.7 = smoke test)
-      --conf <p>           confidence level: 0.9 | 0.95 | 0.99 (default 0.95)
+      --conf <p>           confidence: 0.9 | 0.95 | 0.99 (default 0.95). A verdict is wrong with
+                           probability <= 1-conf, however early the run stops.
+      --max-turns <n>      agent turns per probe before reading the outcome (default 1)
+      --timeout <sec>      per-probe timeout in seconds (default 120)
+      --user-config        also load your personal runtime config (user settings, plugins, MCP
+                           servers) into probes. Off by default (claude-code): results then
+                           reproduce in CI. opencode always loads its global config.
+      --concurrency <n>    cases probed in parallel (default 1; mind your rate limits)
+      --max-cost <usd>     stop probing once runtime probes have spent this much; cases cut short
+                           are inconclusive (error if never started), never a pass. LLM calls
+                           (gen, fix's rewrite, diagnose's forced choice) aren't counted
       --json               emit JSON instead of a table
       --markdown           emit a Markdown table (paste into a PR/README)
       --no-cost            hide the cost line (e.g. when on a Claude subscription)
       --quiet              suppress the live progress line on stderr
-  audit only (CI gating):
+  -v, --version            print the version
+  audit only:
+      --fail-on-theft      fail a case when ANY other skill fires, even if reliability clears the bar
       --save-baseline <f>  after the run, save per-case counts + a run manifest to <f>
       --baseline <f>       compare this run against a saved baseline: per-case Fisher's exact,
                            Benjamini-Hochberg corrected — exit 1 ONLY on a significant regression
@@ -50,7 +67,7 @@ OPTIONS
   doctor only:
       --cwd <dir>          project dir containing .claude/skills/ (default: .)
       --config <file>      also sanity-check a config (expected skills exist, k/threshold feasible)
-      --skip-probe         don't run the harmless live probe (skip the auth check; no cost)
+      --skip-probe         don't run the harmless live probes (skip the auth/load checks; no cost)
   gen only (uses ANTHROPIC_API_KEY if set, else the logged-in 'claude' CLI):
       --cwd <dir>          project dir containing .claude/skills/ (default: .)
       --per-skill <n>      should-fire prompts to draft per skill (default 3)
@@ -60,8 +77,8 @@ OPTIONS
       --apply-threshold <p> min P(improvement) to keep the rewrite (default 0.9)
   -h, --help               show this help
 
-Reliability carries a Wilson CI + sample size k. fix rewrites the description, runs an INTERLEAVED
-before/after, and applies it only if the Bayesian P(improvement) clears the apply-threshold.`;
+Reliability carries a Wilson CI + sample size k. fix runs a PAIRED before/after in two sandboxes and
+writes the new description only if the Bayesian P(improvement) clears the apply-threshold.`;
 let progressLen = 0;
 function makeProgress(values) {
     if (values["json"] || values["quiet"] || !process.stderr.isTTY)
@@ -103,13 +120,32 @@ function makeDiagProgress(values) {
         progressLen = line.length;
     };
 }
-async function audit(values) {
-    const cfg = loadConfig(values["config"], {
-        runtime: values["runtime"], model: values["model"],
-        k: values["k"], threshold: values["threshold"],
-        conf: values["conf"],
+/** The config plus every CLI override that changes what is measured. */
+function configFrom(values, opts = {}) {
+    const str = (k) => values[k];
+    return loadConfig(values["config"], {
+        runtime: str("runtime"), model: str("model"), k: str("k"), conf: str("conf"),
+        ...(opts.threshold === false ? {} : { threshold: str("threshold") }),
+        maxTurns: str("max-turns"), timeout: str("timeout"),
+        userConfig: Boolean(values["user-config"]), failOnTheft: Boolean(values["fail-on-theft"]),
     });
-    const result = await runAudit(cfg, makeProgress(values));
+}
+/** Run-time knobs that DON'T change what is measured (parallelism, spend cap). */
+function runOpts(values) {
+    const concurrency = parseIntFlag(values["concurrency"], "--concurrency", 1, 1);
+    const raw = values["max-cost"];
+    if (raw === undefined)
+        return { concurrency };
+    const max = Number(raw);
+    if (!Number.isFinite(max) || max <= 0) {
+        throw new ConfigError([`--max-cost must be a positive number of USD (got ${JSON.stringify(raw)})`]);
+    }
+    return { concurrency, budget: new Budget(max) };
+}
+async function audit(values) {
+    const cfg = configFrom(values);
+    const progress = makeProgress(values);
+    const result = await runAudit(cfg, { ...runOpts(values), ...(progress ? { onProgress: progress } : {}) });
     clearProgress();
     const showCost = !values["no-cost"];
     let cmp;
@@ -130,14 +166,9 @@ async function audit(values) {
     return result.exitCode;
 }
 async function diagnose(values) {
-    const cfg = loadConfig(values["config"], {
-        runtime: values["runtime"], model: values["model"],
-        k: values["k"], threshold: values["threshold"],
-        conf: values["conf"],
-    });
-    const result = await runDiagnose(cfg, {
-        ...(makeDiagProgress(values) ? { onProgress: makeDiagProgress(values) } : {}),
-    });
+    const cfg = configFrom(values);
+    const onProgress = makeDiagProgress(values);
+    const result = await runDiagnose(cfg, { ...runOpts(values), ...(onProgress ? { onProgress } : {}) });
     clearProgress();
     const showCost = !values["no-cost"];
     console.log(values["json"] ? renderJson(result) : renderDiagnose(result, { showCost }));
@@ -150,20 +181,17 @@ async function doctor(values) {
         ...(values["runtime"] ? { runtime: values["runtime"] } : {}),
         ...(values["model"] ? { model: values["model"] } : {}),
         ...(values["skip-probe"] ? { skipProbe: true } : {}),
+        ...(values["user-config"] ? { userConfig: true } : {}),
     });
     console.log(values["json"] ? renderJson(result) : renderDoctor(result));
     return result.exitCode;
 }
 async function context(values) {
-    const cfg = loadConfig(values["config"], {
-        runtime: values["runtime"], model: values["model"],
-        k: values["k"], threshold: values["threshold"],
-        conf: values["conf"],
-    });
+    const cfg = configFrom(values);
     const suspects = parseIntFlag(values["suspects"], "--suspects", 3, 1);
     const onProgress = makeContextProgress(values);
     const result = await runContextAudit(cfg, {
-        ablate: Boolean(values["ablate"]), suspects,
+        ablate: Boolean(values["ablate"]), suspects, ...runOpts(values),
         ...(onProgress ? { onProgress } : {}),
     });
     clearProgress();
@@ -195,12 +223,11 @@ async function fix(values) {
         console.error("fix requires --skill <name>");
         return 2;
     }
-    const cfg = loadConfig(values["config"], {
-        runtime: values["runtime"], model: values["model"],
-        k: values["k"], conf: values["conf"],
-    });
+    const cfg = configFrom(values, { threshold: false });
     const applyBar = parseProbability(values["apply-threshold"], "--apply-threshold", 0.9);
-    const result = await runFix(cfg, { skill, k: cfg.k, applyBar, ...(cfg.model ? { model: cfg.model } : {}) });
+    const result = await runFix(cfg, {
+        skill, k: cfg.k, applyBar, ...runOpts(values), ...(cfg.model ? { model: cfg.model } : {}),
+    });
     console.log(values["json"] ? renderJson(result) : renderFix(result));
     return result.applied ? 0 : 1;
 }
@@ -224,6 +251,13 @@ async function main() {
             baseline: { type: "string" },
             "save-baseline": { type: "string" },
             "skip-probe": { type: "boolean", default: false },
+            "max-turns": { type: "string" },
+            timeout: { type: "string" },
+            "user-config": { type: "boolean", default: false },
+            "fail-on-theft": { type: "boolean", default: false },
+            concurrency: { type: "string" },
+            "max-cost": { type: "string" },
+            version: { type: "boolean", short: "v", default: false },
             json: { type: "boolean", default: false },
             markdown: { type: "boolean", default: false },
             "no-cost": { type: "boolean", default: false },
@@ -235,7 +269,16 @@ async function main() {
         console.log(HELP);
         process.exit(0);
     }
-    const cmd = positionals[0];
+    if (values.version) {
+        console.log(toolVersion());
+        process.exit(0);
+    }
+    const cmd = positionals[0] ?? "audit";
+    if (!COMMANDS.has(cmd) || positionals.length > 1) {
+        console.error(`unknown command: ${positionals.join(" ")} (expected one of: ${[...COMMANDS].join(", ")})`);
+        process.exit(2);
+    }
+    installSignalHandlers();
     let code;
     if (cmd === "gen") {
         code = await gen(values);
@@ -245,7 +288,7 @@ async function main() {
     }
     else {
         if (!values.config) {
-            console.log(HELP);
+            console.error(HELP);
             process.exit(2);
         }
         code = cmd === "fix" ? await fix(values)

package/dist/config.d.ts

@@ -18,5 +18,9 @@ export interface Overrides {
     k?: string | undefined;
     threshold?: string | undefined;
     conf?: string | undefined;
+    maxTurns?: string | undefined;
+    timeout?: string | undefined;
+    userConfig?: boolean | undefined;
+    failOnTheft?: boolean | undefined;
 }
 export declare function loadConfig(configPath: string, overrides?: Overrides): Config;

package/dist/config.js

@@ -3,6 +3,10 @@
 import { readFileSync } from "node:fs";
 import { resolve, dirname } from "node:path";
 const VALID_CONF = new Set([0.9, 0.95, 0.99]);
+const KNOWN_KEYS = new Set([
+    "runtime", "model", "cwd", "k", "threshold", "conf", "cases",
+    "maxTurns", "timeout", "userConfig", "failOnTheft",
+]);
 export class ConfigError extends Error {
     errors;
     constructor(errors) {
@@ -17,6 +21,13 @@ export class ConfigError extends Error {
 export function validateConfig(raw, configPath) {
     const r = (raw ?? {});
     const errs = [];
+    // a typo'd key ("treshold") would otherwise silently fall back to the default — fail loud.
+    // "$schema" and "_"-prefixed keys are allowed for tooling and comments.
+    for (const key of Object.keys(r)) {
+        if (!KNOWN_KEYS.has(key) && !key.startsWith("$") && !key.startsWith("_")) {
+            errs.push(`unknown config key "${key}" (known: ${[...KNOWN_KEYS].join(", ")})`);
+        }
+    }
     const runtime = typeof r["runtime"] === "string" && r["runtime"] ? r["runtime"] : "claude-code";
     let model;
     if (r["model"] !== undefined) {
@@ -41,6 +52,20 @@ export function validateConfig(raw, configPath) {
     if (typeof conf !== "number" || !VALID_CONF.has(conf)) {
         errs.push(`conf must be one of 0.9, 0.95, 0.99 (got ${JSON.stringify(r["conf"])})`);
     }
+    const maxTurns = r["maxTurns"];
+    if (maxTurns !== undefined && (typeof maxTurns !== "number" || !Number.isInteger(maxTurns) || maxTurns < 1)) {
+        errs.push(`maxTurns must be a positive integer (got ${JSON.stringify(maxTurns)})`);
+    }
+    // seconds in the file / on the CLI; ms internally
+    const timeout = r["timeout"];
+    if (timeout !== undefined && (typeof timeout !== "number" || !Number.isFinite(timeout) || timeout <= 0)) {
+        errs.push(`timeout must be a positive number of seconds (got ${JSON.stringify(timeout)})`);
+    }
+    for (const key of ["userConfig", "failOnTheft"]) {
+        if (r[key] !== undefined && typeof r[key] !== "boolean") {
+            errs.push(`${key} must be true or false (got ${JSON.stringify(r[key])})`);
+        }
+    }
     const cases = Array.isArray(r["cases"]) ? r["cases"] : [];
     if (cases.length === 0)
         errs.push("config needs a non-empty `cases` array");
@@ -63,6 +88,10 @@ export function validateConfig(raw, configPath) {
         threshold: threshold,
         conf: conf,
         cases: cases,
+        ...(maxTurns !== undefined ? { maxTurns: maxTurns } : {}),
+        ...(timeout !== undefined ? { timeoutMs: Math.round(timeout * 1000) } : {}),
+        ...(r["userConfig"] === true ? { userConfig: true } : {}),
+        ...(r["failOnTheft"] === true ? { failOnTheft: true } : {}),
     };
 }
 /** Validate a probability-ish flag (e.g. --apply-threshold): must be a number strictly in (0,1). */
@@ -104,5 +133,13 @@ export function loadConfig(configPath, overrides = {}) {
         merged["threshold"] = Number(overrides.threshold);
     if (overrides.conf !== undefined)
         merged["conf"] = Number(overrides.conf);
+    if (overrides.maxTurns !== undefined)
+        merged["maxTurns"] = Number(overrides.maxTurns);
+    if (overrides.timeout !== undefined)
+        merged["timeout"] = Number(overrides.timeout);
+    if (overrides.userConfig)
+        merged["userConfig"] = true;
+    if (overrides.failOnTheft)
+        merged["failOnTheft"] = true;
     return validateConfig(merged, configPath);
 }

package/dist/context.d.ts

@@ -1,13 +1,20 @@
-import type { Config } from "./types.ts";
+/** `skill-probe context` — activation-rate-BY-CONTEXT. The audit measures every skill co-loaded
+ * (the real, hard condition). This command additionally measures each skill in ISOLATION (only that
+ * skill present) and compares the two: it catches the "fires fine alone, fails under load" case that
+ * a single aggregate rate hides. The drop is tested with Fisher's exact test, so interference is
+ * reported as a real effect (p-value), not eyeballed.
+ *
+ * Every condition (isolation, co-loaded, each leave-one-out ablation) runs in a sandbox built the
+ * same way — see sandbox.ts — so the ONLY difference between conditions is which skills are
+ * present. When the runtime reports what it loaded, each condition is checked against what it was
+ * meant to contain; a mismatch makes the comparison untrustworthy instead of silently wrong. */
+import { type Config } from "./types.ts";
 import { reliability } from "./stats.ts";
-import { type RunStats } from "./orchestrator.ts";
+import { type Budget, type RunStats } from "./orchestrator.ts";
+import { buildSandbox } from "./sandbox.ts";
 import { type RunManifest } from "./manifest.ts";
-/** Build a throwaway project dir whose .claude/skills/ holds ONLY the named skills (copied from
- * srcCwd). Returns its path plus a cleanup(). The probe runs against this dir to load a subset. */
-export declare function buildContextDir(srcCwd: string, skillNames: string[]): {
-    cwd: string;
-    cleanup: () => void;
-};
+/** Kept for compatibility: a sandbox holding only the named skills (plus instruction files). */
+export declare const buildContextDir: typeof buildSandbox;
 type Rel = ReturnType<typeof reliability>;
 interface Condition {
     stats: RunStats;
@@ -25,6 +32,8 @@ export interface AblationResult {
     fisherP: number;
     fisherPAdj: number;
     untrustworthy: boolean;
+    /** why the run isn't a valid comparison, when the runtime loaded the wrong skill set */
+    loadIssue?: string;
     culprit: boolean;
 }
 export interface ContextCaseResult {
@@ -38,8 +47,14 @@ export interface ContextCaseResult {
      * test per case is a FAMILY, and raw p<0.05 over a 40-case library expects ~2 false flags by
      * chance alone. Interference is decided on THIS value. (1 for untrustworthy cases.) */
     fisherPAdj: number;
-    /** infra failures dominated one side — the comparison is not trustworthy (NOT a behavioral result) */
+    /** infra failures dominated one side, or a condition loaded the wrong skills — the comparison is
+     * not trustworthy (NOT a behavioral result) */
     untrustworthy: boolean;
+    /** why a condition isn't valid, when the runtime loaded the wrong skill set */
+    loadIssue?: string;
+    /** skills OUTSIDE the library (e.g. runtime built-ins) that fired in the co-loaded run — they
+     * can't be ablated, only avoided (narrow or rename the expected skill) */
+    externalThieves?: string[];
     /** fires reliably alone but drops significantly (BH-corrected) when the library is co-loaded */
     interference: boolean;
     /** leave-one-out runs (only with { ablate: true }, only for interference cases). Empty array =
@@ -59,6 +74,8 @@ export interface ContextAuditResult {
     exitCode: 0 | 1 | 2;
     /** who/what/when of this run — makes reports citable and comparable */
     manifest: RunManifest;
+    /** run-level caveats (built-in name clashes, budget cut-offs) */
+    warnings: string[];
 }
 export interface ContextProgress {
     caseIndex: number;
@@ -77,6 +94,9 @@ export interface ContextOpts {
     /** max suspect siblings to ablate per flagged case, ranked by probes stolen (default 3) */
     suspects?: number;
     onProgress?: (e: ContextProgress) => void;
+    /** cases (and ablation runs) probed in parallel (default 1) */
+    concurrency?: number;
+    budget?: Budget;
 }
 export declare function runContextAudit(cfg: Config, optsOrProgress?: ContextOpts | ((e: ContextProgress) => void)): Promise<ContextAuditResult>;
 export {};