skill-probe 0.8.0 → 0.10.0

package/dist/adapters/openCode.js

@@ -1,13 +1,27 @@
-/** OpenCode adapter — the second runtime (cross-vendor; BYO-key). Reads `.claude/skills/`.
- * Activation surfaces as a tool part named "skill" with the name in state.input.name (confirmed
- * against opencode v1.17.9 `run --format json`). Distinguishes behavioral outcome from infra
- * failure (finding #1). GOTCHA: opencode `run` hangs unless stdin is closed — we use stdio ignore. */
-import { spawn } from "node:child_process";
+import { runProcess } from "../proc.js";
+const MODEL_OUTPUT_TYPES = new Set([
+    "step_start", "step_finish", "step-start", "step-finish", "text", "tool", "tool_use", "reasoning",
+]);
+function errorMessage(o) {
+    const e = o["error"];
+    if (typeof e === "string")
+        return e;
+    if (e && typeof e === "object") {
+        const eo = e;
+        const data = eo["data"];
+        const msg = data?.["message"] ?? eo["message"] ?? eo["name"];
+        if (typeof msg === "string")
+            return msg;
+    }
+    return typeof o["message"] === "string" ? o["message"] : "opencode reported an error";
+}
 /** Recursively scan parsed OpenCode output for a fired skill + tool trajectory. Defensive:
  * handles "skill name in input.name" or "input.skill", single JSON or JSONL of events. */
 export function extractFromOpenCode(parsed) {
     let skillFired = null;
     let cost = 0;
+    let sawModelOutput = false;
+    let error;
     const trajectory = [];
     const visit = (node) => {
         if (Array.isArray(node)) {
@@ -17,7 +31,12 @@ export function extractFromOpenCode(parsed) {
         if (!node || typeof node !== "object")
             return;
         const o = node;
-        if (o["type"] === "tool" && typeof o["tool"] === "string") {
+        const type = o["type"];
+        if (type === "error" && error === undefined)
+            error = errorMessage(o);
+        if (typeof type === "string" && MODEL_OUTPUT_TYPES.has(type))
+            sawModelOutput = true;
+        if (type === "tool" && typeof o["tool"] === "string") {
             const tool = o["tool"];
             trajectory.push(tool);
             if (tool === "skill") {
@@ -34,13 +53,14 @@ export function extractFromOpenCode(parsed) {
             visit(v);
     };
     visit(parsed);
-    return { skillFired, trajectory, cost };
+    return { skillFired, trajectory, cost, sawModelOutput, ...(error !== undefined ? { error } : {}) };
 }
 /** Parse stdout (whole-JSON or JSONL) and report how many events were seen (0 = no output). */
 export function parseOutput(stdout) {
     const trimmed = stdout.trim();
-    if (!trimmed)
-        return { result: { skillFired: null, trajectory: [], cost: 0 }, eventCount: 0 };
+    if (!trimmed) {
+        return { result: { skillFired: null, trajectory: [], cost: 0, sawModelOutput: false }, eventCount: 0 };
+    }
     try {
         const whole = JSON.parse(trimmed);
         return { result: extractFromOpenCode(whole), eventCount: Array.isArray(whole) ? whole.length : 1 };
@@ -59,51 +79,48 @@ export function parseOutput(stdout) {
         return { result: extractFromOpenCode(events), eventCount: events.length };
     }
 }
+/** Decide what a finished (non-timeout) run means. Exported for testing. */
+export function interpretOpenCode(stdout, stderr) {
+    const { result, eventCount } = parseOutput(stdout);
+    const fail = (error) => ({ status: "error", skillFired: null, trajectory: result.trajectory, cost: result.cost, error });
+    if (eventCount === 0) {
+        // surface whatever diagnostic we have: stderr, else a non-JSON stdout log (opencode
+        // prints provider/model errors to stdout), else a generic hint.
+        const diag = stderr.trim() || stdout.trim();
+        return fail(diag ? diag.slice(-200) : "no output (auth/model/hang? check `opencode auth list`)");
+    }
+    if (result.error !== undefined)
+        return fail(`opencode error: ${result.error.slice(0, 200)}`);
+    if (!result.sawModelOutput) {
+        return fail("no model step in the output — the model did not actually run (check `opencode auth list`)");
+    }
+    return { status: "ok", skillFired: result.skillFired, trajectory: result.trajectory, cost: result.cost };
+}
 export const openCodeAdapter = {
     name: "opencode",
-    probe(prompt, { cwd, timeoutMs = 120_000, model }) {
+    // `opencode run` always loads its global config and has no turn limit — say so, don't pretend
+    isolatesUserConfig: false,
+    supportsMaxTurns: false,
+    async probe(prompt, { cwd, timeoutMs = 120_000, model }) {
         // config/CLI --model wins; env var is a fallback convenience
         const effectiveModel = model ?? process.env["SKILL_PROBE_OPENCODE_MODEL"];
         const args = ["run", prompt, "--format", "json"];
         if (effectiveModel)
             args.push("--model", effectiveModel);
-        return new Promise((resolve) => {
-            let settled = false, timedOut = false;
-            const child = spawn("opencode", args, { cwd, stdio: ["ignore", "pipe", "pipe"] });
-            let out = "", err = "";
-            const timer = setTimeout(() => { timedOut = true; child.kill("SIGKILL"); }, timeoutMs);
-            const finish = (r) => {
-                if (settled)
-                    return;
-                settled = true;
-                clearTimeout(timer);
-                resolve(r);
-            };
-            child.stdout.on("data", (d) => { out += d.toString(); });
-            child.stderr.on("data", (d) => { err += d.toString(); });
-            child.on("error", (e) => finish({
+        const r = await runProcess("opencode", args, { cwd, timeoutMs });
+        if (r.spawnError) {
+            return {
                 status: "error", skillFired: null, trajectory: [], cost: 0,
-                error: `spawn failed: ${e.message} (is 'opencode' installed and on PATH?)`,
-            }));
-            child.on("close", () => {
-                const { result, eventCount } = parseOutput(out);
-                if (timedOut) {
-                    return finish({
-                        status: "error", skillFired: null, trajectory: result.trajectory, cost: result.cost,
-                        error: `timed out after ${timeoutMs}ms`,
-                    });
-                }
-                if (eventCount === 0) {
-                    // surface whatever diagnostic we have: stderr, else a non-JSON stdout log (opencode
-                    // prints provider/model errors to stdout), else a generic hint.
-                    const diag = err.trim() || out.trim();
-                    return finish({
-                        status: "error", skillFired: null, trajectory: [], cost: 0,
-                        error: diag ? diag.slice(-200) : "no output (auth/model/hang? check `opencode auth list`)",
-                    });
-                }
-                finish({ status: "ok", skillFired: result.skillFired, trajectory: result.trajectory, cost: result.cost });
-            });
-        });
+                error: `spawn failed: ${r.spawnError} (is 'opencode' installed and on PATH?)`,
+            };
+        }
+        if (r.timedOut) {
+            const { result } = parseOutput(r.stdout);
+            return {
+                status: "error", skillFired: null, trajectory: result.trajectory, cost: result.cost,
+                error: `timed out after ${timeoutMs}ms`,
+            };
+        }
+        return interpretOpenCode(r.stdout, r.stderr);
     },
 };

package/dist/baseline.d.ts

@@ -0,0 +1,56 @@
+import type { AuditResult } from "./eval.ts";
+import type { RunManifest } from "./manifest.ts";
+export interface BaselineCase {
+    prompt: string;
+    expected: string | null;
+    hits: number;
+    n: number;
+}
+export interface BaselineFile {
+    manifest: RunManifest;
+    cases: BaselineCase[];
+}
+export declare function toBaseline(result: AuditResult): BaselineFile;
+export declare function saveBaseline(result: AuditResult, path: string): void;
+export declare function loadBaseline(path: string): BaselineFile;
+export interface RegressionCase {
+    prompt: string;
+    expected: string | null;
+    before: {
+        hits: number;
+        n: number;
+        pHat: number;
+    };
+    after: {
+        hits: number;
+        n: number;
+        pHat: number;
+    };
+    /** after.pHat - before.pHat (negative = dropped since baseline) */
+    delta: number;
+    fisherP: number;
+    /** BH-adjusted across all compared cases — regression is decided on THIS */
+    fisherPAdj: number;
+    /** one side has no valid probes — not comparable (never reads as "no regression") */
+    untrustworthy: boolean;
+    /** significant drop since baseline (BH-corrected) */
+    regressed: boolean;
+    /** significant rise since baseline (informational — consider re-saving the baseline) */
+    improved: boolean;
+}
+export interface BaselineComparison {
+    baselineDate: string;
+    baselineVersion: string;
+    /** apples-to-oranges warnings: config hash / runtime / model drift between runs */
+    mismatches: string[];
+    cases: RegressionCase[];
+    /** prompts in the current run with no baseline entry (measured, but not gated) */
+    newCases: string[];
+    /** prompts in the baseline that the current run no longer has */
+    missingCases: string[];
+    regressions: number;
+    improvements: number;
+    /** 1 = at least one significant regression (CI should fail) */
+    exitCode: 0 | 1;
+}
+export declare function compareToBaseline(result: AuditResult, base: BaselineFile): BaselineComparison;

package/dist/baseline.js

@@ -0,0 +1,121 @@
+/** Baseline / regression gating — the CI story. `--save-baseline` records a run's per-case
+ * counts; `--baseline` compares a new run against them with Fisher's exact test per case and
+ * Benjamini-Hochberg across cases, failing ONLY on a statistically significant drop. Raw-rate
+ * comparisons make CI flaky (activation is stochastic — rates wobble run to run); testing the
+ * drop makes the gate honest: noise passes, real regressions fail. */
+import { readFileSync, writeFileSync } from "node:fs";
+import { fisherExact2x2, bhAdjust } from "./stats.js";
+import { ConfigError } from "./config.js";
+export function toBaseline(result) {
+    return {
+        manifest: result.manifest,
+        cases: result.cases.map((c) => ({
+            prompt: c.prompt, expected: c.expected, hits: c.stats.hits, n: c.stats.n,
+        })),
+    };
+}
+export function saveBaseline(result, path) {
+    writeFileSync(path, JSON.stringify(toBaseline(result), null, 2) + "\n");
+}
+export function loadBaseline(path) {
+    let raw;
+    try {
+        raw = readFileSync(path, "utf8");
+    }
+    catch (e) {
+        throw new ConfigError([`cannot read baseline file ${path}: ${e instanceof Error ? e.message : e}`]);
+    }
+    let parsed;
+    try {
+        parsed = JSON.parse(raw);
+    }
+    catch {
+        throw new ConfigError([`baseline file ${path} is not valid JSON`]);
+    }
+    const b = parsed;
+    if (!b || typeof b !== "object" || !Array.isArray(b.cases) || !b.manifest ||
+        b.cases.some((c) => typeof c?.prompt !== "string" || typeof c?.hits !== "number" || typeof c?.n !== "number")) {
+        throw new ConfigError([
+            `baseline file ${path} doesn't look like a skill-probe baseline ` +
+                `(expected { manifest, cases: [{prompt, expected, hits, n}] } — create one with --save-baseline)`,
+        ]);
+    }
+    return b;
+}
+const caseKey = (prompt, expected) => `${prompt}\u0000${expected ?? "\u0001null"}`;
+export function compareToBaseline(result, base) {
+    const mismatches = [];
+    const bm = base.manifest, cm = result.manifest;
+    if (bm.configHash !== cm.configHash) {
+        mismatches.push(`config changed since baseline (${bm.configHash} → ${cm.configHash}) — ` +
+            `only cases matching by prompt+expected are gated`);
+    }
+    if (bm.runtime !== cm.runtime)
+        mismatches.push(`runtime changed: ${bm.runtime} → ${cm.runtime}`);
+    if (bm.model !== cm.model)
+        mismatches.push(`model changed: ${bm.model} → ${cm.model}`);
+    // what the runtime ACTUALLY ran with (older baselines don't record it — skip silently)
+    const be = bm.environment, ce = cm.environment;
+    if (be?.resolvedModel && ce?.resolvedModel && be.resolvedModel !== ce.resolvedModel) {
+        mismatches.push(`the runtime used a different model: ${be.resolvedModel} → ${ce.resolvedModel}`);
+    }
+    if (be?.runtimeVersion && ce?.runtimeVersion && be.runtimeVersion !== ce.runtimeVersion) {
+        mismatches.push(`runtime version changed: ${be.runtimeVersion} → ${ce.runtimeVersion} ` +
+            `(built-in skills and routing can differ between versions)`);
+    }
+    if (be && ce && be.userConfig !== ce.userConfig) {
+        mismatches.push(`personal runtime config was ${ce.userConfig ? "loaded" : "excluded"} this run but ` +
+            `${be.userConfig ? "loaded" : "excluded"} in the baseline`);
+    }
+    if (be?.extraSkills && ce?.extraSkills) {
+        const was = new Set(be.extraSkills), now = new Set(ce.extraSkills);
+        const added = ce.extraSkills.filter((s) => !was.has(s));
+        const removed = be.extraSkills.filter((s) => !now.has(s));
+        if (added.length || removed.length) {
+            mismatches.push(`skills loaded alongside the library changed: ` +
+                [added.length ? `+${added.join(", +")}` : "", removed.length ? `−${removed.join(", −")}` : ""]
+                    .filter(Boolean).join(" "));
+        }
+    }
+    const baseByKey = new Map(base.cases.map((c) => [caseKey(c.prompt, c.expected), c]));
+    const currentKeys = new Set(result.cases.map((c) => caseKey(c.prompt, c.expected)));
+    const compared = [];
+    const newCases = [];
+    for (const c of result.cases) {
+        const b = baseByKey.get(caseKey(c.prompt, c.expected));
+        if (!b) {
+            newCases.push(c.prompt);
+            continue;
+        }
+        const after = { hits: c.stats.hits, n: c.stats.n, pHat: c.stats.n ? c.stats.hits / c.stats.n : 0 };
+        const before = { hits: b.hits, n: b.n, pHat: b.n ? b.hits / b.n : 0 };
+        const untrustworthy = before.n === 0 || after.n === 0;
+        const fisherP = untrustworthy ? 1
+            : fisherExact2x2(before.hits, before.n - before.hits, after.hits, after.n - after.hits);
+        compared.push({
+            prompt: c.prompt, expected: c.expected, before, after,
+            delta: after.pHat - before.pHat,
+            fisherP, fisherPAdj: 1, untrustworthy,
+            regressed: false, improved: false,
+        });
+    }
+    const missingCases = base.cases
+        .filter((b) => !currentKeys.has(caseKey(b.prompt, b.expected)))
+        .map((b) => b.prompt);
+    // one Fisher test per gated case = a family — correct before deciding, same as context
+    const family = compared.filter((r) => !r.untrustworthy);
+    const adj = bhAdjust(family.map((r) => r.fisherP));
+    family.forEach((r, j) => { r.fisherPAdj = adj[j]; });
+    for (const r of compared) {
+        r.regressed = !r.untrustworthy && r.delta < 0 && r.fisherPAdj < 0.05;
+        r.improved = !r.untrustworthy && r.delta > 0 && r.fisherPAdj < 0.05;
+    }
+    const regressions = compared.filter((r) => r.regressed).length;
+    return {
+        baselineDate: bm.date, baselineVersion: bm.version,
+        mismatches, cases: compared, newCases, missingCases,
+        regressions,
+        improvements: compared.filter((r) => r.improved).length,
+        exitCode: regressions > 0 ? 1 : 0,
+    };
+}

package/dist/cli.js

@@ -2,7 +2,8 @@
 /** skill-probe CLI.
  *   skill-probe --config <f>                  audit a co-loaded skill library
  *   skill-probe fix --config <f> --skill <n>  rewrite a skill's description & prove the lift
- * Exit: 0 = pass/applied, 1 = behavioral fail/reverted, 2 = inconclusive/infra error/usage. */
+ * Exit: 0 = pass/applied, 1 = behavioral fail/not applied, 2 = inconclusive/infra error/usage,
+ *       130/143 = interrupted (children killed, sandboxes removed). */
 import { parseArgs } from "node:util";
 import { resolve } from "node:path";
 import { runAudit } from "./eval.js";
@@ -12,11 +13,16 @@ import { runDiagnose } from "./diagnose.js";
 import { runDoctor } from "./doctor.js";
 import { listSkills, generateCases, genConfig } from "./gen.js";
 import { loadConfig, parseProbability, parseIntFlag, ConfigError } from "./config.js";
-import { renderTable, renderMarkdown, renderJson, renderFix, renderContext, renderDoctor, renderDiagnose } from "./report.js";
+import { loadBaseline, saveBaseline, compareToBaseline } from "./baseline.js";
+import { renderTable, renderMarkdown, renderJson, renderFix, renderContext, renderDoctor, renderDiagnose, renderBaseline } from "./report.js";
+import { Budget } from "./orchestrator.js";
+import { toolVersion } from "./manifest.js";
+import { installSignalHandlers } from "./proc.js";
+const COMMANDS = new Set(["audit", "context", "diagnose", "doctor", "gen", "fix"]);
 const HELP = `skill-probe — audit a co-loaded agent skill library by real activation behavior.
 
 USAGE
-  skill-probe --config <file.json> [options]                 # audit
+  skill-probe [audit] --config <file.json> [options]         # audit
   skill-probe context --config <file.json> [--ablate]        # isolation vs co-loaded activation rates
   skill-probe diagnose --config <file.json> [options]        # why a skill fails: routing-miss vs description
   skill-probe doctor [--cwd <dir>] [--config <file.json>]    # preflight: setup/auth/config sanity
@@ -31,11 +37,28 @@ OPTIONS
                            Recorded in the report — REQUIRED for a fair cross-runtime comparison.
       --k <n>              max VALID repeats per prompt (positive integer; default 10)
       --threshold <p>      min acceptable reliability, strictly 0..1 (default 0.7 = smoke test)
-      --conf <p>           confidence level: 0.9 | 0.95 | 0.99 (default 0.95)
+      --conf <p>           confidence: 0.9 | 0.95 | 0.99 (default 0.95). A verdict is wrong with
+                           probability <= 1-conf, however early the run stops.
+      --max-turns <n>      agent turns per probe before reading the outcome (default 1)
+      --timeout <sec>      per-probe timeout in seconds (default 120)
+      --user-config        also load your personal runtime config (user settings, plugins, MCP
+                           servers) into probes. Off by default (claude-code): results then
+                           reproduce in CI. opencode always loads its global config.
+      --concurrency <n>    cases probed in parallel (default 1; mind your rate limits)
+      --max-cost <usd>     stop probing once runtime probes have spent this much; cases cut short
+                           are inconclusive (error if never started), never a pass. LLM calls
+                           (gen, fix's rewrite, diagnose's forced choice) aren't counted
       --json               emit JSON instead of a table
       --markdown           emit a Markdown table (paste into a PR/README)
       --no-cost            hide the cost line (e.g. when on a Claude subscription)
       --quiet              suppress the live progress line on stderr
+  -v, --version            print the version
+  audit only:
+      --fail-on-theft      fail a case when ANY other skill fires, even if reliability clears the bar
+      --save-baseline <f>  after the run, save per-case counts + a run manifest to <f>
+      --baseline <f>       compare this run against a saved baseline: per-case Fisher's exact,
+                           Benjamini-Hochberg corrected — exit 1 ONLY on a significant regression
+                           (noise passes; real drops fail)
   context only:
       --ablate             leave-one-out: for each interference case, re-run with each suspect
                            sibling removed and NAME the thief when activation recovers (Fisher +
@@ -44,7 +67,7 @@ OPTIONS
   doctor only:
       --cwd <dir>          project dir containing .claude/skills/ (default: .)
       --config <file>      also sanity-check a config (expected skills exist, k/threshold feasible)
-      --skip-probe         don't run the harmless live probe (skip the auth check; no cost)
+      --skip-probe         don't run the harmless live probes (skip the auth/load checks; no cost)
   gen only (uses ANTHROPIC_API_KEY if set, else the logged-in 'claude' CLI):
       --cwd <dir>          project dir containing .claude/skills/ (default: .)
       --per-skill <n>      should-fire prompts to draft per skill (default 3)
@@ -54,8 +77,8 @@ OPTIONS
       --apply-threshold <p> min P(improvement) to keep the rewrite (default 0.9)
   -h, --help               show this help
 
-Reliability carries a Wilson CI + sample size k. fix rewrites the description, runs an INTERLEAVED
-before/after, and applies it only if the Bayesian P(improvement) clears the apply-threshold.`;
+Reliability carries a Wilson CI + sample size k. fix runs a PAIRED before/after in two sandboxes and
+writes the new description only if the Bayesian P(improvement) clears the apply-threshold.`;
 let progressLen = 0;
 function makeProgress(values) {
     if (values["json"] || values["quiet"] || !process.stderr.isTTY)
@@ -97,30 +120,55 @@ function makeDiagProgress(values) {
         progressLen = line.length;
     };
 }
-async function audit(values) {
-    const cfg = loadConfig(values["config"], {
-        runtime: values["runtime"], model: values["model"],
-        k: values["k"], threshold: values["threshold"],
-        conf: values["conf"],
+/** The config plus every CLI override that changes what is measured. */
+function configFrom(values, opts = {}) {
+    const str = (k) => values[k];
+    return loadConfig(values["config"], {
+        runtime: str("runtime"), model: str("model"), k: str("k"), conf: str("conf"),
+        ...(opts.threshold === false ? {} : { threshold: str("threshold") }),
+        maxTurns: str("max-turns"), timeout: str("timeout"),
+        userConfig: Boolean(values["user-config"]), failOnTheft: Boolean(values["fail-on-theft"]),
     });
-    const result = await runAudit(cfg, makeProgress(values));
+}
+/** Run-time knobs that DON'T change what is measured (parallelism, spend cap). */
+function runOpts(values) {
+    const concurrency = parseIntFlag(values["concurrency"], "--concurrency", 1, 1);
+    const raw = values["max-cost"];
+    if (raw === undefined)
+        return { concurrency };
+    const max = Number(raw);
+    if (!Number.isFinite(max) || max <= 0) {
+        throw new ConfigError([`--max-cost must be a positive number of USD (got ${JSON.stringify(raw)})`]);
+    }
+    return { concurrency, budget: new Budget(max) };
+}
+async function audit(values) {
+    const cfg = configFrom(values);
+    const progress = makeProgress(values);
+    const result = await runAudit(cfg, { ...runOpts(values), ...(progress ? { onProgress: progress } : {}) });
     clearProgress();
     const showCost = !values["no-cost"];
-    const text = values["json"] ? renderJson(result)
-        : values["markdown"] ? renderMarkdown(result, { showCost })
-            : renderTable(result, { showCost });
+    let cmp;
+    if (values["baseline"])
+        cmp = compareToBaseline(result, loadBaseline(values["baseline"]));
+    if (values["save-baseline"]) {
+        saveBaseline(result, values["save-baseline"]);
+        console.error(`baseline saved to ${values["save-baseline"]}`);
+    }
+    const text = values["json"] ? renderJson(cmp ? { ...result, baseline: cmp } : result)
+        : values["markdown"] ? renderMarkdown(result, { showCost }) + (cmp ? "\n\n" + renderBaseline(cmp) : "")
+            : renderTable(result, { showCost }) + (cmp ? "\n\n" + renderBaseline(cmp) : "");
     console.log(text);
+    // precedence mirrors the rest of the tool: a behavioral signal (fail/regression, 1) outranks
+    // inconclusive/infra (2), which outranks clean (0).
+    if (result.exitCode === 1 || cmp?.exitCode === 1)
+        return 1;
     return result.exitCode;
 }
 async function diagnose(values) {
-    const cfg = loadConfig(values["config"], {
-        runtime: values["runtime"], model: values["model"],
-        k: values["k"], threshold: values["threshold"],
-        conf: values["conf"],
-    });
-    const result = await runDiagnose(cfg, {
-        ...(makeDiagProgress(values) ? { onProgress: makeDiagProgress(values) } : {}),
-    });
+    const cfg = configFrom(values);
+    const onProgress = makeDiagProgress(values);
+    const result = await runDiagnose(cfg, { ...runOpts(values), ...(onProgress ? { onProgress } : {}) });
     clearProgress();
     const showCost = !values["no-cost"];
     console.log(values["json"] ? renderJson(result) : renderDiagnose(result, { showCost }));
@@ -133,20 +181,17 @@ async function doctor(values) {
         ...(values["runtime"] ? { runtime: values["runtime"] } : {}),
         ...(values["model"] ? { model: values["model"] } : {}),
         ...(values["skip-probe"] ? { skipProbe: true } : {}),
+        ...(values["user-config"] ? { userConfig: true } : {}),
     });
     console.log(values["json"] ? renderJson(result) : renderDoctor(result));
     return result.exitCode;
 }
 async function context(values) {
-    const cfg = loadConfig(values["config"], {
-        runtime: values["runtime"], model: values["model"],
-        k: values["k"], threshold: values["threshold"],
-        conf: values["conf"],
-    });
+    const cfg = configFrom(values);
     const suspects = parseIntFlag(values["suspects"], "--suspects", 3, 1);
     const onProgress = makeContextProgress(values);
     const result = await runContextAudit(cfg, {
-        ablate: Boolean(values["ablate"]), suspects,
+        ablate: Boolean(values["ablate"]), suspects, ...runOpts(values),
         ...(onProgress ? { onProgress } : {}),
     });
     clearProgress();
@@ -178,12 +223,11 @@ async function fix(values) {
         console.error("fix requires --skill <name>");
         return 2;
     }
-    const cfg = loadConfig(values["config"], {
-        runtime: values["runtime"], model: values["model"],
-        k: values["k"], conf: values["conf"],
-    });
+    const cfg = configFrom(values, { threshold: false });
     const applyBar = parseProbability(values["apply-threshold"], "--apply-threshold", 0.9);
-    const result = await runFix(cfg, { skill, k: cfg.k, applyBar, ...(cfg.model ? { model: cfg.model } : {}) });
+    const result = await runFix(cfg, {
+        skill, k: cfg.k, applyBar, ...runOpts(values), ...(cfg.model ? { model: cfg.model } : {}),
+    });
     console.log(values["json"] ? renderJson(result) : renderFix(result));
     return result.applied ? 0 : 1;
 }
@@ -204,7 +248,16 @@ async function main() {
             decoys: { type: "string" },
             ablate: { type: "boolean", default: false },
             suspects: { type: "string" },
+            baseline: { type: "string" },
+            "save-baseline": { type: "string" },
             "skip-probe": { type: "boolean", default: false },
+            "max-turns": { type: "string" },
+            timeout: { type: "string" },
+            "user-config": { type: "boolean", default: false },
+            "fail-on-theft": { type: "boolean", default: false },
+            concurrency: { type: "string" },
+            "max-cost": { type: "string" },
+            version: { type: "boolean", short: "v", default: false },
             json: { type: "boolean", default: false },
             markdown: { type: "boolean", default: false },
             "no-cost": { type: "boolean", default: false },
@@ -216,7 +269,16 @@ async function main() {
         console.log(HELP);
         process.exit(0);
     }
-    const cmd = positionals[0];
+    if (values.version) {
+        console.log(toolVersion());
+        process.exit(0);
+    }
+    const cmd = positionals[0] ?? "audit";
+    if (!COMMANDS.has(cmd) || positionals.length > 1) {
+        console.error(`unknown command: ${positionals.join(" ")} (expected one of: ${[...COMMANDS].join(", ")})`);
+        process.exit(2);
+    }
+    installSignalHandlers();
     let code;
     if (cmd === "gen") {
         code = await gen(values);
@@ -226,7 +288,7 @@ async function main() {
     }
     else {
         if (!values.config) {
-            console.log(HELP);
+            console.error(HELP);
             process.exit(2);
         }
         code = cmd === "fix" ? await fix(values)

package/dist/config.d.ts

@@ -18,5 +18,9 @@ export interface Overrides {
     k?: string | undefined;
     threshold?: string | undefined;
     conf?: string | undefined;
+    maxTurns?: string | undefined;
+    timeout?: string | undefined;
+    userConfig?: boolean | undefined;
+    failOnTheft?: boolean | undefined;
 }
 export declare function loadConfig(configPath: string, overrides?: Overrides): Config;

package/dist/config.js

@@ -3,6 +3,10 @@
 import { readFileSync } from "node:fs";
 import { resolve, dirname } from "node:path";
 const VALID_CONF = new Set([0.9, 0.95, 0.99]);
+const KNOWN_KEYS = new Set([
+    "runtime", "model", "cwd", "k", "threshold", "conf", "cases",
+    "maxTurns", "timeout", "userConfig", "failOnTheft",
+]);
 export class ConfigError extends Error {
     errors;
     constructor(errors) {
@@ -17,6 +21,13 @@ export class ConfigError extends Error {
 export function validateConfig(raw, configPath) {
     const r = (raw ?? {});
     const errs = [];
+    // a typo'd key ("treshold") would otherwise silently fall back to the default — fail loud.
+    // "$schema" and "_"-prefixed keys are allowed for tooling and comments.
+    for (const key of Object.keys(r)) {
+        if (!KNOWN_KEYS.has(key) && !key.startsWith("$") && !key.startsWith("_")) {
+            errs.push(`unknown config key "${key}" (known: ${[...KNOWN_KEYS].join(", ")})`);
+        }
+    }
     const runtime = typeof r["runtime"] === "string" && r["runtime"] ? r["runtime"] : "claude-code";
     let model;
     if (r["model"] !== undefined) {
@@ -41,6 +52,20 @@ export function validateConfig(raw, configPath) {
     if (typeof conf !== "number" || !VALID_CONF.has(conf)) {
         errs.push(`conf must be one of 0.9, 0.95, 0.99 (got ${JSON.stringify(r["conf"])})`);
     }
+    const maxTurns = r["maxTurns"];
+    if (maxTurns !== undefined && (typeof maxTurns !== "number" || !Number.isInteger(maxTurns) || maxTurns < 1)) {
+        errs.push(`maxTurns must be a positive integer (got ${JSON.stringify(maxTurns)})`);
+    }
+    // seconds in the file / on the CLI; ms internally
+    const timeout = r["timeout"];
+    if (timeout !== undefined && (typeof timeout !== "number" || !Number.isFinite(timeout) || timeout <= 0)) {
+        errs.push(`timeout must be a positive number of seconds (got ${JSON.stringify(timeout)})`);
+    }
+    for (const key of ["userConfig", "failOnTheft"]) {
+        if (r[key] !== undefined && typeof r[key] !== "boolean") {
+            errs.push(`${key} must be true or false (got ${JSON.stringify(r[key])})`);
+        }
+    }
     const cases = Array.isArray(r["cases"]) ? r["cases"] : [];
     if (cases.length === 0)
         errs.push("config needs a non-empty `cases` array");
@@ -63,6 +88,10 @@ export function validateConfig(raw, configPath) {
         threshold: threshold,
         conf: conf,
         cases: cases,
+        ...(maxTurns !== undefined ? { maxTurns: maxTurns } : {}),
+        ...(timeout !== undefined ? { timeoutMs: Math.round(timeout * 1000) } : {}),
+        ...(r["userConfig"] === true ? { userConfig: true } : {}),
+        ...(r["failOnTheft"] === true ? { failOnTheft: true } : {}),
     };
 }
 /** Validate a probability-ish flag (e.g. --apply-threshold): must be a number strictly in (0,1). */
@@ -104,5 +133,13 @@ export function loadConfig(configPath, overrides = {}) {
         merged["threshold"] = Number(overrides.threshold);
     if (overrides.conf !== undefined)
         merged["conf"] = Number(overrides.conf);
+    if (overrides.maxTurns !== undefined)
+        merged["maxTurns"] = Number(overrides.maxTurns);
+    if (overrides.timeout !== undefined)
+        merged["timeout"] = Number(overrides.timeout);
+    if (overrides.userConfig)
+        merged["userConfig"] = true;
+    if (overrides.failOnTheft)
+        merged["failOnTheft"] = true;
     return validateConfig(merged, configPath);
 }