skill-probe 0.1.0

package/dist/eval.js

@@ -0,0 +1,50 @@
+import { reliability, sequentialDecision } from "./stats.js";
+import { measure } from "./orchestrator.js";
+import { getAdapter } from "./adapters/index.js";
+function verdictFor(stats, decision, theft) {
+    // `error` means INFRASTRUCTURE failures dominate — untrustworthy. It is NOT about small k:
+    // a small-but-all-valid run is `inconclusive` (wide CI), which the report explains.
+    if (stats.n === 0 || (stats.errors > 0 && stats.errors >= stats.n))
+        return "error";
+    if (theft.length > 0)
+        return "fail";
+    if (decision === "above")
+        return "pass";
+    if (decision === "below")
+        return "fail";
+    return "inconclusive"; // includes "k too small to decide" — report says how many runs are needed
+}
+export async function runAudit(cfg) {
+    const adapter = getAdapter(cfg.runtime);
+    const cases = [];
+    let totalCost = 0;
+    for (const c of cfg.cases) {
+        const stats = await measure(adapter, c.prompt, c.expected, {
+            cwd: cfg.cwd, maxK: cfg.k, threshold: cfg.threshold, conf: cfg.conf,
+            ...(cfg.model ? { model: cfg.model } : {}),
+        });
+        totalCost += stats.totalCost;
+        const rel = reliability(stats.hits, stats.n, cfg.conf);
+        const want = c.expected ?? "None";
+        const theft = Object.keys(stats.dist).filter((s) => s !== want && s !== "None");
+        const decision = sequentialDecision(stats.hits, stats.n, cfg.threshold, cfg.conf);
+        cases.push({
+            prompt: c.prompt, expected: c.expected, stats, reliability: rel,
+            theft, decision, verdict: verdictFor(stats, decision, theft),
+        });
+    }
+    // NOTE: Benjamini-Hochberg multiplicity control over per-case exact-test p-values lands in M2,
+    // once each case carries a real p-value rather than a CI decision. The functions are tested
+    // in stats.ts but are NOT yet wired into this audit path.
+    const counts = {
+        pass: cases.filter((c) => c.verdict === "pass").length,
+        fail: cases.filter((c) => c.verdict === "fail").length,
+        inconclusive: cases.filter((c) => c.verdict === "inconclusive").length,
+        error: cases.filter((c) => c.verdict === "error").length,
+    };
+    const exitCode = counts.fail > 0 ? 1 : (counts.error > 0 || counts.inconclusive > 0) ? 2 : 0;
+    return {
+        runtime: cfg.runtime, model: cfg.model ?? "(runtime default)",
+        threshold: cfg.threshold, cases, totalCost, counts, exitCode,
+    };
+}

package/dist/fix.d.ts

@@ -0,0 +1,37 @@
+import type { Config } from "./types.ts";
+import { reliability } from "./stats.ts";
+export declare function readDesc(file: string): string;
+/** Replace only the single-line description; backups are handled explicitly in runFix (not here,
+ * because the interleaved loop calls this many times and must not snapshot intermediate states). */
+export declare function writeDesc(file: string, desc: string): void;
+export interface Sibling {
+    name: string;
+    desc: string;
+}
+export declare function defaultRewrite(skill: string, cur: string, prompts: string[], siblings: Sibling[], model?: string): Promise<string>;
+export interface FixResult {
+    skill: string;
+    oldDesc: string;
+    newDesc: string;
+    before: ReturnType<typeof reliability>;
+    after: ReturnType<typeof reliability>;
+    pImprove: number;
+    deltaMean: number;
+    deltaCi: [number, number];
+    n: number;
+    errors: number;
+    applied: boolean;
+    applyBar: number;
+    /** path of the kept backup (only when applied); null when reverted (no change made) */
+    backupPath: string | null;
+}
+export interface FixOpts {
+    skill: string;
+    k: number;
+    applyBar: number;
+    model?: string;
+}
+export interface FixDeps {
+    rewrite?: (skill: string, cur: string, prompts: string[], siblings: Sibling[], model?: string) => Promise<string>;
+}
+export declare function runFix(cfg: Config, opts: FixOpts, deps?: FixDeps): Promise<FixResult>;

package/dist/fix.js

@@ -0,0 +1,111 @@
+/** `skill-probe fix` — rewrite a failing skill's description, then PROVE the lift is real with
+ * the Bayesian Beta-Binomial (not "it looks better"). Upgrades on the Python prototype:
+ *  - INTERLEAVED before/after (old desc -> probe -> new desc -> probe, paired) controls for
+ *    session drift at the same probe cost as block-wise.
+ *  - COLLISION-AWARE rewrite: the LLM is told sibling descriptions so it won't steal triggers.
+ *  - Applies the new description only if P(improvement) >= bar AND the effect is positive;
+ *    otherwise reverts. Original is always backed up to SKILL.md.bak. */
+import { readFileSync, writeFileSync, existsSync, readdirSync, copyFileSync, unlinkSync } from "node:fs";
+import { join } from "node:path";
+import { getAdapter } from "./adapters/index.js";
+import { probImprovement, reliability } from "./stats.js";
+import { complete } from "./llm.js";
+const DESC_RE = /^description:\s*(.*)$/m;
+function skillFile(cfg, name) {
+    return join(cfg.cwd, ".claude", "skills", name, "SKILL.md");
+}
+export function readDesc(file) {
+    const m = DESC_RE.exec(readFileSync(file, "utf8"));
+    if (!m)
+        throw new Error(`no single-line 'description:' found in ${file} (multi-line block scalars not yet supported)`);
+    return m[1].trim();
+}
+/** Replace only the single-line description; backups are handled explicitly in runFix (not here,
+ * because the interleaved loop calls this many times and must not snapshot intermediate states). */
+export function writeDesc(file, desc) {
+    const txt = readFileSync(file, "utf8");
+    writeFileSync(file, txt.replace(DESC_RE, `description: ${desc}`));
+}
+/** Unique per-run snapshot path, never clobbering a prior run's backup. */
+function backupPathFor(file) {
+    const base = `${file}.bak.${Date.now()}`;
+    let p = base, i = 1;
+    while (existsSync(p))
+        p = `${base}-${i++}`;
+    return p;
+}
+export async function defaultRewrite(skill, cur, prompts, siblings, model) {
+    const sib = siblings.length ? siblings.map((s) => `- ${s.name}: "${s.desc}"`).join("\n") : "(none)";
+    const sys = "You optimize Agent Skill trigger descriptions. The description is the routing signal a "
+        + "model reads to decide whether to load the skill. Return ONLY the new one-line description — no "
+        + "quotes, no preamble, no trailing notes.";
+    const user = `Skill \`${skill}\` current description:\n${cur}\n\n`
+        + `It FAILS to reliably trigger on these user phrasings:\n${prompts.map((p) => `- "${p}"`).join("\n")}\n\n`
+        + `Rewrite the description so it reliably triggers on those AND similar phrasings, front-loading `
+        + `trigger keywords. CRITICAL: do NOT overlap with these sibling skills — you must not start `
+        + `stealing their triggers:\n${sib}\n\nKeep it ONE sentence.`;
+    const out = await complete(sys, user, model ? { model, maxTokens: 300 } : { maxTokens: 300 });
+    return out.replace(/^["']+|["']+$/g, "").trim();
+}
+export async function runFix(cfg, opts, deps = {}) {
+    const adapter = getAdapter(cfg.runtime);
+    const file = skillFile(cfg, opts.skill);
+    if (!existsSync(file))
+        throw new Error(`skill not found: ${file}`);
+    const root = join(cfg.cwd, ".claude", "skills");
+    const siblings = readdirSync(root)
+        .filter((n) => n !== opts.skill && existsSync(join(root, n, "SKILL.md")))
+        .map((n) => ({ name: n, desc: readDesc(join(root, n, "SKILL.md")) }));
+    const prompts = cfg.cases.filter((c) => c.expected === opts.skill).map((c) => c.prompt);
+    if (prompts.length === 0)
+        throw new Error(`no cases in the config expect skill '${opts.skill}'`);
+    const oldDesc = readDesc(file);
+    const rewrite = deps.rewrite ?? defaultRewrite;
+    const newDesc = await rewrite(opts.skill, oldDesc, prompts, siblings, opts.model);
+    // Snapshot the EXACT pre-fix file once, to a unique path (never clobber a prior run's backup).
+    const backupPath = backupPathFor(file);
+    copyFileSync(file, backupPath);
+    const probeWith = async (desc, prompt) => {
+        writeDesc(file, desc);
+        return adapter.probe(prompt, { cwd: cfg.cwd, ...(cfg.model ? { model: cfg.model } : {}) });
+    };
+    let beforeHits = 0, afterHits = 0, n = 0, errors = 0;
+    try {
+        for (let i = 0; i < opts.k; i++) {
+            for (const p of prompts) {
+                const rb = await probeWith(oldDesc, p);
+                const ra = await probeWith(newDesc, p);
+                if (rb.status === "error" || ra.status === "error") {
+                    errors++;
+                    continue;
+                } // drop incomplete pair
+                if (rb.skillFired === opts.skill)
+                    beforeHits++;
+                if (ra.skillFired === opts.skill)
+                    afterHits++;
+                n++;
+            }
+        }
+        const imp = probImprovement(beforeHits, n, afterHits, n);
+        const applied = n > 0 && imp.pImprove >= opts.applyBar && imp.deltaMean > 0;
+        if (applied) {
+            writeDesc(file, newDesc); // commit; keep the timestamped backup
+        }
+        else {
+            copyFileSync(backupPath, file); // restore exact original
+            unlinkSync(backupPath); // no change made -> don't leave a redundant backup
+        }
+        return {
+            skill: opts.skill, oldDesc, newDesc,
+            before: reliability(beforeHits, n), after: reliability(afterHits, n),
+            pImprove: imp.pImprove, deltaMean: imp.deltaMean, deltaCi: imp.deltaCi,
+            n, errors, applied, applyBar: opts.applyBar,
+            backupPath: applied ? backupPath : null,
+        };
+    }
+    catch (e) {
+        copyFileSync(backupPath, file); // restore exact pre-fix file on any error
+        unlinkSync(backupPath);
+        throw e;
+    }
+}

package/dist/llm.d.ts

@@ -0,0 +1,4 @@
+export declare function complete(system: string, user: string, opts?: {
+    model?: string;
+    maxTokens?: number;
+}): Promise<string>;

package/dist/llm.js

@@ -0,0 +1,33 @@
+/** Minimal Anthropic client for the `fix` command's description rewrite. Uses Node's built-in
+ * fetch (Node >=18) — zero deps. Needs ANTHROPIC_API_KEY (only `fix` uses this, not the audit). */
+const ANTHROPIC_URL = "https://api.anthropic.com/v1/messages";
+export async function complete(system, user, opts = {}) {
+    const key = process.env["ANTHROPIC_API_KEY"];
+    if (!key) {
+        throw new Error("ANTHROPIC_API_KEY is required for `skill-probe fix` (it rewrites the description).");
+    }
+    const model = opts.model ?? process.env["SKILL_PROBE_FIX_MODEL"] ?? "claude-sonnet-4-6";
+    const res = await fetch(ANTHROPIC_URL, {
+        method: "POST",
+        headers: {
+            "content-type": "application/json",
+            "x-api-key": key,
+            "anthropic-version": "2023-06-01",
+        },
+        body: JSON.stringify({
+            model,
+            max_tokens: opts.maxTokens ?? 400,
+            system,
+            messages: [{ role: "user", content: user }],
+        }),
+    });
+    if (!res.ok) {
+        throw new Error(`Anthropic API ${res.status}: ${(await res.text()).slice(0, 200)}`);
+    }
+    const body = (await res.json());
+    return (body.content ?? [])
+        .filter((b) => b.type === "text")
+        .map((b) => b.text ?? "")
+        .join("")
+        .trim();
+}

package/dist/orchestrator.d.ts

@@ -0,0 +1,24 @@
+/** Runs one prompt up to k VALID times with sequential early-stopping vs the threshold.
+ * Infra errors (status="error") are retried up to a budget and never counted as a behavioral
+ * outcome (finding #1) — they're tracked separately so the audit can flag untrustworthy cases. */
+import type { RuntimeAdapter } from "./types.ts";
+export interface RunStats {
+    /** valid probes whose outcome matched the expected behavior */
+    hits: number;
+    /** number of VALID probes (excludes infra errors) */
+    n: number;
+    /** infrastructure failures (timeout/auth/crash/empty) */
+    errors: number;
+    /** last infra-error reason (for the report) */
+    lastError?: string;
+    /** distribution of outcomes over valid probes: skill name (or "None") -> count */
+    dist: Record<string, number>;
+    totalCost: number;
+}
+export declare function measure(adapter: RuntimeAdapter, prompt: string, expected: string | null, opts: {
+    cwd: string;
+    maxK: number;
+    threshold: number;
+    conf: number;
+    model?: string;
+}): Promise<RunStats>;

package/dist/orchestrator.js

@@ -0,0 +1,29 @@
+import { sequentialDecision } from "./stats.js";
+export async function measure(adapter, prompt, expected, opts) {
+    const want = expected ?? "None";
+    const dist = {};
+    let hits = 0, n = 0, errors = 0, totalCost = 0;
+    let lastError;
+    // allow up to 50% extra attempts to absorb transient infra errors without inflating k
+    const maxAttempts = opts.maxK + Math.ceil(opts.maxK / 2);
+    for (let attempt = 0; attempt < maxAttempts && n < opts.maxK; attempt++) {
+        const r = await adapter.probe(prompt, { cwd: opts.cwd, ...(opts.model ? { model: opts.model } : {}) });
+        totalCost += r.cost;
+        if (r.status === "error") {
+            errors++;
+            lastError = r.error;
+            continue; // do NOT record as a behavioral outcome
+        }
+        const key = r.skillFired ?? "None";
+        dist[key] = (dist[key] ?? 0) + 1;
+        if (key === want)
+            hits++;
+        n++;
+        if (n >= 3) {
+            const d = sequentialDecision(hits, n, opts.threshold, opts.conf);
+            if (d !== "undecided")
+                break;
+        }
+    }
+    return { hits, n, errors, dist, totalCost, ...(lastError ? { lastError } : {}) };
+}

package/dist/report.d.ts

@@ -0,0 +1,5 @@
+import type { AuditResult } from "./eval.ts";
+import type { FixResult } from "./fix.ts";
+export declare function renderTable(a: AuditResult): string;
+export declare function renderJson(a: AuditResult | unknown): string;
+export declare function renderFix(f: FixResult): string;

package/dist/report.js

@@ -0,0 +1,74 @@
+/** Human table + machine JSON rendering. Every number ships with its CI and k; infrastructure
+ * errors are shown distinctly from behavioral results. */
+import { wilsonInterval } from "./stats.js";
+const pct = (x) => `${Math.round(x * 100)}%`;
+const LABEL = {
+    pass: "PASS        ", fail: "FAIL        ",
+    inconclusive: "INCONCLUSIVE", error: "ERROR       ",
+};
+/** Smallest k at which an all-success run could certify reliability >= threshold. */
+function kToCertify(threshold, conf) {
+    for (let k = 1; k <= 500; k++) {
+        if (wilsonInterval(k, k, conf).lo > threshold)
+            return k;
+    }
+    return 500;
+}
+export function renderTable(a) {
+    const out = [];
+    out.push(`skill-probe — runtime: ${a.runtime}  model: ${a.model}  threshold: ${pct(a.threshold)}`);
+    out.push("");
+    for (const c of a.cases) {
+        const r = c.reliability;
+        out.push(`  ${LABEL[c.verdict]}  expect=${c.expected ?? "(none)"}  | ${c.prompt}`);
+        if (c.verdict === "error") {
+            out.push(`        ⛔ infrastructure error — result NOT trustworthy ` +
+                `(${c.stats.errors} failed, ${c.stats.n} valid)`);
+            if (c.stats.lastError)
+                out.push(`        reason: ${c.stats.lastError}`);
+            continue;
+        }
+        out.push(`        reliability ${pct(r.pHat)} [${pct(r.ciLow)}, ${pct(r.ciHigh)}] k=${r.k}` +
+            (c.stats.errors ? `  (+${c.stats.errors} infra errors ignored)` : ""));
+        const dist = Object.entries(c.stats.dist)
+            .sort((p, q) => q[1] - p[1]).map(([s, n]) => `${s}×${n}`).join(", ");
+        out.push(`        outcomes: ${dist}`);
+        if (c.theft.length)
+            out.push(`        ⚠ trigger-theft by: ${c.theft.join(", ")}`);
+        if (c.verdict === "inconclusive") {
+            out.push(`        ↳ CI straddles threshold — need ~k=${kToCertify(a.threshold, r.conf)} ` +
+                `all-success runs to certify ≥${pct(a.threshold)} (currently k=${r.k})`);
+        }
+    }
+    out.push("");
+    out.push(`Result: ${a.counts.pass} pass / ${a.counts.fail} fail / ` +
+        `${a.counts.inconclusive} inconclusive / ${a.counts.error} error  |  ` +
+        `exit ${a.exitCode}  |  cost $${a.totalCost.toFixed(4)}`);
+    return out.join("\n");
+}
+export function renderJson(a) {
+    return JSON.stringify(a, null, 2);
+}
+export function renderFix(f) {
+    const out = [];
+    out.push(`skill-probe fix — skill: ${f.skill}`);
+    out.push("");
+    out.push(`  old: ${f.oldDesc}`);
+    out.push(`  new: ${f.newDesc}`);
+    out.push("");
+    out.push(`  before: ${pct(f.before.pHat)} [${pct(f.before.ciLow)}, ${pct(f.before.ciHigh)}]  ` +
+        `after: ${pct(f.after.pHat)} [${pct(f.after.ciLow)}, ${pct(f.after.ciHigh)}]  ` +
+        `(${f.n} paired runs${f.errors ? `, ${f.errors} dropped to infra errors` : ""})`);
+    out.push(`  P(rewrite improved reliability) = ${pct(f.pImprove)}   ` +
+        `Δ = ${f.deltaMean >= 0 ? "+" : ""}${pct(f.deltaMean)} [${pct(f.deltaCi[0])}, ${pct(f.deltaCi[1])}]`);
+    out.push("");
+    if (f.applied) {
+        out.push(`  ✅ APPLIED — P(improvement) ${pct(f.pImprove)} ≥ bar ${pct(f.applyBar)} and Δ>0.`);
+        out.push(`     original backed up to: ${f.backupPath}`);
+    }
+    else {
+        out.push(`  ↩ REVERTED — not enough evidence the rewrite helps ` +
+            `(need P(improvement) ≥ ${pct(f.applyBar)} and Δ>0). Description unchanged, no backup kept.`);
+    }
+    return out.join("\n");
+}

package/dist/stats.d.ts

@@ -0,0 +1,34 @@
+/**
+ * Statistical core (TS port of stats_ref/stats.py — the validated reference impl).
+ * Skill activation is Bernoulli with unknown p; we estimate from k runs and never
+ * report a bare point. See ../../STATS.md.
+ */
+export interface Interval {
+    lo: number;
+    hi: number;
+    center: number;
+}
+export type Decision = "above" | "below" | "undecided";
+/** Wilson score interval — well-behaved at p≈0/1 and small n (unlike Wald). */
+export declare function wilsonInterval(x: number, n: number, conf?: number): Interval;
+export interface Reliability {
+    pHat: number;
+    ciLow: number;
+    ciHigh: number;
+    k: number;
+    conf: number;
+}
+export declare function reliability(x: number, n: number, conf?: number): Reliability;
+/** Sequential stop rule: is p reliably above/below threshold yet, or keep sampling? */
+export declare function sequentialDecision(x: number, n: number, threshold: number, conf?: number): Decision;
+/** Two-sided Fisher's exact test on [[a,b],[c,d]] — exact, for small samples. */
+export declare function fisherExact2x2(a: number, b: number, c: number, d: number): number;
+export interface ImprovementResult {
+    pImprove: number;
+    deltaMean: number;
+    deltaCi: [number, number];
+}
+/** Bayesian Beta-Binomial: P(p_after > p_before) + delta credible interval. */
+export declare function probImprovement(xBefore: number, nBefore: number, xAfter: number, nAfter: number, prior?: number, samples?: number, seed?: number): ImprovementResult;
+/** Benjamini-Hochberg FDR. Returns booleans (true = flag as real). */
+export declare function benjaminiHochberg(pvals: number[], q?: number): boolean[];

package/dist/stats.js

@@ -0,0 +1,154 @@
+/**
+ * Statistical core (TS port of stats_ref/stats.py — the validated reference impl).
+ * Skill activation is Bernoulli with unknown p; we estimate from k runs and never
+ * report a bare point. See ../../STATS.md.
+ */
+const Z = {
+    "0.9": 1.6448536269514722,
+    "0.95": 1.959963984540054,
+    "0.99": 2.5758293035489004,
+};
+/** Wilson score interval — well-behaved at p≈0/1 and small n (unlike Wald). */
+export function wilsonInterval(x, n, conf = 0.95) {
+    if (n === 0)
+        return { lo: 0, hi: 1, center: 0.5 };
+    const z = Z[String(conf)];
+    if (z === undefined) {
+        throw new Error(`unsupported confidence ${conf}; use 0.9, 0.95, or 0.99`);
+    }
+    const p = x / n;
+    const denom = 1 + (z * z) / n;
+    const center = (p + (z * z) / (2 * n)) / denom;
+    const half = (z / denom) * Math.sqrt((p * (1 - p)) / n + (z * z) / (4 * n * n));
+    return { lo: Math.max(0, center - half), hi: Math.min(1, center + half), center };
+}
+export function reliability(x, n, conf = 0.95) {
+    const { lo, hi } = wilsonInterval(x, n, conf);
+    return { pHat: n ? x / n : 0, ciLow: lo, ciHigh: hi, k: n, conf };
+}
+/** Sequential stop rule: is p reliably above/below threshold yet, or keep sampling? */
+export function sequentialDecision(x, n, threshold, conf = 0.95) {
+    const { lo, hi } = wilsonInterval(x, n, conf);
+    if (lo > threshold)
+        return "above";
+    if (hi < threshold)
+        return "below";
+    return "undecided";
+}
+// --- lgamma (Lanczos) ---
+const LANCZOS = [
+    0.99999999999980993, 676.5203681218851, -1259.1392167224028,
+    771.32342877765313, -176.61502916214059, 12.507343278686905,
+    -0.13857109526572012, 9.9843695780195716e-6, 1.5056327351493116e-7,
+];
+function lgamma(z) {
+    if (z < 0.5) {
+        return Math.log(Math.PI / Math.sin(Math.PI * z)) - lgamma(1 - z);
+    }
+    z -= 1;
+    let a = LANCZOS[0];
+    const t = z + 7.5;
+    for (let i = 1; i < 9; i++)
+        a += LANCZOS[i] / (z + i);
+    return 0.5 * Math.log(2 * Math.PI) + (z + 0.5) * Math.log(t) - t + Math.log(a);
+}
+function hypergeomLogP(a, r1, c1, n) {
+    const b = r1 - a, c = c1 - a, d = n - r1 - c1 + a;
+    if (Math.min(a, b, c, d) < 0)
+        return -Infinity;
+    return (lgamma(r1 + 1) - lgamma(a + 1) - lgamma(b + 1) +
+        lgamma(n - r1 + 1) - lgamma(c + 1) - lgamma(d + 1) -
+        (lgamma(n + 1) - lgamma(c1 + 1) - lgamma(n - c1 + 1)));
+}
+/** Two-sided Fisher's exact test on [[a,b],[c,d]] — exact, for small samples. */
+export function fisherExact2x2(a, b, c, d) {
+    const n = a + b + c + d, r1 = a + b, c1 = a + c;
+    const pObs = Math.exp(hypergeomLogP(a, r1, c1, n));
+    const aMin = Math.max(0, c1 - (n - r1)), aMax = Math.min(r1, c1);
+    let total = 0;
+    for (let ai = aMin; ai <= aMax; ai++) {
+        const pa = Math.exp(hypergeomLogP(ai, r1, c1, n));
+        if (pa <= pObs * (1 + 1e-9))
+            total += pa;
+    }
+    return Math.min(1, total);
+}
+// --- seeded RNG + Beta sampling for the Bayesian before/after ---
+function mulberry32(seed) {
+    let a = seed >>> 0;
+    return () => {
+        a = (a + 0x6d2b79f5) | 0;
+        let t = Math.imul(a ^ (a >>> 15), 1 | a);
+        t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t;
+        return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
+    };
+}
+function gaussian(rng) {
+    let u = 0, v = 0;
+    while (u === 0)
+        u = rng();
+    while (v === 0)
+        v = rng();
+    return Math.sqrt(-2 * Math.log(u)) * Math.cos(2 * Math.PI * v);
+}
+function gammaSample(k, rng) {
+    if (k < 1)
+        return gammaSample(1 + k, rng) * Math.pow(rng(), 1 / k);
+    const d = k - 1 / 3, c = 1 / Math.sqrt(9 * d);
+    for (;;) {
+        let x, v;
+        do {
+            x = gaussian(rng);
+            v = 1 + c * x;
+        } while (v <= 0);
+        v = v * v * v;
+        const u = rng();
+        if (u < 1 - 0.0331 * x * x * x * x)
+            return d * v;
+        if (Math.log(u) < 0.5 * x * x + d * (1 - v + Math.log(v)))
+            return d * v;
+    }
+}
+function betaSample(a, b, rng) {
+    const ga = gammaSample(a, rng), gb = gammaSample(b, rng);
+    return ga / (ga + gb);
+}
+/** Bayesian Beta-Binomial: P(p_after > p_before) + delta credible interval. */
+export function probImprovement(xBefore, nBefore, xAfter, nAfter, prior = 0.5, samples = 40000, seed = 12345) {
+    const rng = mulberry32(seed);
+    const ab = prior + xBefore, bb = prior + (nBefore - xBefore);
+    const aa = prior + xAfter, ba = prior + (nAfter - xAfter);
+    let wins = 0, sum = 0;
+    const deltas = new Array(samples);
+    for (let i = 0; i < samples; i++) {
+        const pa = betaSample(aa, ba, rng), pb = betaSample(ab, bb, rng);
+        if (pa > pb)
+            wins++;
+        const d = pa - pb;
+        deltas[i] = d;
+        sum += d;
+    }
+    deltas.sort((p, q) => p - q);
+    return {
+        pImprove: wins / samples,
+        deltaMean: sum / samples,
+        deltaCi: [deltas[Math.floor(0.025 * samples)], deltas[Math.floor(0.975 * samples)]],
+    };
+}
+/** Benjamini-Hochberg FDR. Returns booleans (true = flag as real). */
+export function benjaminiHochberg(pvals, q = 0.05) {
+    const m = pvals.length;
+    if (m === 0)
+        return [];
+    const order = [...pvals.keys()].sort((i, j) => pvals[i] - pvals[j]);
+    let maxRank = 0;
+    order.forEach((i, idx) => {
+        const rank = idx + 1;
+        if (pvals[i] <= (rank / m) * q)
+            maxRank = rank;
+    });
+    const rejected = new Array(m).fill(false);
+    order.forEach((i, idx) => { if (idx + 1 <= maxRank)
+        rejected[i] = true; });
+    return rejected;
+}

package/dist/types.d.ts

@@ -0,0 +1,45 @@
+/** Core types. The normalized ProbeResult is the seam every runtime adapter maps to. */
+export interface ProbeResult {
+    /** "ok" = the runtime ran and we trust the reading; "error" = infrastructure failure
+     * (timeout, auth, crash, empty/unparseable output) — must NOT count as a behavioral outcome. */
+    status: "ok" | "error";
+    /** name of the skill that activated, or null if none fired. Meaningful only when status==="ok". */
+    skillFired: string | null;
+    /** tool names in invocation order (for execution-trajectory checks later) */
+    trajectory: string[];
+    /** USD cost of this single probe, if the runtime reports it */
+    cost: number;
+    /** failure reason when status==="error" */
+    error?: string;
+}
+export interface ProbeOpts {
+    cwd: string;
+    timeoutMs?: number;
+    /** model id to pin (runtime-specific format, e.g. "claude-sonnet-4-6" or
+     * "anthropic/claude-haiku-4-5"); omitted = the runtime's default. */
+    model?: string;
+}
+export interface RuntimeAdapter {
+    readonly name: string;
+    /** Run one prompt with the library at cwd co-loaded; report what fired. */
+    probe(prompt: string, opts: ProbeOpts): Promise<ProbeResult>;
+}
+export interface EvalCase {
+    prompt: string;
+    /** the skill that SHOULD fire, or null = a decoy that should fire nothing */
+    expected: string | null;
+}
+export interface Config {
+    runtime: string;
+    /** model id to pin for reproducible, fair cross-runtime comparison (optional) */
+    model?: string;
+    /** project dir that contains .claude/skills/ */
+    cwd: string;
+    /** max repeats per prompt (sequential stop may finish earlier) */
+    k: number;
+    /** minimum acceptable trigger reliability (0..1) */
+    threshold: number;
+    /** confidence level for intervals */
+    conf: number;
+    cases: EvalCase[];
+}

package/dist/types.js

@@ -0,0 +1,2 @@
+/** Core types. The normalized ProbeResult is the seam every runtime adapter maps to. */
+export {};

package/examples/audit.config.json

@@ -0,0 +1,14 @@
+{
+  "runtime": "claude-code",
+  "cwd": "./sample-skills",
+  "k": 10,
+  "threshold": 0.7,
+  "conf": 0.95,
+  "cases": [
+    { "prompt": "write a git commit message for my staged changes", "expected": "commit-writer" },
+    { "prompt": "commit my changes with a clear message", "expected": "commit-writer" },
+    { "prompt": "write a pull request description summarizing my commits", "expected": "pr-describer" },
+    { "prompt": "draft a PR description for this branch", "expected": "pr-describer" },
+    { "prompt": "what is the capital of France?", "expected": null }
+  ]
+}

package/examples/fix.config.json

@@ -0,0 +1,9 @@
+{
+  "runtime": "claude-code",
+  "cwd": "./sample-skills",
+  "k": 4,
+  "threshold": 0.7,
+  "cases": [
+    { "prompt": "write a git commit message for my staged changes", "expected": "commit-writer" }
+  ]
+}

package/examples/sample-skills/.claude/skills/commit-writer/SKILL.md

@@ -0,0 +1,7 @@
+---
+name: commit-writer
+description: Use this skill when the user wants to commit, write a commit message, or save their staged or current changes with a clear message to git history.
+---
+# Commit Writer
+When asked to write a commit message, produce a concise conventional-commits style message.
+Output ONLY the commit message, nothing else.

package/examples/sample-skills/.claude/skills/pr-describer/SKILL.md

@@ -0,0 +1,7 @@
+---
+name: pr-describer
+description: Use this skill when the user wants to draft, write, or generate a pull request (PR) description, summary, or overview for a branch, diff, or set of commits to prepare for code review.
+---
+# PR Describer
+When asked to describe a pull request, produce a short PR title and bullet summary.
+Output ONLY the PR description, nothing else.