skill-probe 0.8.0 → 0.10.0

package/dist/skills.js

@@ -0,0 +1,282 @@
+/** Reading and writing SKILL.md frontmatter. One parser for every command — gen, diagnose, doctor,
+ * context and fix all used to grep `description:` with a single-line regex, which read a YAML block
+ * scalar (`description: >-` followed by indented lines) as the literal text ">-": doctor passed it,
+ * diagnose's forced choice was shown ">-" as the skill's description, and fix refused the skill.
+ *
+ * Scope: the YAML subset SKILL.md files use for top-level string keys — plain scalars (including
+ * indented continuation lines), single- and double-quoted scalars (possibly multi-line), and
+ * literal/folded block scalars with chomping indicators. Nested mappings are skipped. */
+import { readFileSync, existsSync, readdirSync, statSync } from "node:fs";
+import { join } from "node:path";
+const KEY_RE = /^([A-Za-z_][\w-]*)\s*:(?:\s+(.*)|\s*)$/;
+const BLOCK_RE = /^([|>])([+-]?)([1-9]?)([+-]?)\s*(?:#.*)?$/;
+const indentOf = (l) => l.length - l.trimStart().length;
+const isBlank = (l) => l.trim() === "";
+/** Fold YAML flow lines: single newlines become spaces, blank lines become "\n". */
+function foldLines(parts) {
+    let out = "";
+    let pendingBreaks = 0;
+    for (const p of parts) {
+        const t = p.trim();
+        if (!t) {
+            pendingBreaks++;
+            continue;
+        }
+        if (out)
+            out += pendingBreaks ? "\n".repeat(pendingBreaks) : " ";
+        out += t;
+        pendingBreaks = 0;
+    }
+    return out;
+}
+function unescapeDouble(s) {
+    const map = {
+        "0": "\0", a: "\x07", b: "\b", t: "\t", "\t": "\t", n: "\n", v: "\v", f: "\f", r: "\r",
+        e: "\x1b", " ": " ", '"': '"', "/": "/", "\\": "\\", N: "\x85", _: "\xa0",
+    };
+    return s.replace(/\\(x[0-9a-fA-F]{2}|u[0-9a-fA-F]{4}|U[0-9a-fA-F]{8}|.)/g, (m, e) => {
+        if (e.length > 1)
+            return String.fromCodePoint(parseInt(e.slice(1), 16));
+        return map[e] ?? m;
+    });
+}
+/** Parse the value starting on line i (whose text after `key:` is `first`). Returns the value and
+ * the index one past its last line. */
+function parseValue(lines, i, first, end) {
+    const block = BLOCK_RE.exec(first);
+    if (block) {
+        const style = block[1];
+        const chomp = block[2] || block[4] || "";
+        let j = i + 1;
+        let indent = block[3] ? Number(block[3]) : 0;
+        const body = [];
+        while (j < end) {
+            const l = lines[j];
+            if (isBlank(l)) {
+                body.push("");
+                j++;
+                continue;
+            }
+            const ind = indentOf(l);
+            if (!indent)
+                indent = ind;
+            if (ind < indent || ind === 0)
+                break;
+            body.push(l.slice(indent));
+            j++;
+        }
+        let trailing = 0;
+        while (body.length && body[body.length - 1] === "") {
+            body.pop();
+            trailing++;
+        }
+        let text;
+        if (style === "|")
+            text = body.join("\n");
+        else {
+            // folded: lines joined by spaces, blank lines kept as newlines, more-indented lines kept verbatim
+            text = "";
+            let prevMoreIndented = false;
+            for (let b = 0; b < body.length; b++) {
+                const l = body[b];
+                const moreIndented = /^\s/.test(l);
+                if (b === 0) {
+                    text = l;
+                    prevMoreIndented = moreIndented;
+                    continue;
+                }
+                if (l === "") {
+                    text += "\n";
+                    continue;
+                }
+                const prev = body[b - 1];
+                text += prev === "" || moreIndented || prevMoreIndented ? (prev === "" ? "" : "\n") + l : " " + l;
+                prevMoreIndented = moreIndented;
+            }
+        }
+        if (chomp === "+")
+            text += "\n".repeat(trailing + 1);
+        else if (chomp !== "-" && text)
+            text += "\n";
+        // the block ends at the last non-blank line it consumed
+        let next = j;
+        while (next > i + 1 && isBlank(lines[next - 1]))
+            next--;
+        return { value: text, next };
+    }
+    if (first.startsWith('"') || first.startsWith("'")) {
+        const q = first[0];
+        const parts = [];
+        let rest = first.slice(1);
+        let j = i;
+        let terminated = false;
+        for (;;) {
+            // find the closing quote on this line
+            let closeAt = -1;
+            for (let c = 0; c < rest.length; c++) {
+                if (q === '"' && rest[c] === "\\") {
+                    c++;
+                    continue;
+                }
+                if (rest[c] === q) {
+                    if (q === "'" && rest[c + 1] === "'") {
+                        c++;
+                        continue;
+                    }
+                    closeAt = c;
+                    break;
+                }
+            }
+            if (closeAt >= 0) {
+                parts.push(rest.slice(0, closeAt));
+                terminated = true;
+                break;
+            }
+            parts.push(rest);
+            j++;
+            if (j >= end)
+                break; // unterminated — take what we have
+            rest = lines[j];
+        }
+        const folded = foldLines(parts.map((p, idx) => (idx === 0 ? p.trimEnd() : p)));
+        const value = q === '"' ? unescapeDouble(folded) : folded.replace(/''/g, "'");
+        return terminated ? { value, next: j + 1 } : { value, next: end, broken: true };
+    }
+    // plain scalar: this line plus any more-indented continuation lines
+    const parts = [first.replace(/\s+#.*$/, "")];
+    let j = i + 1;
+    while (j < end) {
+        const l = lines[j];
+        if (isBlank(l)) {
+            parts.push("");
+            j++;
+            continue;
+        }
+        if (indentOf(l) === 0)
+            break;
+        parts.push(l.replace(/\s+#.*$/, ""));
+        j++;
+    }
+    let next = j;
+    while (next > i + 1 && isBlank(lines[next - 1]))
+        next--;
+    return { value: foldLines(parts.slice(0, next - i)), next };
+}
+export function parseFrontmatter(text) {
+    const eol = text.includes("\r\n") ? "\r\n" : "\n";
+    const body = text.replace(/^\uFEFF/, "");
+    const lines = [];
+    const offsets = [];
+    const brk = /\r?\n/g;
+    let pos = 0;
+    for (let m = brk.exec(body); m; m = brk.exec(body)) {
+        offsets.push(pos);
+        lines.push(body.slice(pos, m.index));
+        pos = m.index + m[0].length;
+    }
+    offsets.push(pos, body.length);
+    lines.push(body.slice(pos));
+    const empty = { ok: false, fields: {}, spans: [], lines, close: -1, eol, offsets };
+    if (lines[0]?.trim() !== "---")
+        return empty;
+    let close = -1;
+    for (let i = 1; i < lines.length; i++) {
+        const t = lines[i].trimEnd();
+        if (t === "---" || t === "...") {
+            close = i;
+            break;
+        }
+    }
+    if (close < 0)
+        return empty;
+    const fields = {};
+    const spans = [];
+    let i = 1;
+    while (i < close) {
+        const line = lines[i];
+        const m = indentOf(line) === 0 ? KEY_RE.exec(line) : null;
+        if (!m) {
+            i++;
+            continue;
+        }
+        const { value, next, broken } = parseValue(lines, i, (m[2] ?? "").trim(), close);
+        const key = m[1];
+        if (!(key in fields))
+            fields[key] = value;
+        spans.push({ key, value, start: i, end: next, ...(broken ? { broken } : {}) });
+        i = Math.max(next, i + 1);
+    }
+    return { ok: true, fields, spans, lines, close, eol, offsets };
+}
+/** One-line form of a description, as a router would read it. */
+export function normalizeDescription(s) {
+    return s.replace(/\s+/g, " ").trim();
+}
+/** Format a string as a YAML scalar for a single `key: value` line: plain when unambiguous,
+ * otherwise double-quoted (JSON string syntax is valid YAML). */
+export function yamlScalar(s) {
+    const plainSafe = s.length > 0 && s === s.trim() && !/[\n\r\t]/.test(s) &&
+        !/^[-?:,[\]{}#&*!|>'"%@`]/.test(s) && !/:(\s|$)/.test(s) && !/\s#/.test(s) &&
+        !/^(true|false|yes|no|on|off|null|~|[-+]?(\d[\d_]*)?(\.\d+)?([eE][-+]?\d+)?|\.inf|\.nan)$/i.test(s);
+    return plainSafe ? s : JSON.stringify(s);
+}
+/** Return `text` with its frontmatter `description` replaced by `desc` (written on one line).
+ * Every other byte of the file — including each line's own line ending — is preserved. Throws if
+ * there is no frontmatter block or the existing description is an unterminated quoted string. */
+export function replaceDescription(text, desc) {
+    const fm = parseFrontmatter(text);
+    if (!fm.ok)
+        throw new Error("SKILL.md has no `---` frontmatter block");
+    const bom = text.startsWith("\uFEFF") ? "\uFEFF" : "";
+    const body = text.slice(bom.length);
+    const line = `description: ${yamlScalar(normalizeDescription(desc))}`;
+    const lineEnd = (i) => fm.offsets[i] + fm.lines[i].length;
+    const span = fm.spans.find((s) => s.key === "description");
+    if (span?.broken) {
+        throw new Error("the SKILL.md description has an unterminated quote — fix the frontmatter first");
+    }
+    if (span) {
+        // swap the entry's text; the last replaced line keeps its own terminator
+        return bom + body.slice(0, fm.offsets[span.start]) + line + body.slice(lineEnd(span.end - 1));
+    }
+    const nameSpan = fm.spans.find((s) => s.key === "name");
+    const at = nameSpan ? nameSpan.end : fm.close;
+    const term = body.slice(lineEnd(at - 1), fm.offsets[at]) || fm.eol;
+    return bom + body.slice(0, fm.offsets[at]) + line + term + body.slice(fm.offsets[at]);
+}
+export function readSkillFile(file, folder) {
+    const fm = parseFrontmatter(readFileSync(file, "utf8"));
+    const desc = fm.fields["description"];
+    const fmName = fm.fields["name"];
+    return {
+        name: folder,
+        description: desc === undefined ? "" : normalizeDescription(desc),
+        ...(fmName !== undefined ? { frontmatterName: normalizeDescription(fmName) } : {}),
+    };
+}
+export function skillsRoot(cwd) {
+    return join(cwd, ".claude", "skills");
+}
+export function skillFile(cwd, name) {
+    return join(skillsRoot(cwd), name, "SKILL.md");
+}
+/** Every folder under <cwd>/.claude/skills/, following symlinks (a symlinked skill folder is a
+ * skill like any other), sorted. Throws if there is no skills dir. */
+export function skillFolders(cwd) {
+    const root = skillsRoot(cwd);
+    if (!existsSync(root))
+        throw new Error(`no .claude/skills/ directory under ${cwd}`);
+    const isDir = (p) => { try {
+        return statSync(p).isDirectory();
+    }
+    catch {
+        return false;
+    } };
+    return readdirSync(root).filter((n) => isDir(join(root, n))).sort((a, b) => a.localeCompare(b));
+}
+/** Every skill folder under <cwd>/.claude/skills/ that has a SKILL.md, sorted by name. */
+export function listSkills(cwd) {
+    return skillFolders(cwd)
+        .filter((n) => existsSync(skillFile(cwd, n)))
+        .map((n) => readSkillFile(skillFile(cwd, n), n));
+}

package/dist/stats.d.ts

@@ -1,7 +1,7 @@
 /**
- * Statistical core (TS port of stats_ref/stats.py — the validated reference impl).
- * Skill activation is Bernoulli with unknown p; we estimate from k runs and never
- * report a bare point. See ../../STATS.md.
+ * Statistical core (TS port of the validated Python reference implementation from the research
+ * phase). Skill activation is Bernoulli with unknown p; we estimate from k runs and never report a
+ * bare point. The README's "How verdicts are decided" section explains the sequential verdicts.
  */
 export interface Interval {
     lo: number;
@@ -19,8 +19,30 @@ export interface Reliability {
     conf: number;
 }
 export declare function reliability(x: number, n: number, conf?: number): Reliability;
-/** Sequential stop rule: is p reliably above/below threshold yet, or keep sampling? */
-export declare function sequentialDecision(x: number, n: number, threshold: number, conf?: number): Decision;
+export interface SequentialPlan {
+    k: number;
+    threshold: number;
+    /** per-direction error bound (1 - conf) */
+    alpha: number;
+    /** nominal one-sided level used at interim looks */
+    interimAlpha: number;
+    /** declare "above" at n valid probes when hits >= upper[n] (n+1 = can't) */
+    upper: number[];
+    /** declare "below" at n valid probes when hits <= lower[n] (-1 = can't) */
+    lower: number[];
+    /** exact P(declare above | p = threshold) and P(declare below | p = threshold); both <= alpha */
+    errAbove: number;
+    errBelow: number;
+}
+/** The verdict plan for up to k valid probes against `threshold` at confidence `conf`. Memoized. */
+export declare function sequentialPlan(k: number, threshold: number, conf?: number): SequentialPlan;
+/** Verdict after n valid probes with x hits. Valid at ANY n <= k: stopping early (decided) or
+ * because infra errors exhausted the attempt budget both keep the error guarantee. */
+export declare function planDecision(plan: SequentialPlan, x: number, n: number): Decision;
+/** Convenience: the verdict for x/n under the plan for max k (defaults to k = n, a fixed-n test). */
+export declare function sequentialDecision(x: number, n: number, threshold: number, conf?: number, k?: number): Decision;
+/** Smallest k at which an all-success run can certify reliability > threshold. */
+export declare function kToCertify(threshold: number, conf?: number): number;
 /** Two-sided Fisher's exact test on [[a,b],[c,d]] — exact, for small samples. */
 export declare function fisherExact2x2(a: number, b: number, c: number, d: number): number;
 export interface ImprovementResult {

package/dist/stats.js

@@ -1,7 +1,7 @@
 /**
- * Statistical core (TS port of stats_ref/stats.py — the validated reference impl).
- * Skill activation is Bernoulli with unknown p; we estimate from k runs and never
- * report a bare point. See ../../STATS.md.
+ * Statistical core (TS port of the validated Python reference implementation from the research
+ * phase). Skill activation is Bernoulli with unknown p; we estimate from k runs and never report a
+ * bare point. The README's "How verdicts are decided" section explains the sequential verdicts.
  */
 const Z = {
     "0.9": 1.6448536269514722,
@@ -26,15 +26,139 @@ export function reliability(x, n, conf = 0.95) {
     const { lo, hi } = wilsonInterval(x, n, conf);
     return { pHat: n ? x / n : 0, ciLow: lo, ciHigh: hi, k: n, conf };
 }
-/** Sequential stop rule: is p reliably above/below threshold yet, or keep sampling? */
-export function sequentialDecision(x, n, threshold, conf = 0.95) {
-    const { lo, hi } = wilsonInterval(x, n, conf);
-    if (lo > threshold)
+// --- exact group-sequential verdicts ---
+// Checking a CI after every probe and stopping as soon as it clears the threshold ("peeking")
+// inflates the error rate far past the nominal level: measured on the old Wilson rule at k=35, a
+// skill sitting exactly at the threshold was wrongly passed 9% and wrongly failed 18% of the time
+// at a nominal 2.5% per side. A verdict here comes from a plan whose boundaries are calibrated by
+// exact dynamic programming over every probe path, so the guarantee holds no matter when it stops:
+//
+//   P(verdict "above" | true reliability <= threshold) <= 1 - conf
+//   P(verdict "below" | true reliability >= threshold) <= 1 - conf
+//
+// (one-sided per direction; the worst case is p == threshold because the boundaries are monotone).
+// Design is Haybittle-Peto style: interim looks use a strict nominal level so only clear-cut cases
+// stop early, and the final look spends whatever error budget is left.
+/** interim looks never decide on fewer valid probes than this */
+const MIN_LOOK = 3;
+function binomPmfRow(n, p) {
+    const row = new Array(n + 1).fill(0);
+    if (p <= 0) {
+        row[0] = 1;
+        return row;
+    }
+    if (p >= 1) {
+        row[n] = 1;
+        return row;
+    }
+    const lp = Math.log(p), lq = Math.log(1 - p), ln = lgamma(n + 1);
+    for (let x = 0; x <= n; x++) {
+        row[x] = Math.exp(ln - lgamma(x + 1) - lgamma(n - x + 1) + x * lp + (n - x) * lq);
+    }
+    return row;
+}
+function buildPlan(k, theta, alpha, interim) {
+    const upper = Array.from({ length: k + 1 }, (_, n) => n + 1);
+    const lower = new Array(k + 1).fill(-1);
+    if (interim > 0) {
+        for (let n = MIN_LOOK; n < k; n++) {
+            const pmf = binomPmfRow(n, theta);
+            let acc = 0;
+            for (let x = n; x >= 0 && acc + pmf[x] <= interim; x--) {
+                acc += pmf[x];
+                upper[n] = x;
+            }
+            acc = 0;
+            for (let x = 0; x <= n && acc + pmf[x] <= interim; x++) {
+                acc += pmf[x];
+                lower[n] = x;
+            }
+        }
+    }
+    // dist[x] = P(X_n = x AND no boundary crossed yet | p = theta)
+    let dist = [1];
+    let errAbove = 0, errBelow = 0;
+    for (let n = 1; n <= k; n++) {
+        const next = new Array(n + 1).fill(0);
+        for (let x = 0; x < n; x++) {
+            next[x] += dist[x] * (1 - theta);
+            next[x + 1] += dist[x] * theta;
+        }
+        if (n < k) {
+            for (let x = 0; x <= n; x++) {
+                if (x >= upper[n]) {
+                    errAbove += next[x];
+                    next[x] = 0;
+                }
+                else if (x <= lower[n]) {
+                    errBelow += next[x];
+                    next[x] = 0;
+                }
+            }
+        }
+        dist = next;
+    }
+    // interim looks may use at most half the budget; the final look must keep real power
+    if (interim > 0 && (errAbove > alpha / 2 || errBelow > alpha / 2))
+        return null;
+    const tol = alpha * 1e-12;
+    for (let x = k; x >= 0 && errAbove + dist[x] <= alpha + tol; x--) {
+        errAbove += dist[x];
+        upper[k] = x;
+    }
+    for (let x = 0; x < upper[k] && errBelow + dist[x] <= alpha + tol; x++) {
+        errBelow += dist[x];
+        lower[k] = x;
+    }
+    return { k, threshold: theta, alpha, interimAlpha: interim, upper, lower, errAbove, errBelow };
+}
+const planCache = new Map();
+/** The verdict plan for up to k valid probes against `threshold` at confidence `conf`. Memoized. */
+export function sequentialPlan(k, threshold, conf = 0.95) {
+    if (!Number.isInteger(k) || k < 1)
+        throw new Error(`k must be a positive integer (got ${k})`);
+    if (!(threshold > 0 && threshold < 1))
+        throw new Error(`threshold must be in (0,1) (got ${threshold})`);
+    if (!(conf > 0.5 && conf < 1))
+        throw new Error(`conf must be in (0.5,1) (got ${conf})`);
+    const key = `${k}|${threshold}|${conf}`;
+    const hit = planCache.get(key);
+    if (hit)
+        return hit;
+    const alpha = 1 - conf;
+    let plan = null;
+    for (let interim = alpha / 10; !plan && interim > 1e-9; interim /= 2) {
+        plan = buildPlan(k, threshold, alpha, interim);
+    }
+    plan ??= buildPlan(k, threshold, alpha, 0); // no early stopping at all
+    planCache.set(key, plan);
+    return plan;
+}
+/** Verdict after n valid probes with x hits. Valid at ANY n <= k: stopping early (decided) or
+ * because infra errors exhausted the attempt budget both keep the error guarantee. */
+export function planDecision(plan, x, n) {
+    if (n < 1)
+        return "undecided";
+    const i = Math.min(n, plan.k);
+    if (x >= plan.upper[i])
         return "above";
-    if (hi < threshold)
+    if (x <= plan.lower[i])
         return "below";
     return "undecided";
 }
+/** Convenience: the verdict for x/n under the plan for max k (defaults to k = n, a fixed-n test). */
+export function sequentialDecision(x, n, threshold, conf = 0.95, k = n) {
+    return planDecision(sequentialPlan(Math.max(k, 1), threshold, conf), x, n);
+}
+/** Smallest k at which an all-success run can certify reliability > threshold. */
+export function kToCertify(threshold, conf = 0.95) {
+    const est = Math.ceil(Math.log(1 - conf) / Math.log(threshold));
+    for (let k = Math.max(1, est - 3); k <= Math.max(est + 50, 10); k++) {
+        if (sequentialPlan(k, threshold, conf).upper[k] <= k)
+            return k;
+    }
+    return est;
+}
 // --- lgamma (Lanczos) ---
 const LANCZOS = [
     0.99999999999980993, 676.5203681218851, -1259.1392167224028,

package/dist/types.d.ts

@@ -1,4 +1,13 @@
 /** Core types. The normalized ProbeResult is the seam every runtime adapter maps to. */
+/** What the runtime reported about its own session — the ground truth of what was co-loaded. */
+export interface ProbeEnv {
+    /** every skill the runtime loaded for this probe (library + runtime built-ins + anything else) */
+    loadedSkills?: string[];
+    /** runtime CLI version, e.g. "2.1.305" */
+    runtimeVersion?: string;
+    /** model the runtime actually used (resolves "(runtime default)") */
+    model?: string;
+}
 export interface ProbeResult {
     /** "ok" = the runtime ran and we trust the reading; "error" = infrastructure failure
      * (timeout, auth, crash, empty/unparseable output) — must NOT count as a behavioral outcome. */
@@ -11,6 +20,8 @@ export interface ProbeResult {
     cost: number;
     /** failure reason when status==="error" */
     error?: string;
+    /** session facts reported by the runtime, when it exposes them */
+    env?: ProbeEnv;
 }
 export interface ProbeOpts {
     cwd: string;
@@ -18,9 +29,18 @@ export interface ProbeOpts {
     /** model id to pin (runtime-specific format, e.g. "claude-sonnet-4-6" or
      * "anthropic/claude-haiku-4-5"); omitted = the runtime's default. */
     model?: string;
+    /** agent turns the runtime may take before we read the outcome (default 1) */
+    maxTurns?: number;
+    /** also load the user's personal runtime config (user settings, plugins, MCP servers). Off by
+     * default: personal config differs per machine, so results wouldn't reproduce in CI. */
+    userConfig?: boolean;
 }
 export interface RuntimeAdapter {
     readonly name: string;
+    /** honours ProbeOpts.userConfig=false by keeping the user's personal config out of probes */
+    readonly isolatesUserConfig?: boolean;
+    /** honours ProbeOpts.maxTurns */
+    readonly supportsMaxTurns?: boolean;
     /** Run one prompt with the library at cwd co-loaded; report what fired. */
     probe(prompt: string, opts: ProbeOpts): Promise<ProbeResult>;
 }
@@ -39,7 +59,17 @@ export interface Config {
     k: number;
     /** minimum acceptable trigger reliability (0..1) */
     threshold: number;
-    /** confidence level for intervals */
+    /** confidence level for intervals and verdicts */
     conf: number;
     cases: EvalCase[];
+    /** agent turns per probe (default 1) */
+    maxTurns?: number;
+    /** load the user's personal runtime config into probes (default false — hermetic) */
+    userConfig?: boolean;
+    /** per-probe timeout in ms (default 120000) */
+    timeoutMs?: number;
+    /** fail a case when ANY other skill fires, even if reliability clears the threshold (default false) */
+    failOnTheft?: boolean;
 }
+/** The probe settings a Config implies (shared by every command). */
+export declare function probeSettings(cfg: Config): Omit<ProbeOpts, "cwd">;

package/dist/types.js

@@ -1,2 +1,10 @@
 /** Core types. The normalized ProbeResult is the seam every runtime adapter maps to. */
-export {};
+/** The probe settings a Config implies (shared by every command). */
+export function probeSettings(cfg) {
+    return {
+        ...(cfg.model ? { model: cfg.model } : {}),
+        ...(cfg.maxTurns !== undefined ? { maxTurns: cfg.maxTurns } : {}),
+        ...(cfg.userConfig ? { userConfig: true } : {}),
+        ...(cfg.timeoutMs !== undefined ? { timeoutMs: cfg.timeoutMs } : {}),
+    };
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "skill-probe",
-  "version": "0.8.0",
+  "version": "0.10.0",
   "description": "Runtime-agnostic, co-loaded-aware auditor for AI agent skill libraries: measures which skill actually fires, with statistical confidence.",
   "type": "module",
   "bin": {