skilldrop-cli 0.16.35 → 0.16.37

package/packs/ai-engineering/skills/agent-incident-response/scripts/find_secret_exposure.py

@@ -0,0 +1,239 @@
+#!/usr/bin/env python3
+"""Find where a leaked key appears in a repository's git history and in log files.
+
+Read-only. Never prints the key: every match is shown masked (first 8 characters,
+an ellipsis, the last 4, and the length). Identify the key by a prefix you already
+know (--prefix, optionally with --suffix for the last characters) or by piping the
+full key on stdin (--key-stdin) so it never lands in shell history.
+
+    python3 find_secret_exposure.py --prefix sk-live-AbCdEf12 --repo . --log ci.txt
+    printf '%s' "$LEAKED" | python3 find_secret_exposure.py --key-stdin --repo .
+
+Reports: commits that added or removed it (author and commit time, UTC), whether the
+current tree and working copy still hold it, which branches and tags still reach the
+adding commits, commits reachable only from the local reflog, and every log line that
+holds it with the line's own timestamp. Writes only the -o path when given.
+"""
+import argparse
+import datetime as dt
+import os
+import re
+import subprocess
+import sys
+
+TOKEN_CHARS = r"[A-Za-z0-9_\-+/=.]"
+STAMP = re.compile(
+    r"(\d{4}-\d{2}-\d{2})[T ](\d{2}:\d{2}:\d{2})(?:\.\d+)?(Z|[+-]\d{2}:?\d{2})?"
+)
+MIN_PREFIX = 10
+
+
+def mask(token):
+    if len(token) <= 12:
+        return token[:4] + "…" + " (len %d)" % len(token)
+    return "%s…%s (len %d)" % (token[:8], token[-4:], len(token))
+
+
+class Needle:
+    def __init__(self, prefix=None, suffix=None, full=None):
+        self.full = full
+        self.prefix = full[:12] if full else prefix
+        self.suffix = suffix
+        self.rx = re.compile(re.escape(self.prefix) + TOKEN_CHARS + "*")
+        self.seen = set()
+
+    def tokens(self, text):
+        """Masked tokens in text that match the key."""
+        out = []
+        for m in self.rx.finditer(text):
+            tok = m.group(0).rstrip(".=")  # trailing punctuation from prose or logs
+            if self.full and self.full not in m.group(0):
+                continue
+            if self.suffix and not tok.endswith(self.suffix):
+                continue
+            out.append(mask(self.full if self.full else tok))
+        self.seen.update(out)
+        return out
+
+
+def utc(epoch):
+    return dt.datetime.fromtimestamp(int(epoch), dt.timezone.utc).strftime("%Y-%m-%d %H:%M:%SZ")
+
+
+def line_stamp(line):
+    m = STAMP.search(line)
+    if not m:
+        return "no timestamp on the line"
+    day, clock, tz = m.groups()
+    if not tz:
+        return "%s %s (no timezone on the line)" % (day, clock)
+    tz = "+00:00" if tz == "Z" else (tz if ":" in tz else tz[:3] + ":" + tz[3:])
+    try:
+        when = dt.datetime.fromisoformat("%sT%s%s" % (day, clock, tz))
+    except ValueError:
+        return "%s %s%s (unparsed)" % (day, clock, tz)
+    return when.astimezone(dt.timezone.utc).strftime("%Y-%m-%d %H:%M:%SZ")
+
+
+def git(repo, *args):
+    r = subprocess.run(["git", "-C", repo] + list(args), capture_output=True)
+    return r.returncode, r.stdout.decode("utf-8", "replace"), r.stderr.decode("utf-8", "replace")
+
+
+def scan_repo(repo, needle):
+    rc, _, err = git(repo, "rev-parse", "--git-dir")
+    if rc != 0:
+        return None, "not a git repository: %s (%s)" % (repo, err.strip())
+    rc, out, err = git(
+        repo, "log", "--all", "--reflog", "--no-color", "--no-ext-diff", "-p",
+        "-S" + needle.prefix, "--format=%x00COMMIT %H %at %ct",
+    )
+    if rc != 0:
+        return None, "git log failed: %s" % err.strip()
+    commits, cur, path = [], None, None
+    for line in out.splitlines():
+        if line.startswith("\x00COMMIT "):
+            sha, at, ct = line.split()[1:4]
+            cur = {"sha": sha, "authored": utc(at), "committed": utc(ct), "added": [], "removed": []}
+            commits.append(cur)
+        elif cur is None:
+            continue
+        elif line.startswith("diff --git "):
+            path = line.split(" b/", 1)[-1]
+        elif line.startswith("+") and not line.startswith("+++"):
+            if needle.tokens(line):
+                cur["added"].append(path)
+        elif line.startswith("-") and not line.startswith("---"):
+            if needle.tokens(line):
+                cur["removed"].append(path)
+    commits = [c for c in commits if c["added"] or c["removed"]]
+    for c in commits:
+        c["added"], c["removed"] = sorted(set(c["added"])), sorted(set(c["removed"]))
+        if c["added"]:
+            _, b, _ = git(repo, "branch", "-a", "--contains", c["sha"])
+            _, t, _ = git(repo, "tag", "--contains", c["sha"])
+            c["refs"] = [x.strip("* ").strip() for x in b.splitlines() if x.strip()]
+            c["refs"] += ["tag " + x.strip() for x in t.splitlines() if x.strip()]
+    present = {}
+    for label, args in (("HEAD", ["grep", "-n", "-I", "-F", "-e", needle.prefix, "HEAD", "--"]),
+                        ("working copy", ["grep", "-n", "-I", "-F", "--untracked", "-e", needle.prefix])):
+        _, g, _ = git(repo, *args)
+        hits = []
+        for gl in g.splitlines():
+            parts = gl.split(":", 3 if label == "HEAD" else 2)
+            loc = ":".join(parts[1:3]) if label == "HEAD" else ":".join(parts[0:2])
+            if needle.tokens(gl):
+                hits.append(loc)
+        present[label] = hits
+    return {"commits": commits, "present": present}, None
+
+
+def scan_log(path, needle):
+    hits = []
+    with open(path, "r", encoding="utf-8", errors="replace") as fh:
+        for n, line in enumerate(fh, 1):
+            toks = needle.tokens(line)
+            if toks:
+                hits.append((n, line_stamp(line), toks[0]))
+    return hits
+
+
+def report(args, needle, repo_res, repo_err, logs):
+    o = ["# Where the key appears", ""]
+    if len(needle.seen) > 1:
+        o += ["%d different keys match this prefix: %s. Pass --suffix with the leaked key's last "
+              "characters to keep only that one." % (len(needle.seen), ", ".join(sorted(needle.seen))), ""]
+    if args.repo:
+        o.append("## Git history: %s" % args.repo)
+        if repo_err:
+            o += ["", "Not scanned: " + repo_err]
+        else:
+            cs = repo_res["commits"]
+            adds = [c for c in cs if c["added"]]
+            if not cs:
+                o += ["", "No commit adds or removes it (searched every branch, tag, stash and the local reflog)."]
+            if adds:
+                first = min(adds, key=lambda c: c["authored"])
+                o += ["", "First commit that adds it: `%s`, authored %s, committed %s. Commit time is "
+                      "when it was committed locally; when it reached a remote is in the host's audit or push log."
+                      % (first["sha"][:12], first["authored"], first["committed"])]
+            for c in cs:
+                o.append("")
+                o.append("- `%s` authored %s, committed %s" % (c["sha"][:12], c["authored"], c["committed"]))
+                if c["added"]:
+                    o.append("  - adds it in: %s" % ", ".join(c["added"]))
+                    o.append("  - still reachable from: %s" % (", ".join(c["refs"]) if c["refs"]
+                             else "no branch or tag; only the local reflog keeps it"))
+                if c["removed"]:
+                    o.append("  - removes it from: %s (the earlier commits still hold it)" % ", ".join(c["removed"]))
+            for label, hits in repo_res["present"].items():
+                o.append("")
+                o.append("In %s now: %s" % (label, ", ".join(hits) if hits else "not present"))
+        o.append("")
+    for path, hits, err in logs:
+        o.append("## Log: %s" % path)
+        o.append("")
+        if err:
+            o.append("Not scanned: " + err)
+        elif not hits:
+            o.append("Not found.")
+        else:
+            for n, stamp, tok in hits:
+                o.append("- line %d, %s, %s" % (n, stamp, tok))
+        o.append("")
+    o += ["## Not searched", "",
+          "Forks, other clones, pull-request refs that were never fetched, CI caches and artifacts, "
+          "the hosting provider's own copies, chat and ticket systems. A clean result here does not "
+          "mean the key is safe: revoke it regardless.", ""]
+    return "\n".join(o)
+
+
+def main(argv=None):
+    p = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
+    who = p.add_mutually_exclusive_group(required=True)
+    who.add_argument("--prefix", help="the first characters of the key (at least %d)" % MIN_PREFIX)
+    who.add_argument("--key-stdin", action="store_true", help="read the full key from stdin")
+    p.add_argument("--suffix", help="the last characters of the key, to tell it apart from keys with the same prefix")
+    p.add_argument("--repo", help="path to a git repository to search")
+    p.add_argument("--log", action="append", default=[], help="a log file to search (repeatable)")
+    p.add_argument("-o", "--out", help="write the report here instead of stdout")
+    a = p.parse_args(argv)
+    if a.key_stdin:
+        full = sys.stdin.read().strip()
+        if len(full) < MIN_PREFIX:
+            p.error("the key on stdin is shorter than %d characters" % MIN_PREFIX)
+        needle = Needle(full=full)
+    else:
+        if len(a.prefix) < MIN_PREFIX:
+            p.error("--prefix needs at least %d characters: a short prefix such as a provider's "
+                    "key-type marker matches every key of that type" % MIN_PREFIX)
+        needle = Needle(prefix=a.prefix, suffix=a.suffix)
+    if not a.repo and not a.log:
+        p.error("give --repo, --log, or both")
+    repo_res, repo_err = (None, None)
+    if a.repo:
+        if not os.path.isdir(a.repo):
+            repo_err = "no such directory: %s" % a.repo
+        else:
+            repo_res, repo_err = scan_repo(a.repo, needle)
+    logs = []
+    for path in a.log:
+        if not os.path.isfile(path):
+            logs.append((path, [], "no such file"))
+            continue
+        try:
+            logs.append((path, scan_log(path, needle), None))
+        except OSError as e:
+            logs.append((path, [], str(e)))
+    text = report(a, needle, repo_res, repo_err, logs)
+    if a.out:
+        with open(a.out, "w", encoding="utf-8") as fh:
+            fh.write(text)
+        print("wrote %s" % a.out)
+    else:
+        print(text)
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

package/packs/ai-engineering/skills/agent-incident-response/scripts/usage_split.py

@@ -0,0 +1,194 @@
+#!/usr/bin/env python3
+"""Split a provider usage or billing export (CSV) by who spent it.
+
+Read-only. Sums cost exactly (decimal, not float) per group of columns, so a bill
+is split from the export rather than estimated. Groups by the key, model and
+user-agent columns it finds (override with --by). Each group shows its total,
+its share of the whole, its row and request counts, and the time of its first and
+last row as written in the export.
+
+    python3 usage_split.py usage.csv
+    python3 usage_split.py usage.csv --since 2026-10-19T20:00 --expect model=model-medium
+    python3 usage_split.py usage.csv --by api_key_name,user_agent -o split.txt
+    cat usage.csv | python3 usage_split.py -
+
+--expect col=value (repeatable) marks every group whose value in that column
+differs as UNEXPECTED: traffic the known job does not explain. --since/--until
+compare the time column as text, so give them in the export's own format.
+Writes only the -o path when given.
+"""
+import argparse
+import csv
+import io
+import sys
+from decimal import Decimal, InvalidOperation
+
+COST_HINTS = ("cost_usd", "cost", "amount_usd", "amount", "spend", "usd", "price")
+TIME_HINTS = ("date_hour_utc", "timestamp", "time", "hour", "date", "start_time", "period")
+REQ_HINTS = ("requests", "request_count", "num_requests", "calls")
+GROUP_HINTS = (
+    ("api_key_name", "api_key", "key_name", "key_id", "key"),
+    ("model", "model_name", "model_id"),
+    ("user_agent", "useragent", "ua", "client"),
+)
+
+
+def find_col(headers, hints, contains=False):
+    low = {h.lower().strip(): h for h in headers}
+    for hint in hints:
+        if hint in low:
+            return low[hint]
+    if contains:
+        for hint in hints:
+            for key, orig in low.items():
+                if hint in key:
+                    return orig
+    return None
+
+
+def to_decimal(text):
+    cleaned = (text or "").strip().replace("$", "").replace(",", "")
+    if cleaned == "":
+        return Decimal("0")
+    return Decimal(cleaned)
+
+
+def money(value):
+    return "${:,.2f}".format(value)
+
+
+def main(argv=None):
+    ap = argparse.ArgumentParser(description=__doc__.split("\n")[0])
+    ap.add_argument("export", help="CSV usage export path, or - for stdin")
+    ap.add_argument("--by", help="comma-separated columns to group by (default: key, model, user agent)")
+    ap.add_argument("--cost", help="cost column name (default: auto-detect)")
+    ap.add_argument("--time", help="time column name (default: auto-detect)")
+    ap.add_argument("--since", help="keep rows whose time is >= this text")
+    ap.add_argument("--until", help="keep rows whose time is < this text")
+    ap.add_argument("--expect", action="append", default=[], metavar="COL=VALUE",
+                    help="mark groups whose COL differs from VALUE as UNEXPECTED")
+    ap.add_argument("-o", "--output", help="write the report to this path instead of stdout")
+    args = ap.parse_args(argv)
+
+    if args.export == "-":
+        text = sys.stdin.read()
+    else:
+        try:
+            with open(args.export, encoding="utf-8", newline="") as fh:
+                text = fh.read()
+        except OSError as exc:
+            print("error: cannot read {}: {}".format(args.export, exc), file=sys.stderr)
+            return 2
+    text = text.lstrip("")
+    reader = csv.DictReader(io.StringIO(text))
+    headers = reader.fieldnames or []
+    if not headers:
+        print("error: the export has no header row", file=sys.stderr)
+        return 2
+
+    cost_col = args.cost or find_col(headers, COST_HINTS, contains=True)
+    time_col = args.time or find_col(headers, TIME_HINTS, contains=True)
+    req_col = find_col(headers, REQ_HINTS)
+    if not cost_col or cost_col not in headers:
+        print("error: no cost column found; columns are: {}; pass --cost".format(", ".join(headers)),
+              file=sys.stderr)
+        return 2
+    if args.by:
+        group_cols = [c.strip() for c in args.by.split(",") if c.strip()]
+    else:
+        group_cols = [c for c in (find_col(headers, h) for h in GROUP_HINTS) if c]
+    missing = [c for c in group_cols if c not in headers]
+    if missing or not group_cols:
+        print("error: group columns not in the export: {}; columns are: {}".format(
+            ", ".join(missing) or "(none found)", ", ".join(headers)), file=sys.stderr)
+        return 2
+    if (args.since or args.until) and not time_col:
+        print("error: --since/--until need a time column; pass --time", file=sys.stderr)
+        return 2
+
+    expects = []
+    for item in args.expect:
+        if "=" not in item:
+            print("error: --expect takes COL=VALUE, got {!r}".format(item), file=sys.stderr)
+            return 2
+        col, val = item.split("=", 1)
+        if col not in group_cols:
+            print("error: --expect column {!r} must be one of the group columns: {}".format(
+                col, ", ".join(group_cols)), file=sys.stderr)
+            return 2
+        expects.append((col, val))
+
+    groups = {}
+    total = Decimal("0")
+    kept = 0
+    bad = []
+    for line_no, row in enumerate(reader, start=2):
+        when = (row.get(time_col) or "").strip() if time_col else ""
+        if args.since and when < args.since:
+            continue
+        if args.until and when >= args.until:
+            continue
+        try:
+            cost = to_decimal(row.get(cost_col))
+        except InvalidOperation:
+            bad.append(line_no)
+            continue
+        key = tuple((row.get(c) or "").strip() for c in group_cols)
+        g = groups.setdefault(key, {"cost": Decimal("0"), "rows": 0, "requests": 0,
+                                    "first": None, "last": None, "times": []})
+        g["cost"] += cost
+        g["rows"] += 1
+        if req_col:
+            try:
+                g["requests"] += int(float((row.get(req_col) or "0").replace(",", "")))
+            except ValueError:
+                pass
+        if when:
+            g["times"].append(when)
+            g["first"] = when if g["first"] is None or when < g["first"] else g["first"]
+            g["last"] = when if g["last"] is None or when > g["last"] else g["last"]
+        total += cost
+        kept += 1
+
+    out = []
+    window = ""
+    if args.since or args.until:
+        window = " ({} to {})".format(args.since or "start", args.until or "end")
+    out.append("Rows: {}{}   Total: {}".format(kept, window, money(total)))
+    out.append("Grouped by: {}   Cost column: {}{}".format(
+        ", ".join(group_cols), cost_col, "   Time column: " + time_col if time_col else ""))
+    out.append("")
+    for key, g in sorted(groups.items(), key=lambda kv: kv[1]["cost"], reverse=True):
+        share = (g["cost"] / total * 100) if total else Decimal("0")
+        label = " | ".join("{}={}".format(c, v) for c, v in zip(group_cols, key))
+        flag = ""
+        if expects:
+            off = [c for c, v in expects if key[group_cols.index(c)] != v]
+            if off:
+                flag = "  UNEXPECTED ({} differs)".format(", ".join(off))
+        out.append("{}  {}  ({:.1f}%){}".format(money(g["cost"]), label, share, flag))
+        detail = "    rows={}".format(g["rows"])
+        if req_col:
+            detail += " requests={:,}".format(g["requests"])
+        if g["first"]:
+            detail += " first={} last={}".format(g["first"], g["last"])
+        out.append(detail)
+        if g["times"] and len(g["times"]) <= 12:
+            out.append("    times: " + ", ".join(sorted(g["times"])))
+    if bad:
+        out.append("")
+        out.append("Skipped rows with an unreadable cost (CSV line numbers): " +
+                   ", ".join(str(n) for n in bad))
+    report = "\n".join(out) + "\n"
+
+    if args.output:
+        with open(args.output, "w", encoding="utf-8") as fh:
+            fh.write(report)
+        print("wrote {}".format(args.output))
+    else:
+        sys.stdout.write(report)
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

package/packs/ai-engineering/skills/judge-audit/SKILL.md

@@ -0,0 +1,157 @@
+---
+name: judge-audit
+description: Audit whether an LLM-as-judge, or a panel of judges, can be trusted for the decision its scores drive, before anyone acts on them — above all whether the judge errs differently on the variants being compared (prompt v1 vs v2, model A vs B, short vs long inputs), plus chance-corrected and weighted agreement with expert labels, precision and recall on a rare class, each run reported separately with the items that flipped, position and length bias, and confidence intervals, checked against the user's own gate or criterion and ending in a direct answer (ship or not, which judge to use). Use when the user asks "can we trust our LLM judge", "is our grader reliable enough to gate releases", "the grader says v2 is better, can we ship it", "which is the cheapest judge we can switch to", "our two judges agree 90%, is that good", "can we turn on auto-remove", "check our judge for position bias", or has a file of judge scores and wants to know whether to act on them.
+---
+
+# judge-audit
+
+Decides whether a judge's scores are good enough for one named decision, from the judgements
+themselves, and answers the question the user asked. A grader can match an expert well overall
+and still over-score the very variant it is being used to pick. This skill audits judgements that
+already exist; designing the eval set and grading plan is `llm-eval-harness`, generating eval
+cases for a skill is `eval-harness-generator`, and measuring AI tool usage is `ai-usage-report`.
+
+## How to respond
+
+**Four rules come before the steps and outrank them:**
+
+- **Answer what was asked, first.** Open with the answer, the decision or the artifact, in plain words. Scores, matrices, frameworks and tags come after it, and anything that doesn't change the answer is cut.
+- **Use only what you were given.** Don't add facts, names, numbers, incidents, history, steps or sections the input doesn't contain. What you need and don't have is left out of the artifact and listed once at the end under "To confirm".
+- **Deliver from what you have.** When the request gives you something to work on, state your assumptions in a line and produce the result. When it gives you nothing to work on, ask for it in one or two plain sentences and say what you will do once you have it.
+- **Write for someone who has never heard of this skill.** No skill names, no paths or scripts from this folder, no internal terms, and nothing about how the run was set up. A next step is one plain sentence at the end that describes the work.
+- **For this skill:** The pass mark is the user's, never yours. Judge the numbers against the gate or criterion the user or their files state; when there is none, say what the numbers do to their decision. A threshold the input doesn't contain ("kappa ≥ 0.6", "n ≥ 100") never decides the answer.
+
+1. **Pin the decision, the user's criterion, and the comparison.** Find the decision (rank
+   variants, publish a number, gate a release, act on single items) and the criterion the user
+   or their files give: ✅ *"Gate: a build ships if its precision@5 is within 2.0 points of
+   production; the change compares short against long documents."* Read every file for it
+   (READMEs, run notes) before asking. No criterion stated: no pass mark, and the answer says
+   what the errors would do to the decision. Name the variable the decision compares: version,
+   model, arm, or a property the planned change shifts (length, language).
+
+2. **Compute, don't estimate.** Run the script, grouped by the compared variable, with the
+   user's criterion as `--gate`:
+
+   ```bash
+   # Claude Code
+   python3 "${CLAUDE_SKILL_DIR}/scripts/judge_stats.py" grades.csv --group-by prompt_version
+   python3 "${CLAUDE_SKILL_DIR}/scripts/judge_stats.py" run_1.jsonl run_2.jsonl --gold gold.csv --map true=violation,false=ok --positive violation
+   # Other IDEs (from the skill folder)
+   python3 scripts/judge_stats.py wide.csv --judges big,small --positive relevant --group-by "tokens>=400" --gate "bias_diff<=2.0"
+   ```
+
+   Several files are one run each, joined on the item id with columns resolved per file;
+   `--gold` reads expert labels from their own file; `--judges` reads one column per judge;
+   `--map` renames labels that mean the same; `--col` maps any other column. Given only counts,
+   rebuild one row per item and run it anyway; without a shell, use the formulas in
+   [`reference.md`](reference.md) and show the arithmetic.
+
+3. **Quote every number from the output, with its subset, run and n.** All-rows and
+   labelled-rows figures are different numbers; so are run 1 and run 2. ✅ *"On the 20 items the
+   reviewer scored, the grader averages 3.7 against her 3.1."* — ❌ a run-1 false-positive count
+   presented as the result when run 2 has more. No figure appears that the output doesn't print.
+
+4. **Use the scale and the metric the user uses.** For 1–5 scores report quadratic-weighted
+   kappa (say "quadratic"), exact agreement and mean bias; unweighted kappa treats 4-vs-5 like
+   1-vs-5. Pass `--threshold` only when the user stated a pass score. When the user quotes a
+   figure ("kappa 0.8", "they agree 80%"), reproduce it first, say which statistic it is on which
+   rows, then show what it hides.
+
+5. **Lead with the differential.** Per variant compare exact agreement, bias judge − expert
+   with its interval, how often the judge scores higher and lower, and the bias-vs-length
+   correlation; then the judge's v2 − v1 gain against the expert's own on the labelled rows. ✅
+   *"It matches the reviewer on 18 of 20 A answers but 9 of 20 B answers and over-scores B by
+   +0.55 [+0.30, +0.80]; her own scores show B ahead by +0.10 (interval spans zero)."* When the
+   bias tracks a property the planned change shifts, the shift in bias is what lands on the gate.
+
+6. **Chance-correct agreement, and on a rare class count the rare class.** Two raters who each
+   say "pass" 90% of the time agree 82% by chance: ✅ *"Raw 88%, chance 80%, kappa 0.40."* With
+   one class under about 15%, lead with how many expert positives exist, the confusion counts,
+   recall and precision with intervals; accuracy only beside the always-majority baseline. ✅
+   *"6 of 120 posts are violations, so always saying 'fine' scores 95%. It caught 4 of 6 (30%–90%)
+   and flagged 2 fine posts."* — ❌ *"96% accurate, so it's reliable."*
+
+7. **With repeated runs, report each run, then the flips.** One row per run: precision, recall,
+   false positives. Then run-against-run agreement joined on the item id, the items that
+   flipped and which of them the expert labels positive, and what was caught in both runs. ✅
+   *"Run 1 wrongly flagged 1 post, run 2 flagged 2; 5 posts changed verdict, 2 of them real
+   violations."* Files in different row orders or formats are joined on id, never by line.
+
+8. **For pairwise judges, read the order column before the win rate.** State the direction from
+   the counts: ✅ *"The first-shown answer won 70 of 100; the new variant went first on 15 of 25
+   questions."* Report the order-balanced win rate per judge; the unit is the question, not the
+   judgement. An interval excluding 50% does not fix an order confound; judging both orders does.
+
+9. **Measure stability and judge-to-judge agreement across every trial.** Flip rate per judge,
+   with the flipped items named; trial-matched kappa between judges, and Fleiss' across every
+   judge × trial rating. Judges sharing a blind spot agree on it: expert agreement ranks first.
+
+10. **Say what the data cannot measure, in one line.** One run: no flip rate. One expert: no
+    expert-expert baseline. No expert labels: nothing shows the judge is right. Never fill a gap
+    with a guess.
+
+11. **Answer the question asked, against the user's criterion.** Each gate check reads
+    *clears* (whole interval meets it), *fails* (none of it does) or *can't tell* (the interval
+    spans it). "Which judge can we use" names one: ✅ *"Switch to the mid tier: it is the cheapest
+    judge whose error doesn't move with length (bias shift −2 points against a 2.0-point gate);
+    the small tier says relevant to 9 of 10 long passages the reviewer rejected."* — ❌ *"None
+    can be recommended until 100 more labels exist."* Conditions are actions with an n (label 30
+    more long passages); a small sample widens the interval, it doesn't withhold the answer.
+    When the judge in use today also reads *can't tell*, the sample can't certify anyone: compare
+    the candidate with the incumbent on the same items (the accuracy difference and its interval).
+    Kappa bands (Landis–Koch "moderate") appear only as labelled context, never as the reason.
+    Verdict words: **TRUSTWORTHY FOR THIS DECISION**, **USE WITH CONDITIONS**, **DO NOT RELY ON IT**.
+
+12. **Write the reply short.** First line: the answer (yes/no, which judge, which arm it
+    favours and by how much). Then the two to four numbers behind it, each with its subset and
+    run; what can't be measured; one next step that fits the user's deadline. No kappa tutorial,
+    no tables unless asked. A chat question gets a chat answer.
+
+**Non-interactive runs** (subagent, CI, headless): no criterion in the input becomes
+`[assumption] no stated gate; the reply says what the errors do to the decision`; a compared
+variable the input does not name becomes `[assumption] no grouping`. No judgements and no counts
+emit `BLOCKED: need the per-item judgements (item, judge, label or score) or the agreement
+counts`, because a verdict without them is invented.
+
+## Useful references in this skill
+
+- [`reference.md`](reference.md) — turning the user's gate into the number that answers it, kappa (unweighted, weighted, Fleiss') and Wilson formulas with worked tables, the both-orders protocol, and every input shape the script reads
+- [`scripts/judge_stats.py`](scripts/judge_stats.py) — per-group, per-run and pooled agreement, weighted kappa, rare-class precision and recall, flips joined on item id, position and length bias, `--gate` (stdlib only; `--self-test` checks the maths)
+- [`examples/summary-panel.md`](examples/summary-panel.md) — worked example: a three-judge panel on summary faithfulness, a USE WITH CONDITIONS verdict
+
+## Quality bar
+
+- **The first line answers the question asked:** ship or not, which judge, which arm it favours.
+- **The criterion is the user's;** with none, no invented pass mark decides anything.
+- **Statistics are stratified by the compared variable** before any pooled figure is quoted.
+- **Ordinal scores get weighted kappa, named as quadratic or linear.**
+- **A rare class gets its count, confusion, recall and precision with intervals,** not accuracy.
+- **Each run is reported on its own, with the flips,** joined on item id.
+- **Every number matches the output for the subset and run it is labelled with,** n included.
+- **What the data cannot measure is named,** never estimated.
+
+## When to use this skill
+
+- ✅ "The grader says v2 beats v1 — can we ship on it?" with a file of scores
+- ✅ "Which is the cheapest judge we can switch to?" with several judges on the same items
+- ✅ "It's 96% accurate on two runs — can we turn on auto-remove?"
+- ✅ A pairwise A/B judged by a model, before shipping the winner
+- ✅ "Our two graders agree 90%, good enough?"
+
+## When NOT to use this skill
+
+- ❌ Designing the eval set, grading methods and pass bar for an LLM feature — `llm-eval-harness`
+- ❌ Generating eval cases for a skill's `evals/` folder — `eval-harness-generator`
+- ❌ Measuring how much teams use AI tools — `ai-usage-report`
+- ❌ Inter-rater reliability for humans only, with no model in the loop — the statistics transfer, but the bias checks here assume a model
+
+## Anti-patterns to avoid
+
+- ❌ **An invented gate.** "Kappa must be ≥ 0.6 with n ≥ 100" when the user's gate is a 1.0-point tolerance.
+- ❌ **Refusing to pick.** "Which is cheapest?" answered with "none yet" instead of a name and a condition.
+- ❌ **Run 1 as the result.** Two runs are two rows, then the flips.
+- ❌ **Joining runs by line number.** Rows in a different order make a line-by-line kappa meaningless.
+- ❌ **Accuracy on a rare class.** 96% when 95% is the always-"fine" score.
+- ❌ **A pooled kappa for a comparison,** or unweighted kappa on a 1–5 scale.
+- ❌ **Position bias stated backwards,** or judgements counted as the unit instead of questions.
+- ❌ **Leading with the stats dump.** The reader asked whether to act; tell them, then show why.

package/packs/ai-engineering/skills/judge-audit/evals/eval_queries.json

@@ -0,0 +1,11 @@
+[
+  { "query": "Our LLM grader and a second model agree 88% on pass/fail. Is that enough to use it as a release gate?", "should_trigger": true },
+  { "query": "Here's a CSV of judge scores with three trials per item and some human labels. Can we trust the judge?", "should_trigger": true },
+  { "query": "Check our pairwise LLM judge for position bias before we pick the winning prompt", "should_trigger": true },
+  { "query": "Compute Cohen's kappa between our model judge and the annotators and tell me whether the judge is usable", "should_trigger": true },
+  { "query": "We want to swap our expensive judge model for a cheaper one. Here are both models' grades on the same 100 answers plus expert labels.", "should_trigger": true },
+  { "query": "We want an LLM judge to grade helpfulness on our chatbot. How should we set that up?", "should_trigger": false },
+  { "query": "Design an eval set and pass bar for our RAG assistant before we change the prompt", "should_trigger": false },
+  { "query": "Generate eval cases for this skilldrop SKILL.md so I can drop them in its evals folder", "should_trigger": false },
+  { "query": "Summarise how much each team used AI coding assistants last quarter from this telemetry export", "should_trigger": false }
+]