spec-eval 0.2.0__py3-none-any.whl

spec_eval/cli.py

@@ -0,0 +1,192 @@
+"""spec-eval CLI — portable spec coverage + code↔doc drift + sufficiency + spec authoring.
+
+  spec-eval coverage    <repo> --config pairs.yml                             # free, no key
+  spec-eval generate    <repo> [--config pairs.yml] --model … --env .env      # author specs beside the code
+  spec-eval audit       <repo> --config pairs.yml --model … --env .env        # drift
+  spec-eval sufficiency <repo> --config pairs.yml --model … --env .env        # sufficiency
+
+Keys come from the environment (ANTHROPIC_API_KEY / OPENAI_API_KEY / GOOGLE_API_KEY); --env loads a .env file.
+"""
+import argparse
+import os
+import sys
+import json
+from . import providers, audit, report, coverage as coverage_mod, sufficiency as sufficiency_mod, authoring, runlog
+
+UNCOVERED_LIST_CAP = 25   # cap the per-line uncovered list printed to the terminal; full list always lands in coverage.md
+
+
+def _load_keys(env, config):
+    providers.load_env(env)
+    if config:
+        providers.load_env(os.path.join(os.path.dirname(os.path.abspath(config)), ".env"))
+    providers.load_env(".env")
+
+
+def _file_scope(path, cfg):
+    """PROJECT_DIR may be a single code FILE (audit / sufficiency / generate): narrow to the file's directory
+    with ONE synthesized pair — its co-located `<stem>.md`, or the folder spec under a per-dir layout — so no
+    pairs config is needed to scope a check to the file you just changed. Overviews are forced off."""
+    repo = os.path.dirname(os.path.abspath(path))
+    rel = os.path.basename(path)
+    stem = os.path.splitext(rel)[0]
+    doc = stem + ".md"
+    authoring_cfg = cfg.get("authoring", {})
+    if not os.path.exists(os.path.join(repo, doc)) and authoring_cfg.get("layout") == "per-dir":
+        doc = os.path.basename(coverage_mod.dir_spec_path(rel, os.path.basename(repo),
+                                                          authoring_cfg.get("dir_spec_name", "<dir>")))
+    return repo, {**cfg, "pairs": [{"label": stem, "code": [rel], "docs": [doc]}],
+                  "authoring": {**authoring_cfg, "overview": "none"}}
+
+
+def main(argv=None):
+    ap = argparse.ArgumentParser(prog="spec-eval",
+                                 description="Portable spec coverage + drift + sufficiency + authoring.")
+    sub = ap.add_subparsers(dest="cmd", required=True)
+
+    def model_args(p):
+        p.add_argument("repo", metavar="PROJECT_DIR", help="path to the project — a repo root, a subdirectory, or a single code file")
+        p.add_argument("--config", "-c", default=None,
+                       help="pairs config (YAML/JSON); optional — co-located `<file>.md` specs are auto-paired")
+        p.add_argument("--model", "-m", default=providers.DEFAULT_MODEL,
+                       help="provider:model (e.g. openai:gpt-5.5, google:gemini-3.5-flash), or claude-code "
+                            "(no API key — your Claude subscription via the claude CLI)")
+        p.add_argument("--out", "-o", default="spec-reports", help="output directory")
+        p.add_argument("--env", default=None, help="path to a .env file with API keys")
+        p.add_argument("--fingerprint", action=argparse.BooleanOptionalAction, default=True,
+                       help="include the markdown unicode-bar fingerprint (default: on)")
+
+    model_args(sub.add_parser("audit", help="DRIFT: do the pairs CONTRADICT their specs? (needs a model)"))
+    model_args(sub.add_parser("sufficiency", help="SUFFICIENCY: how completely do the specs capture the code's "
+                                                  "behavior? (needs a model)"))
+
+    g = sub.add_parser("generate", help="AUTHOR an intent-led spec BESIDE each spec-worthy code file with no spec (needs a model)")
+    g.add_argument("repo", metavar="PROJECT_DIR", help="path to the project — a repo root, a subdirectory, or a single code file")
+    g.add_argument("--config", "-c", default=None, help="optional config (YAML/JSON) for code_ext / exclude; defaults to sensible excludes")
+    g.add_argument("--model", "-m", default=providers.DEFAULT_MODEL,
+                   help="provider:model, or claude-code (no API key — your Claude subscription via the claude CLI)")
+    g.add_argument("--out", "-o", default="spec-reports", help="output directory (run summary)")
+    g.add_argument("--env", default=None, help="path to a .env file with API keys")
+    g.add_argument("--overwrite", action="store_true", help="re-author specs that already exist")
+    g.add_argument("--layout", choices=["per-file", "per-dir", "per-pair"], default=None,
+                   help="spec granularity: per-file (default) | per-dir | per-pair")
+    g.add_argument("--overview", choices=["none", "repo", "per-dir", "both"], default=None,
+                   help="also write a navigation index (default: none)")
+    g.add_argument("--template", default=None,
+                   help="path to a custom authoring template (replaces the built-in structure)")
+
+    c = sub.add_parser("coverage", help="COVERAGE: which code files have NO governing spec? (free; no API key)")
+    c.add_argument("repo", metavar="PROJECT_DIR", help="path to the project — a repo root or any subdirectory")
+    c.add_argument("--config", "-c", default=None,
+                   help="pairs config (YAML/JSON); optional — co-located `<file>.md` specs count as covered")
+    c.add_argument("--out", "-o", default="spec-reports", help="output directory")
+    c.add_argument("--min", type=float, default=None, help="fail (exit 1) if coverage %% is below this")
+
+    args = ap.parse_args(argv)
+
+    if args.cmd == "audit":
+        _load_keys(args.env, args.config)
+        cfg = audit.load_config(args.config) if args.config else {}
+        if os.path.isfile(args.repo):
+            args.repo, cfg = _file_scope(args.repo, cfg)
+        os.makedirs(args.out, exist_ok=True)
+        results = audit.audit_repo(args.repo, cfg, args.model)
+        json.dump(results, open(os.path.join(args.out, "findings.json"), "w"), indent=2)
+        total = report.write_markdown(results, args.repo, args.model, os.path.join(args.out, "report.md"), args.fingerprint)
+        audited = sum(1 for r in results if not r.get("skipped"))
+        print(f"{total} high/medium drift finding(s) across {audited}/{len(results)} audited pair(s).")
+        truncated = sum(1 for r in results if r.get("truncated"))
+        if truncated:
+            print(f"⚠ {truncated} pair(s) graded on a partial view (input cap or reply token cap) — "
+                  f"flagged per pair in report.md")
+        print(f"wrote report.md + findings.json → {os.path.abspath(args.out)}")
+        print(f"{providers.USAGE['calls']} model call(s), "
+              f"{providers.USAGE['in']:,} in + {providers.USAGE['out']:,} out tokens")
+        runlog.append_run(args.out, args.repo, "audit", args.model,
+                          {"high_med_drift": total, "pairs_audited": audited, "pairs_truncated": truncated,
+                           "per_module": {r["label"]: report.drift_load(r) for r in results if not r.get("skipped")}})
+
+    elif args.cmd == "sufficiency":
+        _load_keys(args.env, args.config)
+        cfg = audit.load_config(args.config) if args.config else {}
+        if os.path.isfile(args.repo):
+            args.repo, cfg = _file_scope(args.repo, cfg)
+        os.makedirs(args.out, exist_ok=True)
+        results = sufficiency_mod.sufficiency_repo(args.repo, cfg, args.model)
+        json.dump(results, open(os.path.join(args.out, "sufficiency.json"), "w"), indent=2)
+        avg = report.write_sufficiency_markdown(results, args.repo, args.model, os.path.join(args.out, "sufficiency.md"), args.fingerprint)
+        scored = sum(1 for r in results if r.get("sufficiency") is not None)
+        print(f"average spec sufficiency {avg:.2f} across {scored}/{len(results)} pairs (1.0 = no gaps found — an "
+              f"indicator, not a guarantee)")
+        truncated = sum(1 for r in results if r.get("truncated"))
+        if truncated:
+            print(f"⚠ {truncated} pair(s) scored on a partial view (input cap or reply token cap) — "
+                  f"flagged per pair in sufficiency.md")
+        print(f"wrote sufficiency.md + sufficiency.json → {os.path.abspath(args.out)}")
+        print(f"{providers.USAGE['calls']} model call(s), {providers.USAGE['in']:,} in + {providers.USAGE['out']:,} out tokens")
+        runlog.append_run(args.out, args.repo, "sufficiency", args.model,
+                          {"avg_sufficiency": round(avg, 2), "pairs_scored": scored, "pairs_truncated": truncated,
+                           "per_module": {r["label"]: r["sufficiency"] for r in results if r.get("sufficiency") is not None}})
+
+    elif args.cmd == "generate":
+        _load_keys(args.env, args.config)
+        cfg = audit.load_config(args.config) if args.config else {"pairs": []}
+        authoring_cfg = dict(cfg.get("authoring", {}))        # CLI flags override the config's authoring block
+        if args.layout:
+            authoring_cfg["layout"] = args.layout
+        if args.overview:
+            authoring_cfg["overview"] = args.overview
+        if args.template:
+            authoring_cfg["template"] = args.template
+        cfg = {**cfg, "authoring": authoring_cfg}             # coverage() (called inside generate_repo) reads this too
+        if os.path.isfile(args.repo):
+            args.repo, cfg = _file_scope(args.repo, cfg)
+            cfg["authoring"]["layout"] = "per-pair"           # author exactly that file's spec
+        os.makedirs(args.out, exist_ok=True)
+        print("generating specs — one model call per module; progress below (Ctrl-C is safe)…", file=sys.stderr, flush=True)
+        res = authoring.generate_repo(args.repo, cfg, args.model, overwrite=args.overwrite,
+                                      on_progress=lambda m: print(f"  … {m}", file=sys.stderr, flush=True))
+        json.dump(res, open(os.path.join(args.out, "generated.json"), "w"), indent=2)
+        for r in res:
+            note = f"  — {r['note']}" if r.get("note") else ""
+            print(f"  {r['status']:10} {r['spec']}{note}")
+        authored = sum(1 for r in res if r["status"] == "authored")
+        print(f"authored {authored} spec(s) beside the code; {len(res)-authored} skipped. "
+              f"Review with `git diff`; drop any you don't want with `git checkout --`.")
+        print(f"{providers.USAGE['calls']} model call(s), {providers.USAGE['in']:,} in + {providers.USAGE['out']:,} out tokens")
+        runlog.append_run(args.out, args.repo, "generate", args.model,
+                          {"authored": authored, "skipped": len(res) - authored,
+                           "flagged": sum(1 for r in res if r.get("note"))})
+
+    elif args.cmd == "coverage":
+        if os.path.isfile(args.repo):
+            parent = os.path.dirname(args.repo) or "."
+            raise SystemExit(f"coverage reports on directories — try `spec-eval coverage {parent}`. "
+                             "(audit / sufficiency / generate do accept a single file.)")
+        cfg = audit.load_config(args.config) if args.config else {}
+        os.makedirs(args.out, exist_ok=True)
+        cov = coverage_mod.coverage(args.repo, cfg)
+        json.dump(cov, open(os.path.join(args.out, "coverage.json"), "w"), indent=2)
+        open(os.path.join(args.out, "coverage.md"), "w").write(coverage_mod.format_report(cov, args.repo))
+        print(f"spec coverage: {cov['pct']:.0f}% — {len(cov['covered'])}/{cov['spec_worthy']} spec-worthy files covered")
+        if cov["uncovered"]:
+            print(f"  uncovered ({len(cov['uncovered'])}) — spec-worthy files with no governing spec:")
+            for f in cov["uncovered"][:UNCOVERED_LIST_CAP]:
+                print(f"    - {f}")
+            if len(cov["uncovered"]) > UNCOVERED_LIST_CAP:
+                print(f"    … and {len(cov['uncovered']) - UNCOVERED_LIST_CAP} more (full list in coverage.md)")
+        if cov.get("orphans"):
+            print(f"  possible orphaned spec(s) ({len(cov['orphans'])}) — spec-shaped .md with no same-stem code file; review or remove:")
+            for f in cov["orphans"][:UNCOVERED_LIST_CAP]:
+                print(f"    - {f}")
+        print(f"wrote coverage.md + coverage.json → {os.path.abspath(args.out)}  (free; no API key)")
+        runlog.append_run(args.out, args.repo, "coverage", None,
+                          {"coverage_pct": cov["pct"], "covered": len(cov["covered"]),
+                           "spec_worthy": cov["spec_worthy"], "orphans": len(cov.get("orphans", []))})
+        if args.min is not None and cov["pct"] < args.min:
+            print(f"FAIL: coverage {cov['pct']:.0f}% < --min {args.min:.0f}%")
+            raise SystemExit(1)
+
+
+if __name__ == "__main__":
+    main()

spec_eval/coverage.py

@@ -0,0 +1,191 @@
+"""Spec coverage — which code files have NO governing spec? Pure filesystem set-arithmetic: NO API key, NO
+model call, free + CI-fast. The spec analogue of test coverage.
+
+COVERED   = files matched by any pair's `code` globs in the config.
+CANDIDATE = repo code files (.py) minus the EXCLUDES taxonomy (tests / config / generated / glue / skills/docs
+            / user `exclude:` pragmas) — the classes that are impractical to spec (each would create a second
+            source of truth with no independent representation to cross-check, the exact drift the tool sells
+            against). 'Missing spec' (coverage) is a SEPARATE finding type from 'drift' (accuracy).
+UNCOVERED = spec-worthy code files with no governing pair.
+"""
+import os
+import glob
+import fnmatch
+
+# the code universe — language-agnostic by default; override per-repo via the config's `code_ext:` list.
+DEFAULT_CODE_EXT = (".py", ".ts", ".tsx", ".js", ".jsx", ".go", ".rs", ".java", ".rb", ".kt", ".swift", ".php", ".cs")
+
+
+def classify_exclude(rel, user_excludes):
+    """Return the exclusion tier for a path, or None if it is spec-worthy. Language-agnostic conventions."""
+    base = os.path.basename(rel)
+    stem = base.rsplit(".", 1)[0]
+    ext = os.path.splitext(base)[1]
+    parts = set(rel.split(os.sep))
+    if any(fnmatch.fnmatch(rel, p) or fnmatch.fnmatch(base, p) for p in (user_excludes or [])):
+        return "user"
+    # tests (py/ts/js/go/rust conventions): test_x, x_test, x.test.*, x.spec.*, in a tests dir
+    if (base.startswith("test_") or stem.endswith("_test") or ".test." in base or ".spec." in base
+            or {"tests", "__tests__", "test", "spec", "__mocks__"} & parts):
+        return "test"
+    # glue / trivial entrypoints across languages
+    if base in ("__init__.py", "__main__.py", "conftest.py", "setup.py", "index.ts", "index.js",
+                "index.tsx", "index.jsx", "main.go", "mod.rs", "lib.rs"):
+        return "glue"
+    if {"scripts", "tools", "tooling"} & parts:
+        return "tooling"
+    if {"__pycache__", "build", "dist", ".venv", "node_modules", "mutants", "formal"} & parts:
+        return "generated"
+    if base.endswith(".min.js") or base.endswith(".min.mjs"):
+        return "generated"                              # minified vendor bundles are build output, not source
+    if ext in (".yml", ".yaml", ".toml", ".cfg", ".ini", ".txt", ".lock"):
+        return "config"
+    if ext == ".md" or base == "SKILL.md":
+        return "skill/doc"
+    return None
+
+
+# doc names that are never co-located specs — a `README.md` beside code is a doc, not an orphaned spec
+CONVENTIONAL_DOC_STEMS = {"readme", "overview", "changelog", "changes", "contributing", "license", "notice",
+                          "spec-health", "testing", "faq", "getting-started", "getting_started", "security",
+                          "code_of_conduct", "skill", "todo", "authors", "maintainers", "install", "upgrading"}
+
+# directories never worth walking into (vendored deps, build output, caches, generated test artifacts)
+PRUNE_DIRS = {".git", ".venv", "venv", "env", "build", "dist", "node_modules", "__pycache__",
+              ".mypy_cache", ".pytest_cache", ".ruff_cache", ".tox", "htmlcov", "site-packages",
+              "mutants", ".idea", ".vscode", ".eggs",
+              "vendor", "vendored", "third_party", "thirdparty", "bower_components", "jspm_packages"}
+
+
+def dir_spec_path(code_path, repo_name, dir_spec_name="<dir>"):
+    """Per-directory spec path for the directory containing `code_path`. `<dir>` -> the directory's own name
+    (`src/parser/lexer.py` -> `src/parser/parser.md`); a literal like `README` -> `src/parser/README.md`. A
+    repo-root file uses `repo_name`. Shared with `authoring` so `coverage` and `generate` agree on the layout."""
+    d = os.path.dirname(code_path)
+    dirname = os.path.basename(d) if d else repo_name
+    name = dirname if dir_spec_name == "<dir>" else dir_spec_name
+    return os.path.join(d, name + ".md")
+
+
+def infer_pairs(repo, config):
+    """Co-located pairs — each spec-worthy code file + its sibling `<stem>.md`, when the `.md` exists.
+
+    Used when the config declares no explicit `pairs`: the "specs live beside the code" convention, so
+    `audit`/`sufficiency` need no `pairs.yml` — a `model.md` next to `model.py` is auto-paired.
+    """
+    repo = os.path.abspath(repo)
+    exts = tuple(config.get("code_ext") or DEFAULT_CODE_EXT)
+    user_excludes = config.get("exclude", [])
+    pairs = []
+    for root, dirs, files in os.walk(repo):
+        dirs[:] = [d for d in dirs if d not in PRUNE_DIRS]
+        for fn in sorted(files):
+            if not fn.endswith(exts):
+                continue
+            rel = os.path.relpath(os.path.join(root, fn), repo)
+            if classify_exclude(rel, user_excludes):
+                continue
+            md = os.path.splitext(rel)[0] + ".md"
+            if os.path.isfile(os.path.join(repo, md)):
+                pairs.append({"label": os.path.splitext(os.path.basename(rel))[0], "code": [rel], "docs": [md]})
+    return sorted(pairs, key=lambda p: p["label"])
+
+
+def coverage(repo, config):
+    repo = os.path.abspath(repo)
+    exts = tuple(config.get("code_ext") or DEFAULT_CODE_EXT)   # language-agnostic; override per repo
+    authoring = config.get("authoring", {})                    # the authoring layout decides what "covered" means
+    layout = authoring.get("layout", "per-file")
+    dir_spec_name = authoring.get("dir_spec_name", "<dir>")
+    repo_name = os.path.basename(repo)
+    covered = set()
+    for p in config.get("pairs", []):
+        for pat in p.get("code", []):
+            for f in glob.glob(os.path.join(repo, pat), recursive=True):
+                if os.path.isfile(f):
+                    covered.add(os.path.relpath(f, repo))
+
+    candidate = set()
+    mds = set()
+    for root, dirs, files in os.walk(repo):
+        dirs[:] = [d for d in dirs if d not in PRUNE_DIRS]   # don't descend into vendored/generated/cache dirs
+        for fn in files:
+            if fn.endswith(exts):
+                candidate.add(os.path.relpath(os.path.join(root, fn), repo))
+            elif fn.endswith(".md"):
+                mds.add(os.path.relpath(os.path.join(root, fn), repo))
+
+    for rel in candidate:                                    # a co-located `<stem>.md` counts as coverage
+        if os.path.isfile(os.path.join(repo, os.path.splitext(rel)[0] + ".md")):
+            covered.add(rel)
+        elif layout == "per-dir" and os.path.isfile(os.path.join(repo, dir_spec_path(rel, repo_name, dir_spec_name))):
+            covered.add(rel)                                 # per-directory layout: the folder spec governs the file
+
+    user_excludes = config.get("exclude", [])
+    excluded = {}
+    for rel in candidate:
+        tier = classify_exclude(rel, user_excludes)
+        if tier:
+            excluded[rel] = tier
+    spec_worthy = candidate - set(excluded)
+    covered_worthy = covered & spec_worthy
+    uncovered = sorted(spec_worthy - covered)
+    pct = 100 * len(covered_worthy) / max(len(spec_worthy), 1)
+    by_tier = {}
+    for rel, t in excluded.items():
+        by_tier.setdefault(t, []).append(rel)
+
+    # Possible ORPHANED specs (heuristic, advisory only — never affects pct/covered/uncovered): a spec-shaped
+    # `.md` sitting in a directory that has code, whose same-stem code file no longer exists. Catches the
+    # agentic-dev churn case where code moves and its old spec is left behind. Conventional docs (README, …),
+    # folder specs, pair-declared docs, and docs-only directories are never flagged.
+    pair_docs = set()
+    for p in config.get("pairs", []):
+        for pat in p.get("docs", []):
+            for f in glob.glob(os.path.join(repo, pat), recursive=True):
+                if os.path.isfile(f):
+                    pair_docs.add(os.path.relpath(f, repo))
+    code_dirs = {os.path.dirname(rel) for rel in candidate}
+    orphans = []
+    for md in mds:
+        d = os.path.dirname(md)
+        if d not in code_dirs:
+            continue                                     # docs-only dir → docs, not specs
+        if classify_exclude(md, user_excludes) == "user":
+            continue                                     # respect the repo's own exclude: scoping
+        stem = os.path.splitext(os.path.basename(md))[0]
+        if stem.lower() in CONVENTIONAL_DOC_STEMS:
+            continue
+        if stem == (os.path.basename(d) if d else repo_name):
+            continue                                     # a folder spec (`<dir>/<dir>.md`), not an orphan
+        if dir_spec_name != "<dir>" and stem == dir_spec_name:
+            continue
+        if md in pair_docs:
+            continue                                     # explicitly paired — governed, wherever its code is
+        base = os.path.splitext(md)[0]
+        if any(os.path.isfile(os.path.join(repo, base + ext)) for ext in exts):
+            continue                                     # sibling code exists → a normal co-located spec
+        orphans.append(md)
+
+    return {"pct": round(pct, 1), "spec_worthy": len(spec_worthy),
+            "covered": sorted(covered_worthy), "uncovered": uncovered, "orphans": sorted(orphans),
+            "excluded": {t: sorted(v) for t, v in by_tier.items()}}
+
+
+def format_report(cov, repo):
+    name = os.path.basename(os.path.abspath(repo))
+    lines = [f"# Spec coverage — `{name}`",
+             f"**{cov['pct']:.0f}%** — {len(cov['covered'])}/{cov['spec_worthy']} spec-worthy code files have a governing spec.", ""]
+    if cov["uncovered"]:
+        lines.append("## ⚠ Uncovered (spec-worthy, no governing pair)")
+        lines += [f"- `{f}`" for f in cov["uncovered"]]
+        lines.append("")
+    if cov.get("orphans"):
+        lines.append("## Possible orphaned specs *(heuristic — a spec-shaped `.md` whose same-stem code file is gone; the code may have moved. Review, then delete or re-pair.)*")
+        lines += [f"- `{f}`" for f in cov["orphans"]]
+        lines.append("")
+    lines.append("## Excluded (impractical to spec — by class)")
+    for tier, files in sorted(cov["excluded"].items()):
+        lines.append(f"- **{tier}** ({len(files)}): " + ", ".join(f"`{f}`" for f in files[:8]) +
+                     (" …" if len(files) > 8 else ""))
+    return "\n".join(lines)

spec_eval/providers.py

@@ -0,0 +1,119 @@
+"""Portable model providers — anthropic / openai / google (env-keyed) plus the `claude-code` bridge (the local
+Claude Code CLI: your Claude subscription, NO API key). Usage-tracked. No hardcoded paths.
+
+Truncation transparency: every API reply carries a stop/finish reason; when it says the reply hit `max_tokens`,
+that fact is recorded (`LAST["truncated"]` for the call just made, `USAGE["truncated"]` as a tally) so callers
+can flag a partial view instead of passing it off as complete. The bridge exposes no stop reason, so bridge
+calls never set the flag."""
+import os
+
+DEFAULT_MODEL = "anthropic:claude-opus-4-8"   # THE single default model; the CLI's --model defaults read this
+_clients = {}
+USAGE = {"in": 0, "out": 0, "calls": 0, "truncated": 0}   # exact token + call tallies; the tool reports usage, never a dollar figure
+LAST = {"truncated": False}   # the call JUST made (calls are sequential); read it right after gen() returns
+
+
+def load_env(path):
+    """Minimal .env loader (KEY=VALUE), non-overriding. Safe if the file is absent."""
+    if not path or not os.path.exists(path):
+        return
+    for line in open(path):
+        line = line.strip()
+        if line and not line.startswith("#") and "=" in line:
+            k, v = line.split("=", 1)
+            os.environ.setdefault(k.strip(), v.strip().strip('"').strip("'"))
+
+
+def parse_model(spec):
+    """'anthropic:claude-opus-4-8' → (anthropic, claude-opus-4-8); bare names get a sensible default provider.
+    'claude-code' → the Claude Code CLI bridge ('claude-code:<name>' passes <name> to the CLI's --model)."""
+    if spec == "claude-code":
+        return "claude-code", ""                               # bridge, using the CLI's own default model
+    if ":" in spec:
+        p, m = spec.split(":", 1)
+        return p, m
+    if spec.startswith("gpt"):
+        return "openai", spec
+    if spec.startswith("gemini"):
+        return "google", spec
+    return "anthropic", spec
+
+
+def _track(u_in, u_out, truncated=False):
+    """Record one call's token usage and whether its reply was cut off at the token cap (None-safe)."""
+    USAGE["in"] += u_in or 0
+    USAGE["out"] += u_out or 0
+    USAGE["calls"] += 1
+    USAGE["truncated"] += 1 if truncated else 0
+    LAST["truncated"] = bool(truncated)
+
+
+def _gen_claude_code(model, system, user):
+    """Bridge to the Claude Code CLI (`claude -p`): runs on the CLI's own login — a Claude subscription — so NO
+    API key is needed. ANTHROPIC_API_KEY is stripped from the child environment so a stray key (e.g. auto-loaded
+    from a .env) can never silently switch billing from the subscription to the paid API. Reads the CLI's
+    `--output-format json` envelope from stdout only (stderr may carry warnings); token counts are exact, from
+    the envelope's `usage`. `max_tokens` is not passed (the CLI has no such flag), and the envelope exposes no
+    stop reason — so bridge calls can never flag reply truncation."""
+    import json
+    import shutil
+    import subprocess
+    exe = shutil.which("claude")
+    if not exe:
+        raise RuntimeError("model 'claude-code' needs the Claude Code CLI on PATH — install it and log in once, "
+                           "or use an API-key provider (anthropic: / openai: / google:)")
+    env = {k: v for k, v in os.environ.items() if k != "ANTHROPIC_API_KEY"}
+    cmd = [exe, "-p", "--output-format", "json", "--system-prompt", system]
+    if model:
+        cmd += ["--model", model]
+    r = subprocess.run(cmd, input=user, capture_output=True, text=True, env=env, timeout=600)
+    if r.returncode != 0:
+        raise RuntimeError(f"`claude -p` failed (exit {r.returncode}): {(r.stderr or r.stdout).strip()[:400]}")
+    d = json.loads(r.stdout)                                   # {"result": text, "usage": {...}, "is_error": bool, ...}
+    if d.get("is_error") or "result" not in d:
+        raise RuntimeError(f"`claude -p` returned an error envelope: {r.stdout[:400]}")
+    u = d.get("usage") or {}
+    u_in = ((u.get("input_tokens") or 0) + (u.get("cache_creation_input_tokens") or 0)
+            + (u.get("cache_read_input_tokens") or 0))         # the CLI reports cached input separately; sum = tokens actually processed
+    _track(u_in, u.get("output_tokens"))
+    return d.get("result") or ""
+
+
+def gen(model_spec, system, user, max_tokens=1200):
+    prov, model = parse_model(model_spec)
+    if prov == "claude-code":
+        return _gen_claude_code(model, system, user)
+    if prov == "anthropic":
+        if "anthropic" not in _clients:
+            import anthropic
+            _clients["anthropic"] = anthropic.Anthropic()      # reads ANTHROPIC_API_KEY
+        r = _clients["anthropic"].messages.create(
+            model=model, max_tokens=max_tokens, system=system,
+            messages=[{"role": "user", "content": user}])
+        _track(r.usage.input_tokens, r.usage.output_tokens,
+               truncated=(getattr(r, "stop_reason", None) == "max_tokens"))
+        return "".join(b.text for b in r.content if getattr(b, "type", None) == "text")
+    if prov == "openai":
+        if "openai" not in _clients:
+            from openai import OpenAI
+            _clients["openai"] = OpenAI()                       # reads OPENAI_API_KEY
+        r = _clients["openai"].chat.completions.create(
+            model=model, messages=[{"role": "system", "content": system}, {"role": "user", "content": user}])
+        _track(r.usage.prompt_tokens, r.usage.completion_tokens,
+               truncated=(getattr(r.choices[0], "finish_reason", None) == "length" if r.choices else False))
+        return r.choices[0].message.content or ""
+    if prov == "google":
+        from google import genai
+        from google.genai import types
+        if "google" not in _clients:
+            _clients["google"] = genai.Client()                # reads GOOGLE_API_KEY / GEMINI_API_KEY
+        r = _clients["google"].models.generate_content(
+            model=model, contents=user,
+            config=types.GenerateContentConfig(system_instruction=system, max_output_tokens=max_tokens))
+        um = getattr(r, "usage_metadata", None)
+        cands = getattr(r, "candidates", None) or []
+        fr = getattr(cands[0], "finish_reason", None) if cands else None
+        _track(getattr(um, "prompt_token_count", 0), getattr(um, "candidates_token_count", 0),
+               truncated="MAX_TOKENS" in (getattr(fr, "name", None) or str(fr or "")))
+        return r.text or ""
+    raise ValueError(f"unknown provider '{prov}' (use anthropic: / openai: / google: / claude-code)")

spec_eval/report.py

@@ -0,0 +1,102 @@
+"""Render the drift + sufficiency reports (markdown) — the legible product output.
+
+The "fingerprint" is a MARKDOWN unicode-bar table (diffable, code-searchable, renders in any markdown viewer),
+toggled by `include_fingerprint`.
+"""
+import os
+from . import providers
+
+
+def drift_load(r):
+    return sum(1 for f in r["findings"] if f["severity"] in ("high", "medium"))
+
+
+def _bar(v, width=20):
+    """Unicode bar for a 0..1 value (full block = filled, light shade = empty)."""
+    v = max(0.0, min(1.0, float(v)))
+    filled = int(round(v * width))
+    return "█" * filled + "░" * (width - filled)
+
+
+def sufficiency_fingerprint(results):
+    """A markdown unicode-bar table of per-pair sufficiency (worst first). Pure text."""
+    rs = sorted((r for r in results if r.get("sufficiency") is not None), key=lambda r: r["sufficiency"])
+    if not rs:
+        return ""
+    lines = ["", "## Sufficiency fingerprint  *(at a glance — worst first)*", "",
+             "| Pair | Spec completeness | Score |", "|---|---|---|"]
+    for r in rs:
+        lines.append(f"| `{r['label']}` | `{_bar(r['sufficiency'])}` | {r['sufficiency']:.2f} |")
+    return "\n".join(lines) + "\n"
+
+
+def drift_fingerprint(results):
+    """A markdown table of per-pair drift load (✓ clean / count). Pure text."""
+    rs = [r for r in results if not r.get("skipped")]
+    if not rs:
+        return ""
+    lines = ["", "### Drift fingerprint", "", "| Pair | High+med findings |", "|---|---|"]
+    for r in rs:
+        n = drift_load(r)
+        lines.append(f"| `{r['label']}` | {'✓ clean' if n == 0 else f'⚠ {n}'} |")
+    return "\n".join(lines) + "\n"
+
+
+def write_markdown(results, repo, model, out_path, include_fingerprint=True):
+    name = os.path.basename(os.path.abspath(repo))
+    total = sum(drift_load(r) for r in results if not r.get("skipped"))
+    audited = [r for r in results if not r.get("skipped")]
+    lines = [f"# Drift report — `{name}`",
+             f"detector: `{model}` · {len(audited)}/{len(results)} pairs audited · "
+             f"{providers.USAGE['calls']} model call(s)", "",
+             f"**{total} high/medium drift finding(s) across {len(audited)} audited pair(s).**", ""]
+    for r in results:
+        if r.get("skipped"):
+            lines += [f"## {r['label']} — _skipped: {r['skipped']}_", ""]
+            continue
+        n = drift_load(r)
+        lines.append(f"## {r['label']} — {'✓ clean' if n == 0 else f'⚠ {n} drift'}")
+        if r.get("truncated"):
+            lines.append(f"- ⚠ *partial view ({'; '.join(r['truncated'])}) — findings may be incomplete*")
+        for f in r["findings"]:
+            ref = f" (`{f.get('code_ref') or '?'}` vs `{f.get('doc_ref') or '?'}`)" if f.get("code_ref") or f.get("doc_ref") else ""
+            lines.append(f"- **[{f['severity']}]** {f['summary']}{ref}")
+            if f.get("suggestion"):
+                lines.append(f"    - *fix:* {f['suggestion']}")
+        lines.append("")
+    body = "\n".join(lines)
+    if include_fingerprint:
+        body += drift_fingerprint(results)
+    open(out_path, "w").write(body)
+    return total
+
+
+def write_sufficiency_markdown(results, repo, model, out_path, include_fingerprint=True):
+    name = os.path.basename(os.path.abspath(repo))
+    scored = [r for r in results if r.get("sufficiency") is not None]
+    avg = sum(r["sufficiency"] for r in scored) / len(scored) if scored else 0
+    head = [f"# Spec sufficiency — `{name}`",
+            f"detector: `{model}` · {len(scored)}/{len(results)} pairs scored · "
+            f"{providers.USAGE['calls']} model call(s)", "",
+            f"**Average sufficiency {avg:.2f}** — how completely does the spec capture the code's behavior? "
+            f"(1.0 = no gaps found; gaps = behavior in the code but not the spec. An indicator, not a guarantee.)", ""]
+    # Fingerprint FIRST — the at-a-glance summary, so a reader sees the shape before the per-module detail.
+    summary = sufficiency_fingerprint(results) if include_fingerprint else ""
+    detail = ["## Per-module gaps  *(worst first)*", ""]
+    for r in sorted(results, key=lambda x: (x.get("sufficiency") is None, x.get("sufficiency") or 0)):
+        if r.get("skipped"):
+            detail += [f"### {r['label']} — _skipped: {r['skipped']}_", ""]
+            continue
+        if r.get("sufficiency") is None:
+            detail.append(f"### {r['label']} — not scored (unparseable model reply)")
+        else:
+            detail.append(f"### {r['label']} — sufficiency {r['sufficiency']:.2f}")
+        if r.get("truncated"):
+            detail.append(f"- ⚠ *partial view ({'; '.join(r['truncated'])})*")
+        for g in r["gaps"]:
+            ref = f" · `{g['code_ref']}`" if g.get("code_ref") else ""   # '·' — em dashes occur in gap prose; the dot splits unambiguously
+            detail.append(f"- **[{g['severity']}]** {g['missing']}{ref}")
+        detail.append("")
+    body = "\n".join(head) + summary + "\n" + "\n".join(detail)
+    open(out_path, "w").write(body)
+    return avg

spec_eval/rubric.py

@@ -0,0 +1,39 @@
+"""The bundled code↔doc drift rubric (self-contained — no external file dependency).
+
+Captures conservative, prefer-false-negatives drift-detection principles, bundled with the product so it stays
+portable. Severity is reserved;
+silence/scope are not drift. Tune with evidence — loosening it trades trust for noise.
+
+KEEP IN SYNC: `skills/spec-check/SKILL.md` carries the agent-session copy of this rubric (same severity tiers,
+same do-not-flag list). If you change the principles here, change them there — `tests/contract/test_rubric_sync.py`
+asserts the load-bearing phrases match, and this module's own spec is `rubric.md` beside this file (guarded by
+spec-eval's self-audit).
+"""
+
+DRIFT_RUBRIC = """You are a careful technical reviewer auditing a codebase for DRIFT between implementation and
+documentation: places where the code does X but the docs/spec claim Y (or vice versa).
+
+You are shown ONE pair: source file(s) plus the doc(s)/spec(s) meant to describe them. Find real mismatches.
+
+Severity is RESERVED — be strict; the tier is set by the DOC's claim:
+- high: the doc states a MEASURABLE GUARANTEE the code breaks — a numeric value/threshold/default that
+  disagrees with code; an explicit function/class/method signature; a named event/message the code never
+  emits; a CLI flag default; an invariant/acceptance criterion violated. If you must paraphrase the doc to
+  see the violation, it is NOT high.
+- medium: misleading but not load-bearing — a renamed function that still does the same thing; a stale
+  example; a field present in code but missing from a spec table; mechanism described differently.
+- low: cosmetic or trivially-fixable wording.
+
+Do NOT flag: stylistic differences; trivial restatements; missing-but-implied behaviour where a doc could
+plausibly be silent (silence is not drift); a doc describing a BROADER system of which this file is only one
+part (scope is not drift); comments inside code that disagree with each other (only code-vs-doc); drift you
+can only verify by RUNNING the code.
+
+Be conservative: prefer false negatives over false positives. An empty findings list is a perfectly valid
+answer.
+
+Output strict JSON, no preamble:
+{"findings":[{"severity":"high|medium|low","code_ref":"file:Lxx or null","doc_ref":"file:Lxx or null",
+"summary":"one sentence","evidence":"quote the conflicting code and doc snippets","suggestion":"the fix"}]}
+If there is no drift, return {"findings": []}.
+"""