spancheck 0.1.0__py3-none-any.whl

spancheck/__init__.py

@@ -0,0 +1,47 @@
+"""spancheck — score a grounded-answer system on citation accuracy, abstention correctness,
+hallucination rate, cost and latency, and emit an audit log a compliance reviewer can read.
+
+Public API (Phase 1 — the ported measurement core + the thin adapter contract):
+
+    from spancheck import Case, evaluate, adapter, Run, Scorecard, gate, diff
+
+Deterministic graders live in `spancheck.graders`. Citation-span verification (the capability this build exists to
+add) lands in Phase 2; the versioned audit log in Phase 3; the calibrated LLM-judge in Phase 4. See TODO.md.
+
+Nothing here hard-codes a model provider or depends on any other project's internals (CLAUDE.md design pins).
+"""
+from __future__ import annotations
+
+from .adapter import Output, adapter, normalize
+from .core import (
+    Case,
+    GradeResult,
+    CaseResult,
+    Run,
+    Scorecard,
+    evaluate,
+    run_eval,
+    diff,
+    gate,
+    default_graders,
+)
+from .span import citation_accuracy, verify_citation, CitationVerdict
+from .audit import build_audit_log, cost_latency, SCHEMA_VERSION
+from .judge import judge_entailment, judge_support, llm_judge, load_prompt
+from .calibrate import calibrate_judge
+from . import graders
+
+__version__ = "0.0.1"
+
+__all__ = [
+    # the declared public surface
+    "Case", "evaluate", "adapter", "Run", "Scorecard", "gate", "diff",
+    # citation-span verification (the capability spancheck is named for)
+    "citation_accuracy", "verify_citation", "CitationVerdict",
+    # cost/latency + the versioned audit log
+    "build_audit_log", "cost_latency", "SCHEMA_VERSION",
+    # the calibrated LLM-judge (opt-in; stub by default, no provider lock-in)
+    "judge_entailment", "judge_support", "llm_judge", "load_prompt", "calibrate_judge",
+    # supporting types + the grader library
+    "Output", "normalize", "GradeResult", "CaseResult", "run_eval", "default_graders", "graders",
+]

spancheck/_env.py

@@ -0,0 +1,25 @@
+"""Tiny, dependency-free .env loader.
+
+Reads KEY=VALUE lines from a .env file into os.environ **without overwriting** anything already set (a real
+environment variable always wins). Called at CLI/calibration startup so a key placed in `.env` "just works" — no
+python-dotenv dependency, no vendor lock-in. `.env` is gitignored; a key never enters the repo.
+"""
+from __future__ import annotations
+
+import os
+
+
+def load_dotenv(path: str = ".env") -> None:
+    try:
+        with open(path, encoding="utf-8") as f:
+            for line in f:
+                line = line.strip()
+                if not line or line.startswith("#") or "=" not in line:
+                    continue
+                key, _, value = line.partition("=")
+                key = key.strip()
+                value = value.strip().strip('"').strip("'")
+                if key and key not in os.environ:
+                    os.environ[key] = value
+    except FileNotFoundError:
+        pass

spancheck/_text.py

@@ -0,0 +1,29 @@
+"""Shared text helpers used by graders and the citation-span verifier.
+
+Kept in one place so the deterministic checks agree on what a "word" is and how text is normalised.
+Stdlib only.
+"""
+from __future__ import annotations
+
+import re
+
+WORD = re.compile(r"[a-z0-9]+")
+
+STOP = set("the a an and or of to in on for is are was were be been it its this that with as at by from "
+           "your you our we i they he she them his her their not no do does did have has had will would can "
+           "could should may might must shall into over under about than then so if but".split())
+
+
+def norm(s) -> str:
+    """Lowercase and collapse all whitespace to single spaces — the normalisation used for span provenance."""
+    return re.sub(r"\s+", " ", (s or "")).strip().lower()
+
+
+def words(s) -> set:
+    """All word tokens (lowercased) — used as the haystack when checking coverage."""
+    return set(WORD.findall((s or "").lower()))
+
+
+def content_words(s):
+    """Meaningful tokens: lowercased words that aren't stopwords and are longer than two characters."""
+    return [w for w in WORD.findall((s or "").lower()) if w not in STOP and len(w) > 2]

spancheck/adapter.py

@@ -0,0 +1,90 @@
+"""The thin adapter contract — the seam between spancheck and any system under test.
+
+Design pin #2 (no provider lock-in) and #3 (every metric computable offline from a captured run) both live here:
+a system's answer is normalised **once** into a typed `Output` that carries everything the metrics need
+(the answer, the retrieved contexts, the citations, token/cost usage, and latency), so scoring never has to
+call the system again.
+
+`spancheck` never hard-codes a vendor. A caller supplies a plain `input -> answer` callable; `adapter()` wraps it,
+times it, and normalises whatever it returns. The return may be:
+
+  - a **str** — just the answer text;
+  - a **dict** — with any of `answer`/`text`, `contexts`/`context`, `citations`, `usage`, `latency_ms`, `abstained`;
+  - an **Output** — already normalised (idempotent).
+
+Rewritten from evallab's loose "str-or-dict" handling into a pinned dataclass so cost/latency and citations are
+first-class and always present (see LINEAGE.md, DECISIONS.md AB-DEC 006).
+"""
+from __future__ import annotations
+
+import time
+from dataclasses import dataclass, field, asdict
+
+
+@dataclass
+class Output:
+    """The normalised result of one system call — the single shape every grader reads.
+
+    `citations` is carried through as-is in Phase 1; its shape is firmed up by the citation-span verifier
+    (Phase 2). `abstained` is an explicit signal when the system provides one; when it is None the graders
+    infer abstention from the answer text.
+    """
+    answer: str = ""
+    contexts: list = field(default_factory=list)   # retrieved context passages the answer should be grounded in
+    citations: list = field(default_factory=list)  # citations backing the answer (span shape fixed in Phase 2)
+    usage: dict = field(default_factory=dict)       # e.g. {"input_tokens", "output_tokens", "cost_usd"}
+    latency_ms: float | None = None                 # wall-clock for the call; filled by adapter/evaluate if absent
+    abstained: bool | None = None                   # explicit abstention signal, if the system emits one
+    raw: object = None                              # the untouched original return, for the audit log
+
+    def to_dict(self) -> dict:
+        d = asdict(self)
+        # `raw` may not be JSON-serialisable; keep a string fallback for the audit log rather than crash.
+        try:
+            import json
+            json.dumps(d["raw"])
+        except (TypeError, ValueError):
+            d["raw"] = repr(self.raw)
+        return d
+
+
+def normalize(value) -> Output:
+    """Coerce a system's return (str | dict | Output) into an Output. Idempotent on Output."""
+    if isinstance(value, Output):
+        return value
+    if isinstance(value, dict):
+        answer = value.get("answer", value.get("text", ""))
+        ctx = value.get("contexts", value.get("context", []))
+        if not isinstance(ctx, list):
+            ctx = [ctx] if ctx else []
+        citations = value.get("citations", [])
+        if not isinstance(citations, list):
+            citations = [citations]
+        return Output(
+            answer="" if answer is None else str(answer),
+            contexts=[str(c) for c in ctx],
+            citations=citations,
+            usage=dict(value.get("usage", {}) or {}),
+            latency_ms=value.get("latency_ms"),
+            abstained=value.get("abstained"),
+            raw=value,
+        )
+    # a bare string (or anything else) is just the answer text
+    return Output(answer="" if value is None else str(value), raw=value)
+
+
+def adapter(fn, *, capture_time: bool = True):
+    """Wrap a caller's `input -> result` callable so it returns a normalised, timed `Output`.
+
+    Using `adapter()` is optional — `evaluate()` normalises and times its system either way — but it lets a caller
+    attach usage/citations at the source and hand `evaluate` a system that already speaks the contract.
+    """
+    def run(inp) -> Output:
+        start = time.perf_counter()
+        result = fn(inp)
+        elapsed_ms = (time.perf_counter() - start) * 1000.0
+        out = normalize(result)
+        if capture_time and out.latency_ms is None:
+            out.latency_ms = round(elapsed_ms, 3)
+        return out
+    return run

spancheck/audit.py

@@ -0,0 +1,135 @@
+"""Cost/latency aggregation and the versioned, compliance-readable audit log (Phase 3).
+
+Everything here is computed **offline** from a captured run — no system call, no network (design pin #3). The audit
+log is a contract with a reviewer: its shape is versioned (`schema_version`), and a breaking change to it is a major
+version bump (design pin #4). The schema is documented in docs/AUDIT-LOG.md.
+
+Stdlib only.
+"""
+from __future__ import annotations
+
+import json
+from datetime import datetime, timezone
+
+from .adapter import Output
+from .span import verify_citation
+
+# Bump the MAJOR component only on a breaking change to the audit-log shape (design pin #4).
+SCHEMA_VERSION = "1.0"
+
+
+def _percentile(sorted_vals, pct):
+    """Nearest-rank percentile on an already-sorted list. Handles n=1. pct in [0,100]."""
+    if not sorted_vals:
+        return None
+    if len(sorted_vals) == 1:
+        return sorted_vals[0]
+    k = max(0, min(len(sorted_vals) - 1, round((pct / 100.0) * (len(sorted_vals) - 1))))
+    return sorted_vals[k]
+
+
+def cost_latency(run, pricing=None):
+    """Aggregate cost and latency from a run's captured outputs.
+
+    latency: total / mean / p50 / p95 / max (ms), over cases that recorded a latency.
+    cost: summed input/output/total tokens and any explicit `cost_usd` in usage. If `pricing`
+    ({"input_per_1k", "output_per_1k"}) is given, a dollar cost is computed from tokens; otherwise
+    `cost_usd` is only the sum of explicit per-call costs (or None if none were provided).
+    """
+    lats, in_tok, out_tok, tot_tok, explicit_cost = [], 0, 0, 0, []
+    for cr in run.results:
+        out = cr.output
+        if not isinstance(out, Output):
+            continue
+        if out.latency_ms is not None:
+            lats.append(out.latency_ms)
+        u = out.usage or {}
+        in_tok += int(u.get("input_tokens", 0) or 0)
+        out_tok += int(u.get("output_tokens", 0) or 0)
+        tot_tok += int(u.get("total_tokens", 0) or (int(u.get("input_tokens", 0) or 0) + int(u.get("output_tokens", 0) or 0)))
+        if "cost_usd" in u and u["cost_usd"] is not None:
+            explicit_cost.append(float(u["cost_usd"]))
+
+    lats_sorted = sorted(lats)
+    latency = {
+        "n": len(lats),
+        "total_ms": round(sum(lats), 3) if lats else None,
+        "mean_ms": round(sum(lats) / len(lats), 3) if lats else None,
+        "p50_ms": _percentile(lats_sorted, 50),
+        "p95_ms": _percentile(lats_sorted, 95),
+        "max_ms": max(lats) if lats else None,
+    }
+
+    cost_usd = None
+    if pricing:
+        cost_usd = round(
+            (in_tok / 1000.0) * float(pricing.get("input_per_1k", 0.0))
+            + (out_tok / 1000.0) * float(pricing.get("output_per_1k", 0.0)),
+            6,
+        )
+    elif explicit_cost:
+        cost_usd = round(sum(explicit_cost), 6)
+
+    cost = {
+        "input_tokens": in_tok,
+        "output_tokens": out_tok,
+        "total_tokens": tot_tok,
+        "cost_usd": cost_usd,                # None when no pricing and no explicit cost was supplied (not invented)
+        "pricing_applied": bool(pricing),
+    }
+    return {"latency": latency, "cost": cost}
+
+
+def _citation_verdicts(out: Output):
+    """Per-citation verdicts recomputed from the cached output — the 'why' a reviewer needs."""
+    verdicts = []
+    for c in (out.citations or []):
+        v = verify_citation(c, out.contexts, out.answer)
+        verdicts.append({
+            "span": v.span, "span_found": v.span_found, "claim_supported": v.claim_supported,
+            "ok": v.ok, "support": v.support, "reason": v.reason,
+        })
+    return verdicts
+
+
+def build_audit_log(run, path=None, *, pricing=None, verify_citations=True, spancheck_version=None):
+    """Build the versioned audit log (a superset of Run.to_dict). Writes JSON to `path` if given; returns the dict."""
+    if spancheck_version is None:
+        from . import __version__ as spancheck_version
+
+    sc = run.scorecard()
+    cases = []
+    for cr in run.results:
+        out = cr.output
+        entry = {
+            "case_id": cr.case_id,
+            "category": cr.category,
+            "input": cr.input,
+            "expected": cr.expected,
+            "meta": cr.meta,
+            "error": cr.error,
+            "output": cr.output_dict(),
+            "grades": [{"grader": g.grader, "score": g.score, "passed": g.passed, "detail": g.detail}
+                       for g in cr.grades],
+            "citations": _citation_verdicts(out) if (verify_citations and isinstance(out, Output)) else [],
+        }
+        cases.append(entry)
+
+    doc = {
+        "schema_version": SCHEMA_VERSION,
+        "spancheck_version": spancheck_version,
+        "created_at": datetime.now(timezone.utc).isoformat(),
+        "summary": {
+            "n_cases": len(run.results),
+            "overall_pass_rate": sc.overall_pass_rate(),
+            "by_grader": sc.by_grader(),
+            "by_category": sc.by_category(),
+            **cost_latency(run, pricing=pricing),
+        },
+        "cases": cases,
+    }
+
+    if path is not None:
+        with open(path, "w", encoding="utf-8") as f:
+            json.dump(doc, f, indent=2)
+    return doc

spancheck/calibrate.py

@@ -0,0 +1,98 @@
+"""Calibrate the LLM-judge against a human-labelled gold set — a MEASURED agreement number, not an asserted one.
+
+The judge is only trusted after this reports how often it agrees with human labels (AB-DEC 009). With no provider the
+stub is measured (a deterministic baseline that catches negation but misses paraphrase — visible in the confusion
+counts). The real number comes from a keyed run:
+
+    python -m spancheck.calibrate --real --model claude-sonnet-5    # needs ANTHROPIC_API_KEY
+
+Stdlib only. The gold set (calibration/entailment_gold.jsonl) is synthetic and public.
+"""
+from __future__ import annotations
+
+import json
+
+from .judge import judge_entailment
+
+
+def calibrate_judge(gold, provider=None, prompt_version="entailment_v1", threshold=0.5, model=None):
+    """Run the judge over labelled (claim, span, label) examples; return agreement + confusion + per-row detail."""
+    tp = fp = tn = fn = correct = 0
+    rows = []
+    for ex in gold:
+        v = judge_entailment(ex["claim"], ex["span"], provider=provider, prompt_version=prompt_version, model=model)
+        pred = v["score"] >= threshold
+        label = bool(ex["label"])
+        ok = pred == label
+        correct += int(ok)
+        if pred and label:
+            tp += 1
+        elif pred and not label:
+            fp += 1
+        elif not pred and not label:
+            tn += 1
+        else:
+            fn += 1
+        rows.append({"claim": ex["claim"], "span": ex["span"], "label": label,
+                     "score": v["score"], "pred": pred, "correct": ok, "reason": v["reason"]})
+    n = len(gold)
+    return {
+        "n": n,
+        "agreement": round(correct / n, 3) if n else 0.0,
+        "threshold": threshold,
+        "confusion": {"tp": tp, "fp": fp, "tn": tn, "fn": fn},
+        "model": model or ("stub" if provider is None else getattr(provider, "model", "unknown")),
+        "prompt_version": prompt_version,
+        "rows": rows,
+    }
+
+
+def load_gold(path):
+    with open(path, encoding="utf-8") as f:
+        return [json.loads(line) for line in f if line.strip()]
+
+
+def main(argv=None) -> int:
+    import argparse
+    import sys
+    try:
+        sys.stdout.reconfigure(encoding="utf-8")
+    except Exception:
+        pass
+
+    p = argparse.ArgumentParser(prog="spancheck.calibrate",
+                                description="Measure judge-vs-human agreement on a gold set.")
+    p.add_argument("--gold", default="calibration/entailment_gold.jsonl")
+    p.add_argument("--threshold", type=float, default=0.5)
+    p.add_argument("--real", action="store_true", help="use the Anthropic provider (needs ANTHROPIC_API_KEY)")
+    p.add_argument("--model", default="claude-sonnet-5")
+    a = p.parse_args(argv)
+
+    from ._env import load_dotenv
+    load_dotenv()  # pick up ANTHROPIC_API_KEY from a local .env if present (never overwrites a real env var)
+
+    gold = load_gold(a.gold)
+    provider = None
+    model = None
+    if a.real:
+        from .provider import anthropic_provider
+        provider = anthropic_provider(model=a.model)
+        model = a.model
+
+    r = calibrate_judge(gold, provider=provider, threshold=a.threshold, model=model)
+    print(f"judge calibration — {r['model']} vs {r['n']} human-labelled cases (prompt {r['prompt_version']})")
+    print(f"  agreement: {r['agreement']}   (threshold {r['threshold']})")
+    c = r["confusion"]
+    print(f"  confusion: tp={c['tp']}  fp={c['fp']}  tn={c['tn']}  fn={c['fn']}")
+    for row in r["rows"]:
+        mark = "ok" if row["correct"] else "XX"
+        print(f"  [{mark}] label={str(row['label']):5} pred={str(row['pred']):5} score={row['score']:<5} "
+              f"{row['claim'][:44]!r} ⟵ {row['span'][:44]!r}")
+    if not a.real:
+        print("\n(stub judge — a deterministic offline baseline. For the real agreement number: "
+              "python -m spancheck.calibrate --real)")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

spancheck/cli.py

@@ -0,0 +1,148 @@
+"""spancheck CLI — a thin wrapper over the library (design pin #1: library-first, CLI-second).
+
+Three commands, each a few lines over the Python API:
+
+    spancheck run   cases.jsonl --target module:callable --out audit.json   # run a system, write an audit log
+    spancheck score audit.json                                             # recompute metrics from a cached run
+    spancheck gate  audit.json citation_accuracy=1.0 [--baseline base.json] # pass/fail for CI (exit 1 on fail)
+
+`run` and `score` never touch the network beyond whatever the target itself does; `score` and `gate` are fully offline.
+"""
+from __future__ import annotations
+
+import argparse
+import importlib
+import json
+import sys
+
+from . import __version__
+from .core import Case, Run, evaluate, gate, default_graders
+
+
+def _load_cases(path):
+    cases = []
+    with open(path, "r", encoding="utf-8") as f:
+        for line in f:
+            line = line.strip()
+            if not line:
+                continue
+            d = json.loads(line)
+            cases.append(Case(d.get("id") or d.get("case_id"), d.get("input"),
+                              d.get("category", "default"), d.get("expected"), d.get("meta") or {}))
+    return cases
+
+
+def _load_target(spec):
+    """Load a system callable from a 'module:attribute' spec (imports arbitrary user code, like a test runner)."""
+    if ":" not in spec:
+        raise SystemExit(f"--target must be 'module:callable', got {spec!r}")
+    mod_name, attr = spec.split(":", 1)
+    obj = importlib.import_module(mod_name)
+    for part in attr.split("."):
+        obj = getattr(obj, part)
+    return obj
+
+
+def _pricing(a):
+    if getattr(a, "price_in", None) is None and getattr(a, "price_out", None) is None:
+        return None
+    return {"input_per_1k": a.price_in or 0.0, "output_per_1k": a.price_out or 0.0}
+
+
+def _print_summary(doc):
+    s = doc["summary"]
+    print(f"cases: {s['n_cases']}   overall pass rate: {s['overall_pass_rate']}")
+    for g, v in s["by_grader"].items():
+        print(f"  {g:<20} pass_rate={v['pass_rate']}  score={v['score']}  n={v['n']}")
+    lat, cost = s["latency"], s["cost"]
+    print(f"  latency: mean={lat['mean_ms']}ms  p95={lat['p95_ms']}ms   "
+          f"tokens={cost['total_tokens']}  cost_usd={cost['cost_usd']}")
+
+
+def cmd_run(a):
+    cases = _load_cases(a.cases)
+    system = _load_target(a.target)
+    run = evaluate(cases, system)
+    doc = run.audit_log(a.out, pricing=_pricing(a))
+    print(f"wrote {a.out}")
+    _print_summary(doc)
+    return 0
+
+
+def cmd_score(a):
+    run = Run.from_audit_log(a.audit)
+    if a.regrade:
+        run = run.rescore(default_graders())  # re-grade cached outputs offline, no system call
+    _print_summary(run.audit_log(pricing=_pricing(a)))
+    return 0
+
+
+def cmd_gate(a):
+    run = Run.from_audit_log(a.audit)
+    thresholds = {}
+    for t in a.thresholds:
+        k, _, v = t.partition("=")
+        if not _:
+            raise SystemExit(f"threshold must be 'grader=min_pass_rate', got {t!r}")
+        thresholds[k] = float(v)
+    baseline = Run.from_audit_log(a.baseline) if a.baseline else None
+    ok, reasons = gate(run, thresholds, baseline=baseline, max_regression=a.max_regression)
+    if ok:
+        print("gate: PASS")
+    else:
+        print("gate: FAIL")
+        for r in reasons:
+            print("  - " + r)
+    return 0 if ok else 1
+
+
+def _build_parser() -> argparse.ArgumentParser:
+    p = argparse.ArgumentParser(
+        prog="spancheck",
+        description="Score a grounded-answer system on citation accuracy, abstention correctness, "
+                    "hallucination rate, cost and latency — with a compliance-readable audit log.",
+    )
+    p.add_argument("--version", action="version", version=f"spancheck {__version__}")
+    sub = p.add_subparsers(dest="command")
+
+    run = sub.add_parser("run", help="run a test set against a target and write an audit log")
+    run.add_argument("cases", help="path to the cases file (JSONL: one {id,input,category?,expected?,meta?} per line)")
+    run.add_argument("--target", required=True, help="module:callable adapter for the system under test")
+    run.add_argument("--out", default="audit.json", help="where to write the audit log")
+    run.add_argument("--price-in", type=float, default=None, help="USD per 1k input tokens (enables cost)")
+    run.add_argument("--price-out", type=float, default=None, help="USD per 1k output tokens (enables cost)")
+    run.set_defaults(func=cmd_run)
+
+    score = sub.add_parser("score", help="recompute metrics from a cached audit log (no network)")
+    score.add_argument("audit", help="path to a captured audit log")
+    score.add_argument("--regrade", action="store_true", help="re-run the default graders on the cached outputs")
+    score.add_argument("--price-in", type=float, default=None)
+    score.add_argument("--price-out", type=float, default=None)
+    score.set_defaults(func=cmd_score)
+
+    g = sub.add_parser("gate", help="pass/fail a run against thresholds (exit 1 on fail; for CI)")
+    g.add_argument("audit", help="path to a captured audit log")
+    g.add_argument("thresholds", nargs="*", help="grader=min_pass_rate (e.g. citation_accuracy=1.0 groundedness=0.9)")
+    g.add_argument("--baseline", default="", help="optional baseline audit log for regression gating")
+    g.add_argument("--max-regression", type=float, default=0.0, help="max allowed pass_rate drop vs baseline")
+    g.set_defaults(func=cmd_gate)
+    return p
+
+
+def main(argv=None) -> int:
+    try:
+        sys.stdout.reconfigure(encoding="utf-8")
+    except Exception:
+        pass
+    from ._env import load_dotenv
+    load_dotenv()  # a target that calls a model can read its key from a local .env (never overwrites a real env var)
+    parser = _build_parser()
+    args = parser.parse_args(argv)
+    if not getattr(args, "command", None):
+        parser.print_help()
+        return 0
+    return args.func(args)
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())