compass-kb-validation 3.2.1__tar.gz → 3.3.0__tar.gz

{compass_kb_validation-3.2.1 → compass_kb_validation-3.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: compass-kb-validation
-Version: 3.2.1
+Version: 3.3.0
 Summary: Knowledge-base readiness validation for RAG & AI agents — config-driven, deterministic pipeline integrity checks.
 Author: COMPASS
 License: MIT
@@ -147,6 +147,7 @@ Two modes: **live** validation as documents are ingested, and **post-hoc discove
 ✅ **Pluggable steps** — register custom validators in a step registry; the report adapts automatically  
 ✅ **Deterministic & cheap** — structural/byte/schema checks, no LLM, CI-gateable  
 ✅ **Query & analyze the KB** — ask it questions, expose it to agents over MCP, and preview embedding-model migrations / map retrieval coverage — all deterministic, read-only (see *Beyond validation*)  
+✅ **Eval oracle** — enrich any eval with deterministic KB-grounding signals, then measure how accurate its LLM judges are against that ground truth (`compass enrich` / `compass calibrate-judges`) — the eval framework guesses, COMPASS checks the KB  
 ✅ **Interactive reporting** — self-contained HTML dashboard + JSON + JUnit XML  
 ✅ **Trend Tracking** — SQLite historical performance analysis  
 ✅ **Notifications** — Slack/Teams webhooks for alerts  
@@ -238,8 +239,21 @@ compass freshness --project my-kb --stale-after 30
 # chunks when present (Level 1) and the live KB (Level 2); the answer text + a reachable KB alone
 # still catches fabricated figures and phantom citations.
 compass grounding-audit --project my-kb --eval-result case.json --fail-on-fabrication
+
+# Eval oracle (deterministic, no LLM): enrich an eval with KB-grounded signals, then measure how
+# accurate the eval's OWN LLM judges are against that deterministic truth. LLM judges GUESS whether
+# a response is grounded; COMPASS CHECKS it against the KB - so it can tell you which judge is right.
+compass enrich --project my-kb --input eval.json --output enriched.json
+compass calibrate-judges --input enriched.json --metric groundedness
 ```
 
+`enrich` adds a `compass` block per case (a **citation grounding rate** over the verifiable
+citation+numeric subset, citation fidelity, chunk quality, context utilization, `COMPASS_*` signal
+codes). It is **schema-agnostic** — every field resolves through your `eval.gt_*` config, so it
+works whatever shape your eval JSON takes. `calibrate-judges` then reports each judge's error,
+bias, accuracy, and an evidence-based **recommended weight** — something no eval framework can
+compute, because it needs KB access.
+
 `migrate` and `coverage` also render a **self-contained HTML report** (same look as the
 validation dashboard) with `--html [path] --open` — a coverage heatmap, a migration
 stability distribution, verdicts, and the most-changed / dead-zone tables.

{compass_kb_validation-3.2.1 → compass_kb_validation-3.3.0}/README.md

@@ -80,6 +80,7 @@ Two modes: **live** validation as documents are ingested, and **post-hoc discove
 ✅ **Pluggable steps** — register custom validators in a step registry; the report adapts automatically  
 ✅ **Deterministic & cheap** — structural/byte/schema checks, no LLM, CI-gateable  
 ✅ **Query & analyze the KB** — ask it questions, expose it to agents over MCP, and preview embedding-model migrations / map retrieval coverage — all deterministic, read-only (see *Beyond validation*)  
+✅ **Eval oracle** — enrich any eval with deterministic KB-grounding signals, then measure how accurate its LLM judges are against that ground truth (`compass enrich` / `compass calibrate-judges`) — the eval framework guesses, COMPASS checks the KB  
 ✅ **Interactive reporting** — self-contained HTML dashboard + JSON + JUnit XML  
 ✅ **Trend Tracking** — SQLite historical performance analysis  
 ✅ **Notifications** — Slack/Teams webhooks for alerts  
@@ -171,8 +172,21 @@ compass freshness --project my-kb --stale-after 30
 # chunks when present (Level 1) and the live KB (Level 2); the answer text + a reachable KB alone
 # still catches fabricated figures and phantom citations.
 compass grounding-audit --project my-kb --eval-result case.json --fail-on-fabrication
+
+# Eval oracle (deterministic, no LLM): enrich an eval with KB-grounded signals, then measure how
+# accurate the eval's OWN LLM judges are against that deterministic truth. LLM judges GUESS whether
+# a response is grounded; COMPASS CHECKS it against the KB - so it can tell you which judge is right.
+compass enrich --project my-kb --input eval.json --output enriched.json
+compass calibrate-judges --input enriched.json --metric groundedness
 ```
 
+`enrich` adds a `compass` block per case (a **citation grounding rate** over the verifiable
+citation+numeric subset, citation fidelity, chunk quality, context utilization, `COMPASS_*` signal
+codes). It is **schema-agnostic** — every field resolves through your `eval.gt_*` config, so it
+works whatever shape your eval JSON takes. `calibrate-judges` then reports each judge's error,
+bias, accuracy, and an evidence-based **recommended weight** — something no eval framework can
+compute, because it needs KB access.
+
 `migrate` and `coverage` also render a **self-contained HTML report** (same look as the
 validation dashboard) with `--html [path] --open` — a coverage heatmap, a migration
 stability distribution, verdicts, and the most-changed / dead-zone tables.

{compass_kb_validation-3.2.1 → compass_kb_validation-3.3.0}/compass/__init__.py

@@ -5,4 +5,4 @@ end-to-end: API upload -> object storage -> knowledge graph -> vector store ->
 retrieval quality, plus cross-store consistency.
 """
 
-__version__ = "3.2.1"
+__version__ = "3.3.0"

{compass_kb_validation-3.2.1 → compass_kb_validation-3.3.0}/compass/cli.py

@@ -1701,6 +1701,158 @@ def cmd_selftest(argv: list[str]) -> int:
 # --------------------------------------------------------------------------- #
 # Dispatcher
 # --------------------------------------------------------------------------- #
+def cmd_enrich(argv: list[str]) -> int:
+    p = argparse.ArgumentParser(
+        prog="compass enrich",
+        description="Add deterministic, KB-derived signals to an eval file: a citation grounding "
+                    "rate (over the verifiable citation+numeric subset), citation fidelity, chunk "
+                    "quality, context utilization, and COMPASS_* signal codes. Read-only, no LLM. "
+                    "Schema-agnostic - field locations come from your eval.gt_* config.")
+    p.add_argument("--project")
+    p.add_argument("--env", default="")
+    p.add_argument("--input", required=True, metavar="FILE",
+                   help="Eval file: a JSON array of cases, or a JSON object carrying a case array")
+    p.add_argument("--output", metavar="FILE", help="Write enriched JSON here (default: stdout)")
+    p.add_argument("--no-retrieval", action="store_true",
+                   help="Skip live-KB (Level 2) verification; use only the eval's own chunks "
+                        "(Level 1). No backend needed.")
+    p.add_argument("--json", action="store_true", help="(default) JSON output")
+    args = p.parse_args(argv)
+    _health_env(args)
+
+    import json as _json
+
+    from compass.config import load_settings, project_config_file
+    from compass.eval_enrich import discover_cases, enrich_cases
+
+    if args.project and project_config_file(args.project) is None:
+        print(f"  Project '{args.project}' not found (config/projects/{args.project}.yaml).", file=sys.stderr)
+        return 2
+    try:
+        settings = load_settings(project=args.project, env=args.env or None)
+    except Exception as exc:  # noqa: BLE001 - clean message, not a traceback
+        print(f"  Config error: {exc}", file=sys.stderr)
+        return 2
+    try:
+        with open(args.input, encoding="utf-8") as f:
+            data = _json.load(f)
+    except (OSError, ValueError) as exc:
+        print(f"  Cannot read {args.input}: {exc}", file=sys.stderr)
+        return 2
+    cases = discover_cases(data)
+    if not cases:
+        print(f"  No case array found in {args.input}.", file=sys.stderr)
+        return 2
+
+    # A reachable KB enables Level 2 (KB-verified) attribution; without it, Level 1 signals from
+    # the eval's own chunks still compute. Never fail the command over an unreachable KB.
+    adapter = None
+    if args.project and not args.no_retrieval:
+        try:
+            from compass.providers import get_adapter
+            adapter = get_adapter("vectorstore", settings.redis.provider, config=settings.redis)
+            if not getattr(adapter, "ping", lambda: True)():
+                adapter = None
+        except Exception:  # noqa: BLE001 - degrade to Level 1
+            adapter = None
+
+    print(f"  Enriching {len(cases)} case(s) with deterministic KB signals "
+          f"(level {'1+2' if adapter else '1'}) ...", file=sys.stderr)
+    enriched, summary = enrich_cases(cases, settings.eval, adapter)
+    if isinstance(data, dict):
+        data["compass_summary"] = summary
+        out_obj = data
+    else:
+        out_obj = enriched          # a list: each case now carries a "compass" block; shape preserved
+    body = _json.dumps(out_obj, indent=2, ensure_ascii=False)
+    mean = summary["mean_citation_grounding_rate"]
+    note = (f"{summary['cases_enriched']} enriched, {summary['cases_with_verifiable_claims']} "
+            f"with verifiable claims" + (f", mean grounding rate {mean:.0%}" if mean is not None else ""))
+    if args.output:
+        Path(args.output).parent.mkdir(parents=True, exist_ok=True)
+        Path(args.output).write_text(body + "\n", encoding="utf-8")
+        print(f"  Wrote {args.output} ({note})", file=sys.stderr)
+    else:
+        print(body)
+        print(f"  {note}", file=sys.stderr)
+    return 0
+
+
+def _format_calibration(report: dict, metric: str) -> str:
+    lines = [f"Judge calibration for '{metric}' vs COMPASS deterministic truth "
+             f"({report['cases_scored']} case(s), tolerance +/-{report['tolerance']}):", ""]
+    rows = sorted(report["judges"].items(),
+                  key=lambda kv: (kv[1]["avg_error"] is None, kv[1]["avg_error"] or 0))
+    lines.append(f"  {'judge':<18} {'n':>4} {'error':>8} {'bias':>8} {'acc':>7} {'weight':>8}")
+    for name, v in rows:
+        lean = "" if v["bias"] is None else ("  (lenient)" if v["bias"] > 0.03 else
+                                             "  (strict)" if v["bias"] < -0.03 else "")
+        lines.append(f"  {name:<18} {v['n']:>4} {v['avg_error']:>8.3f} {v['bias']:>+8.3f} "
+                     f"{v['accuracy']:>6.0%} {v['recommended_weight']:>8.3f}{lean}")
+    if report.get("most_accurate"):
+        lines += ["", f"  Most accurate: {report['most_accurate']}. Recommended weights are "
+                      "inverse-error (a more accurate judge is trusted more than equal weighting)."]
+    lines += ["", "  " + report["note"]]
+    return "\n".join(lines)
+
+
+def cmd_calibrate_judges(argv: list[str]) -> int:
+    p = argparse.ArgumentParser(
+        prog="compass calibrate-judges",
+        description="Measure how accurate each LLM judge is against COMPASS's deterministic "
+                    "grounding truth: per-judge error, bias, accuracy, and an inverse-error "
+                    "recommended weight. Input cases must carry both COMPASS truth (run "
+                    "'compass enrich' first) and the eval's per-judge scores. Schema-agnostic.")
+    p.add_argument("--input", required=True, metavar="FILE")
+    p.add_argument("--metric", required=True,
+                   help="Judge metric to calibrate (e.g. 'groundedness'); matched case-insensitively")
+    p.add_argument("--truth-path", default="compass.grounding.citation_grounding_rate",
+                   help="Dotted path to COMPASS truth per case (default: the 'enrich' output field)")
+    p.add_argument("--scores-path", default="",
+                   help="Dotted path to the per-case scores block (default: auto-discover)")
+    p.add_argument("--tolerance", type=float, default=0.2,
+                   help="A judge score within this of the truth counts as accurate (default 0.2)")
+    p.add_argument("--json", action="store_true")
+    args = p.parse_args(argv)
+
+    import json as _json
+
+    from compass.eval_enrich import calibrate_judges, discover_cases, extract_judge_scores
+    from compass.eval_interop import get_path
+
+    try:
+        with open(args.input, encoding="utf-8") as f:
+            data = _json.load(f)
+    except (OSError, ValueError) as exc:
+        print(f"  Cannot read {args.input}: {exc}", file=sys.stderr)
+        return 2
+    cases = discover_cases(data)
+    if not cases:
+        print(f"  No case array found in {args.input}.", file=sys.stderr)
+        return 2
+
+    pairs, truth_found, judges_found = [], 0, 0
+    for c in cases:
+        truth = get_path(c, args.truth_path)
+        judges = extract_judge_scores(c, args.metric, args.scores_path or None)
+        if isinstance(truth, (int, float)) and not isinstance(truth, bool):
+            truth_found += 1
+        if judges:
+            judges_found += 1
+        if isinstance(truth, (int, float)) and not isinstance(truth, bool) and judges:
+            pairs.append({"truth": float(truth), "judges": judges})
+    if not pairs:
+        print(f"  No comparable cases. COMPASS truth found in {truth_found}/{len(cases)} case(s) "
+              f"(path '{args.truth_path}'); judge scores for metric '{args.metric}' in "
+              f"{judges_found}/{len(cases)}. Run 'compass enrich' first, and check --metric, "
+              f"--scores-path, and --truth-path.", file=sys.stderr)
+        return 2
+    report = calibrate_judges(pairs, tolerance=args.tolerance)
+    print(_json.dumps(report, indent=2, ensure_ascii=False) if args.json
+          else _format_calibration(report, args.metric))
+    return 0
+
+
 _HELP = """compass - knowledge-base readiness validation
 
 Usage: compass <command> [options]
@@ -1716,6 +1868,8 @@ Commands:
   freshness        flag stale sources + orphan documents (source object gone)
   evidence         produce live, reproducible PROOF of flagged issues (chunks + queries) - no LLM
   grounding-audit  check an eval's grounding finding against the KB (count claims) - no LLM
+  enrich           add deterministic KB signals (grounding rate, chunk quality) to an eval file
+  calibrate-judges measure each LLM judge's accuracy/bias vs COMPASS's deterministic truth
   demo             generate a sample report from synthetic data (no backend needed)
   init [name]      scaffold a project (config + .env.example + marker). -i/--interactive wizard; --graph graph block; --ci workflow
   lint             statically check a project config for silent-failure traps (no backend)
@@ -1764,6 +1918,10 @@ def main(argv: list[str] | None = None) -> int:
         return cmd_evidence(rest)
     if cmd == "grounding-audit":
         return cmd_grounding_audit(rest)
+    if cmd == "enrich":
+        return cmd_enrich(rest)
+    if cmd == "calibrate-judges":
+        return cmd_calibrate_judges(rest)
     if cmd == "freshness":
         return cmd_freshness(rest)
     if cmd == "selftest":

compass_kb_validation-3.3.0/compass/eval_enrich.py

@@ -0,0 +1,305 @@
+"""Deterministic eval enrichment + judge calibration - the "ground-truth oracle".
+
+COMPASS emits deterministic, KB-derived signals for eval cases, and measures how accurate an
+eval's own (LLM-judge) scores are against that deterministic truth. This is the read-only query
+surface applied to an eval pipeline: it imports nothing from the validation pipeline, never calls
+an LLM, and never changes a validation pass/fail - additive, exactly like ``ask`` / ``compare``.
+
+HONEST SCOPE. A fully deterministic groundedness score over *every* claim is not possible without
+an LLM (general prose-claim extraction needs one). COMPASS scores the VERIFIABLE SUBSET only -
+citation-bearing and numeric/aggregation claims - which happens to be exactly where LLM judges
+disagree most, so it is the sharp edge, not a limitation to hide. The signal is therefore named a
+*citation grounding rate*, never "Groundedness", so nothing implies full-claim LLM parity.
+
+Two capabilities:
+  * ``enrich_cases`` - per-case deterministic signals (citation grounding rate + fidelity, chunk
+    quality, context utilization) plus COMPASS_* signal codes.
+  * ``calibrate_judges`` - given each case's deterministic truth and the eval's per-judge scores,
+    report each judge's error, bias, accuracy, and an evidence-based recommended weight. This is
+    the part no eval framework can produce without KB access.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Optional
+
+from compass.eval_interop import get_path
+
+# Deterministic signal codes COMPASS owns. KB-derived only - no answer-quality / guardrail / SQL.
+GROUNDED, PARTIAL, UNGROUNDED = "COMPASS_GROUNDED", "COMPASS_PARTIAL_GROUND", "COMPASS_UNGROUNDED"
+_FAULT_CODES = {
+    "fabricated_source": "COMPASS_FABRICATION",
+    "retrieval": "COMPASS_RETRIEVAL_GAP",
+    "generation": "COMPASS_GENERATION_HALLUCINATION",
+    "citation_error": "COMPASS_CITATION_ERROR",
+}
+# Thresholds (deterministic, overridable by the caller).
+DEFAULTS = {
+    "grounded_at": 0.8,          # citation grounding rate at/above this -> GROUNDED
+    "partial_at": 0.5,           # between partial_at and grounded_at -> PARTIAL_GROUND
+    "context_waste_below_pct": 20.0,
+    "chunk_quality_floor": 0.5,
+    "uncited_numeric_max": 3,
+}
+
+
+def _citation_grounding_rate(summary: dict) -> Optional[float]:
+    """Sound / checkable over the VERIFIABLE subset. None when nothing was checkable (so it is
+    never confused with a real 0.0 - "no verifiable claims" is not "ungrounded")."""
+    checkable = summary.get("checkable", 0) or 0
+    if not checkable:
+        return None
+    return round((summary.get("sound", 0) or 0) / checkable, 4)
+
+
+def _case_chunk_quality(record: dict, cfg: Any, min_chars: int) -> Optional[dict]:
+    """Score the chunks THIS case was given, deterministically. None if no chunks are present."""
+    from compass.corpus_health import _chunk_quality
+
+    chunks = get_path(record, getattr(cfg, "gt_chunks_path", "context.chunks"))
+    if not isinstance(chunks, list) or not chunks:
+        for alt in ("context.chunks", "chunks"):
+            v = get_path(record, alt)
+            if isinstance(v, list) and v:
+                chunks = v
+                break
+    if not isinstance(chunks, list) or not chunks:
+        return None
+    text_field = getattr(cfg, "gt_chunk_text_field", "text") or "text"
+    scores, not_self_contained = [], 0
+    for c in chunks:
+        text = (c.get(text_field) or c.get("text") or "") if isinstance(c, dict) else ""
+        q = _chunk_quality(text, min_chars, boilerplate_lines=set())
+        scores.append(q["score"])
+        if not q["self_contained"]:
+            not_self_contained += 1
+    if not scores:
+        return None
+    return {
+        "mean_score": round(sum(scores) / len(scores), 3),
+        "min_score": round(min(scores), 3),
+        "not_self_contained_pct": round(100.0 * not_self_contained / len(scores), 1),
+        "chunks_scored": len(scores),
+    }
+
+
+def _derive_signals(grounding: dict, chunk_quality: Optional[dict], thr: dict) -> list[dict]:
+    """Turn the deterministic measurements into COMPASS_* signal codes. Every code is
+    check-backed; there is no heuristic guessing here."""
+    signals: list[dict] = []
+    rate = grounding.get("citation_grounding_rate")
+    if rate is not None:
+        if rate >= thr["grounded_at"]:
+            code, sev = GROUNDED, "INFO"
+        elif rate >= thr["partial_at"]:
+            code, sev = PARTIAL, "MEDIUM"
+        else:
+            code, sev = UNGROUNDED, "HIGH"
+        signals.append({"code": code, "severity": sev,
+                        "detail": f"citation grounding rate {rate:.0%} over "
+                                  f"{grounding.get('checkable', 0)} verifiable claim(s)"})
+    for fault, n in (grounding.get("faults") or {}).items():
+        code = _FAULT_CODES.get(fault)
+        if code and n:
+            signals.append({"code": code, "severity": "HIGH", "detail": f"{n} {fault} fault(s)"})
+    util = grounding.get("context_utilization_pct")
+    if util is not None and util < thr["context_waste_below_pct"]:
+        signals.append({"code": "COMPASS_CONTEXT_WASTE", "severity": "LOW",
+                        "detail": f"only {util:.0f}% of retrieved chunks were cited"})
+    if chunk_quality and chunk_quality["min_score"] < thr["chunk_quality_floor"]:
+        signals.append({"code": "COMPASS_CHUNK_QUALITY", "severity": "LOW",
+                        "detail": f"worst chunk score {chunk_quality['min_score']:.2f}, "
+                                  f"{chunk_quality['not_self_contained_pct']:.0f}% not self-contained"})
+    if (grounding.get("uncited_numeric_claims") or 0) > thr["uncited_numeric_max"]:
+        signals.append({"code": "COMPASS_UNCITED_NUMERIC", "severity": "MEDIUM",
+                        "detail": f"{grounding['uncited_numeric_claims']} numeric claim(s) carry no citation"})
+    return signals
+
+
+def enrich_case(record: dict, cfg: Any, adapter: Any = None, *,
+                min_chars: int = 50, thresholds: Optional[dict] = None) -> dict:
+    """Deterministic ``compass`` block for one eval record.
+
+    ``adapter`` optional: with a reachable KB the citation audit also runs Level 2 (KB-verified);
+    without it, Level 1 (against the case's own chunks) still produces a grounding rate. No LLM.
+    """
+    from compass.grounding import run_audit
+
+    thr = {**DEFAULTS, **(thresholds or {})}
+    res = run_audit(record, adapter, cfg)
+    summary = res.citations.get("summary", {}) if isinstance(res.citations, dict) else {}
+    grounding = {
+        "citation_grounding_rate": _citation_grounding_rate(summary),
+        "total_citations": summary.get("total_citations", 0),
+        "checkable": summary.get("checkable", 0),
+        "sound": summary.get("sound", 0),
+        "faults": summary.get("faults", {}),
+        "context_utilization_pct": summary.get("context_utilization_pct"),
+        "uncited_numeric_claims": summary.get("uncited_numeric_claims", 0),
+        "context_empty": res.context_empty,
+        "levels": summary.get("levels", "none"),
+    }
+    if summary.get("diagnostic"):
+        grounding["diagnostic"] = summary["diagnostic"]
+    chunk_quality = _case_chunk_quality(record, cfg, min_chars)
+    block: dict = {"grounding": grounding, "method": "compass_deterministic"}
+    if chunk_quality:
+        block["chunk_quality"] = chunk_quality
+    block["signals"] = _derive_signals(grounding, chunk_quality, thr)
+    return block
+
+
+def enrich_cases(records: list[dict], cfg: Any, adapter: Any = None, *,
+                 min_chars: int = 50, thresholds: Optional[dict] = None) -> tuple[list[dict], dict]:
+    """Enrich every record (adds ``record["compass"]``) and return (records, aggregate summary)."""
+    rates: list[float] = []
+    code_counts: dict[str, int] = {}
+    enriched = 0
+    for rec in records:
+        if not isinstance(rec, dict):
+            continue
+        block = enrich_case(rec, cfg, adapter, min_chars=min_chars, thresholds=thresholds)
+        rec["compass"] = block
+        enriched += 1
+        r = block["grounding"].get("citation_grounding_rate")
+        if r is not None:
+            rates.append(r)
+        for s in block["signals"]:
+            code_counts[s["code"]] = code_counts.get(s["code"], 0) + 1
+    summary = {
+        "cases_enriched": enriched,
+        "cases_with_verifiable_claims": len(rates),
+        "mean_citation_grounding_rate": round(sum(rates) / len(rates), 4) if rates else None,
+        "signal_breakdown": dict(sorted(code_counts.items(), key=lambda kv: (-kv[1], kv[0]))),
+        "note": "Deterministic KB-derived signals over the verifiable (citation + numeric) subset. "
+                "Not an answer-quality judgement.",
+    }
+    return records, summary
+
+
+def discover_cases(data: Any) -> list[dict]:
+    """Find the array of case objects in an arbitrary eval file - COMPASS assumes no fixed shape.
+
+    Accepts a top-level list, or a dict carrying the cases under a common key
+    (cases/results/records/evaluations/data/items), or - failing that - the largest list of
+    objects anywhere at the top level. Returns [] if none is found.
+    """
+    if isinstance(data, list):
+        return [c for c in data if isinstance(c, dict)]
+    if isinstance(data, dict):
+        for key in ("cases", "results", "records", "evaluations", "data", "items"):
+            v = data.get(key)
+            if isinstance(v, list) and v and isinstance(v[0], dict):
+                return v
+        best: list[dict] = []
+        for v in data.values():
+            if isinstance(v, list) and v and isinstance(v[0], dict) and len(v) > len(best):
+                best = v
+        return best
+    return []
+
+
+def _numbers_from(obj: Any) -> dict[str, float]:
+    """Pull judge_name -> score out of whatever sub-shape a metric block uses: a dict of
+    name->number, a dict with a 'judges'/'scores' sub-map, or a list of {judge/model/name, score}."""
+    out: dict[str, float] = {}
+    if isinstance(obj, dict):
+        inner = obj.get("judges") or obj.get("scores") or obj.get("raters")
+        if isinstance(inner, (dict, list)):
+            return _numbers_from(inner)
+        for k, v in obj.items():
+            if isinstance(v, (int, float)) and not isinstance(v, bool) and k.lower() not in (
+                    "score", "spread", "mean", "avg", "average", "std", "count", "n"):
+                out[str(k)] = float(v)
+    elif isinstance(obj, list):
+        for i, item in enumerate(obj):
+            if isinstance(item, dict):
+                name = str(item.get("judge") or item.get("model") or item.get("name") or f"judge_{i}")
+                score = item.get("score")
+                if isinstance(score, (int, float)) and not isinstance(score, bool):
+                    out[name] = float(score)
+    return out
+
+
+def extract_judge_scores(case: dict, metric: str, scores_path: Optional[str] = None) -> dict[str, float]:
+    """Best-effort, schema-agnostic extraction of per-judge scores for ``metric`` from one case.
+
+    ``scores_path`` (optional) points at the scores block; otherwise common locations
+    (scores/metrics/evaluations) are tried. The metric is matched case-insensitively (spaces and
+    underscores treated alike). Returns {judge_name: score}; empty if nothing matched.
+    """
+    blocks = []
+    if scores_path:
+        blocks.append(get_path(case, scores_path))
+    else:
+        for key in ("scores", "metrics", "evaluations", "judges"):
+            if key in case:
+                blocks.append(case.get(key))
+    want = metric.lower().replace(" ", "_")
+    for block in blocks:
+        if not isinstance(block, dict):
+            continue
+        for name, detail in block.items():
+            if name.lower().replace(" ", "_") == want:
+                nums = _numbers_from(detail)
+                if nums:
+                    return nums
+    return {}
+
+
+def calibrate_judges(cases: list[dict], *, tolerance: float = 0.2) -> dict:
+    """Measure each LLM judge against COMPASS's deterministic truth - the oracle capability.
+
+    ``cases``: ``[{"truth": float 0..1, "judges": {judge_name: score 0..1}}]`` where ``truth`` is
+    COMPASS's deterministic grounding rate for that case. For each judge, over the cases where both
+    a truth and that judge's score exist, report:
+      * ``avg_error``   - mean absolute error vs the deterministic truth
+      * ``bias``        - mean signed error (+ = lenient / scores high, - = strict / scores low)
+      * ``accuracy``    - share of scores within ``tolerance`` of the truth
+      * ``recommended_weight`` - inverse-error weight (normalised across judges), so a more
+        accurate judge is trusted more than an equal-weight consensus would.
+    Deterministic and reproducible. No eval framework can compute this - it needs the KB truth.
+    """
+    stats: dict[str, dict] = {}
+    scored = 0
+    for c in cases:
+        truth = c.get("truth")
+        judges = c.get("judges") or {}
+        if not isinstance(truth, (int, float)) or not judges:
+            continue
+        scored += 1
+        for name, score in judges.items():
+            if not isinstance(score, (int, float)):
+                continue
+            s = stats.setdefault(name, {"abs": 0.0, "signed": 0.0, "hits": 0, "n": 0})
+            err = score - truth
+            s["abs"] += abs(err)
+            s["signed"] += err
+            s["hits"] += 1 if abs(err) <= tolerance else 0
+            s["n"] += 1
+
+    judges_out: dict[str, dict] = {}
+    for name, s in stats.items():
+        n = s["n"]
+        judges_out[name] = {
+            "n": n,
+            "avg_error": round(s["abs"] / n, 4) if n else None,
+            "bias": round(s["signed"] / n, 4) if n else None,
+            "accuracy": round(s["hits"] / n, 4) if n else None,
+        }
+    # Recommended weights: inverse average error, normalised. A judge with no scored cases gets 0.
+    inv = {name: (1.0 / (v["avg_error"] + 1e-6)) if v["avg_error"] is not None else 0.0
+           for name, v in judges_out.items()}
+    total_inv = sum(inv.values())
+    for name, v in judges_out.items():
+        v["recommended_weight"] = round(inv[name] / total_inv, 4) if total_inv else None
+    ranked = sorted(judges_out, key=lambda nm: (judges_out[nm]["avg_error"] is None,
+                                                judges_out[nm]["avg_error"] if judges_out[nm]["avg_error"] is not None else 1e9))
+    return {
+        "cases_scored": scored,
+        "tolerance": tolerance,
+        "judges": judges_out,
+        "most_accurate": ranked[0] if ranked else None,
+        "note": "Judge scores measured against COMPASS's deterministic grounding truth "
+                "(citation + numeric subset). Recommended weights are inverse-error, normalised.",
+    }

{compass_kb_validation-3.2.1 → compass_kb_validation-3.3.0}/compass/grounding.py

@@ -422,6 +422,11 @@ def run_audit(eval_record: dict, adapter: Any, eval_cfg: Any, doc_exists: Any =
                          get_path(eval_record, "chunks"))
     has_chunks = any(isinstance(c, list) and len(c) > 0 for c in _chunk_candidates)
     context_empty = _is_empty_context(context) and not has_chunks
+    # If the configured chunk path did not resolve but a common nested one did, use it for the
+    # citation audit too - otherwise a default-path mismatch silently disables Level 1
+    # verification (the same class of schema mismatch as the empty-retrieval false positive).
+    if not (isinstance(eval_chunks, list) and eval_chunks):
+        eval_chunks = next((c for c in _chunk_candidates if isinstance(c, list) and c), eval_chunks)
 
     qtype, intents = classify_query(query, getattr(eval_cfg, "structured_query_patterns", []))
     result = GroundingResult(test_id=test_id, query=query, query_type=qtype, intents=intents,

{compass_kb_validation-3.2.1 → compass_kb_validation-3.3.0}/compass_kb_validation.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: compass-kb-validation
-Version: 3.2.1
+Version: 3.3.0
 Summary: Knowledge-base readiness validation for RAG & AI agents — config-driven, deterministic pipeline integrity checks.
 Author: COMPASS
 License: MIT
@@ -147,6 +147,7 @@ Two modes: **live** validation as documents are ingested, and **post-hoc discove
 ✅ **Pluggable steps** — register custom validators in a step registry; the report adapts automatically  
 ✅ **Deterministic & cheap** — structural/byte/schema checks, no LLM, CI-gateable  
 ✅ **Query & analyze the KB** — ask it questions, expose it to agents over MCP, and preview embedding-model migrations / map retrieval coverage — all deterministic, read-only (see *Beyond validation*)  
+✅ **Eval oracle** — enrich any eval with deterministic KB-grounding signals, then measure how accurate its LLM judges are against that ground truth (`compass enrich` / `compass calibrate-judges`) — the eval framework guesses, COMPASS checks the KB  
 ✅ **Interactive reporting** — self-contained HTML dashboard + JSON + JUnit XML  
 ✅ **Trend Tracking** — SQLite historical performance analysis  
 ✅ **Notifications** — Slack/Teams webhooks for alerts  
@@ -238,8 +239,21 @@ compass freshness --project my-kb --stale-after 30
 # chunks when present (Level 1) and the live KB (Level 2); the answer text + a reachable KB alone
 # still catches fabricated figures and phantom citations.
 compass grounding-audit --project my-kb --eval-result case.json --fail-on-fabrication
+
+# Eval oracle (deterministic, no LLM): enrich an eval with KB-grounded signals, then measure how
+# accurate the eval's OWN LLM judges are against that deterministic truth. LLM judges GUESS whether
+# a response is grounded; COMPASS CHECKS it against the KB - so it can tell you which judge is right.
+compass enrich --project my-kb --input eval.json --output enriched.json
+compass calibrate-judges --input enriched.json --metric groundedness
 ```
 
+`enrich` adds a `compass` block per case (a **citation grounding rate** over the verifiable
+citation+numeric subset, citation fidelity, chunk quality, context utilization, `COMPASS_*` signal
+codes). It is **schema-agnostic** — every field resolves through your `eval.gt_*` config, so it
+works whatever shape your eval JSON takes. `calibrate-judges` then reports each judge's error,
+bias, accuracy, and an evidence-based **recommended weight** — something no eval framework can
+compute, because it needs KB access.
+
 `migrate` and `coverage` also render a **self-contained HTML report** (same look as the
 validation dashboard) with `--html [path] --open` — a coverage heatmap, a migration
 stability distribution, verdicts, and the most-changed / dead-zone tables.

{compass_kb_validation-3.2.1 → compass_kb_validation-3.3.0}/compass_kb_validation.egg-info/SOURCES.txt

@@ -17,6 +17,7 @@ compass/corpus.py
 compass/corpus_health.py
 compass/diagnostics.py
 compass/embedding_drift.py
+compass/eval_enrich.py
 compass/eval_interop.py
 compass/evidence.py
 compass/grounding.py
@@ -86,6 +87,7 @@ tests/test_diagnostics.py
 tests/test_discover_cli.py
 tests/test_discovery_parallel.py
 tests/test_embedding_drift.py
+tests/test_eval_enrich.py
 tests/test_eval_interop.py
 tests/test_evidence.py
 tests/test_gate.py

compass_kb_validation-3.3.0/tests/test_eval_enrich.py

@@ -0,0 +1,109 @@
+"""Deterministic eval enrichment + judge calibration (the ground-truth oracle).
+
+All synthetic and generic - no vendor/domain data, no live services. The point is the
+deterministic scoring math and the schema-agnostic field/shape discovery.
+"""
+
+from __future__ import annotations
+
+from compass.config import Settings
+from compass.eval_enrich import (
+    calibrate_judges,
+    discover_cases,
+    enrich_cases,
+    extract_judge_scores,
+)
+
+
+def _case(output, chunk_text, doc="Doc A", page=1):
+    return {"input": "q", "output": output,
+            "context": {"chunks": [{"text": chunk_text, "document_title": doc, "page_start": page}]}}
+
+
+def test_enrich_scores_a_grounded_numeric_claim():
+    cfg = Settings().eval
+    cases = [_case("Revenue rose 8%. (Doc A, 2026, p. 1) End.", "Revenue rose 8% year over year.")]
+    enriched, summary = enrich_cases(cases, cfg)          # no adapter -> Level 1, no backend
+    g = enriched[0]["compass"]["grounding"]
+    assert g["citation_grounding_rate"] == 1.0 and g["checkable"] == 1 and g["levels"] == "L1"
+    assert "COMPASS_GROUNDED" in [s["code"] for s in enriched[0]["compass"]["signals"]]
+    assert summary["mean_citation_grounding_rate"] == 1.0
+    assert enriched[0]["compass"]["method"] == "compass_deterministic"
+
+
+def test_grounding_rate_is_none_when_nothing_is_verifiable():
+    # Honest subset: a purely qualitative cited claim has nothing deterministically checkable,
+    # so the rate is None (NOT a real 0.0 - "no verifiable claims" is not "ungrounded").
+    cfg = Settings().eval
+    cases = [_case("Growth stays robust. (Doc A, 2026, p. 1) Good.", "Growth stays robust next year.")]
+    enriched, _ = enrich_cases(cases, cfg)
+    g = enriched[0]["compass"]["grounding"]
+    assert g["citation_grounding_rate"] is None and g["checkable"] == 0
+    assert g["total_citations"] == 1                     # the citation was still extracted
+
+
+def test_enrich_is_schema_agnostic_via_config_paths():
+    # A completely different eval shape resolves purely from eval.gt_* config - nothing hardcoded.
+    cfg = Settings().eval
+    cfg.gt_response_path = "answer"
+    cfg.gt_chunks_path = "retrieval.docs"
+    cfg.gt_chunk_text_field = "chunk_text"
+    cfg.gt_chunk_doc_field = "src"
+    cfg.gt_chunk_page_field = "pg"
+    rec = {"query": "q", "answer": "Margins hit 21%. (D, 2026, p. 2) End.",
+           "retrieval": {"docs": [{"chunk_text": "Margins hit 21% in the period.", "src": "D", "pg": 2}]}}
+    enriched, _ = enrich_cases([rec], cfg)
+    assert enriched[0]["compass"]["grounding"]["citation_grounding_rate"] == 1.0
+
+
+def test_calibrate_judges_ranks_by_error_and_weights_inversely():
+    # judge_a is close to the deterministic truth; judge_b is systematically low. The more
+    # accurate judge must rank first and earn the larger recommended weight.
+    cases = [
+        {"truth": 0.9, "judges": {"judge_a": 0.85, "judge_b": 0.55}},
+        {"truth": 0.8, "judges": {"judge_a": 0.82, "judge_b": 0.50}},
+        {"truth": 1.0, "judges": {"judge_a": 0.95, "judge_b": 0.60}},
+    ]
+    r = calibrate_judges(cases, tolerance=0.2)
+    assert r["cases_scored"] == 3 and r["most_accurate"] == "judge_a"
+    a, b = r["judges"]["judge_a"], r["judges"]["judge_b"]
+    assert a["avg_error"] < b["avg_error"]
+    assert a["recommended_weight"] > b["recommended_weight"]
+    assert a["accuracy"] == 1.0 and b["accuracy"] == 0.0
+    assert b["bias"] < 0                                  # judge_b scores low (strict)
+    # weights are a normalised distribution
+    assert abs(a["recommended_weight"] + b["recommended_weight"] - 1.0) < 1e-6
+
+
+def test_calibrate_ignores_cases_missing_truth_or_judges():
+    cases = [
+        {"truth": 0.9, "judges": {"j": 0.9}},
+        {"truth": None, "judges": {"j": 0.5}},            # no truth -> skipped
+        {"judges": {"j": 0.5}},                            # no truth key -> skipped
+        {"truth": 0.8},                                    # no judges -> skipped
+    ]
+    r = calibrate_judges(cases)
+    assert r["cases_scored"] == 1 and r["judges"]["j"]["n"] == 1
+
+
+def test_discover_cases_finds_the_array_in_varied_shapes():
+    assert len(discover_cases([{"a": 1}, {"a": 2}])) == 2          # top-level list
+    assert len(discover_cases({"cases": [{"a": 1}]})) == 1         # common key
+    assert len(discover_cases({"results": [{"a": 1}, {"a": 2}]})) == 2
+    # largest list of objects wins when no known key
+    assert len(discover_cases({"meta": {}, "rows": [{"a": 1}, {"a": 2}, {"a": 3}]})) == 3
+    assert discover_cases({"x": 1}) == []
+
+
+def test_extract_judge_scores_handles_varied_shapes():
+    # dict of name->number nested under 'judges'
+    c1 = {"scores": {"Groundedness": {"judges": {"j1": 0.8, "j2": 0.6}}}}
+    assert extract_judge_scores(c1, "groundedness") == {"j1": 0.8, "j2": 0.6}
+    # list of {judge, score}
+    c2 = {"metrics": {"groundedness": [{"judge": "j1", "score": 0.7}, {"model": "j2", "score": 0.5}]}}
+    assert extract_judge_scores(c2, "Groundedness") == {"j1": 0.7, "j2": 0.5}
+    # case/space-insensitive metric match; aggregate keys (score/spread) are not judges
+    c3 = {"scores": {"answer_relevance": {"judge_x": 0.9, "score": 0.9, "spread": 0.1}}}
+    assert extract_judge_scores(c3, "Answer Relevance") == {"judge_x": 0.9}
+    # metric absent -> empty
+    assert extract_judge_scores({"scores": {"other": {"j": 0.5}}}, "groundedness") == {}