compass-kb-validation 3.2.1__tar.gz → 3.4.0__tar.gz

{compass_kb_validation-3.2.1 → compass_kb_validation-3.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: compass-kb-validation
-Version: 3.2.1
+Version: 3.4.0
 Summary: Knowledge-base readiness validation for RAG & AI agents — config-driven, deterministic pipeline integrity checks.
 Author: COMPASS
 License: MIT
@@ -147,6 +147,7 @@ Two modes: **live** validation as documents are ingested, and **post-hoc discove
 ✅ **Pluggable steps** — register custom validators in a step registry; the report adapts automatically  
 ✅ **Deterministic & cheap** — structural/byte/schema checks, no LLM, CI-gateable  
 ✅ **Query & analyze the KB** — ask it questions, expose it to agents over MCP, and preview embedding-model migrations / map retrieval coverage — all deterministic, read-only (see *Beyond validation*)  
+✅ **Eval oracle** — enrich any eval with deterministic KB-grounding signals, then measure how accurate its LLM judges are against that ground truth (`compass enrich` / `compass calibrate-judges`) — the eval framework guesses, COMPASS checks the KB  
 ✅ **Interactive reporting** — self-contained HTML dashboard + JSON + JUnit XML  
 ✅ **Trend Tracking** — SQLite historical performance analysis  
 ✅ **Notifications** — Slack/Teams webhooks for alerts  
@@ -238,8 +239,21 @@ compass freshness --project my-kb --stale-after 30
 # chunks when present (Level 1) and the live KB (Level 2); the answer text + a reachable KB alone
 # still catches fabricated figures and phantom citations.
 compass grounding-audit --project my-kb --eval-result case.json --fail-on-fabrication
+
+# Eval oracle (deterministic, no LLM): enrich an eval with KB-grounded signals, then measure how
+# accurate the eval's OWN LLM judges are against that deterministic truth. LLM judges GUESS whether
+# a response is grounded; COMPASS CHECKS it against the KB - so it can tell you which judge is right.
+compass enrich --project my-kb --input eval.json --output enriched.json
+compass calibrate-judges --input enriched.json --metric groundedness
 ```
 
+`enrich` adds a `compass` block per case (a **citation grounding rate** over the verifiable
+citation+numeric subset, citation fidelity, chunk quality, context utilization, `COMPASS_*` signal
+codes). It is **schema-agnostic** — every field resolves through your `eval.gt_*` config, so it
+works whatever shape your eval JSON takes. `calibrate-judges` then reports each judge's error,
+bias, accuracy, and an evidence-based **recommended weight** — something no eval framework can
+compute, because it needs KB access.
+
 `migrate` and `coverage` also render a **self-contained HTML report** (same look as the
 validation dashboard) with `--html [path] --open` — a coverage heatmap, a migration
 stability distribution, verdicts, and the most-changed / dead-zone tables.

{compass_kb_validation-3.2.1 → compass_kb_validation-3.4.0}/README.md

@@ -80,6 +80,7 @@ Two modes: **live** validation as documents are ingested, and **post-hoc discove
 ✅ **Pluggable steps** — register custom validators in a step registry; the report adapts automatically  
 ✅ **Deterministic & cheap** — structural/byte/schema checks, no LLM, CI-gateable  
 ✅ **Query & analyze the KB** — ask it questions, expose it to agents over MCP, and preview embedding-model migrations / map retrieval coverage — all deterministic, read-only (see *Beyond validation*)  
+✅ **Eval oracle** — enrich any eval with deterministic KB-grounding signals, then measure how accurate its LLM judges are against that ground truth (`compass enrich` / `compass calibrate-judges`) — the eval framework guesses, COMPASS checks the KB  
 ✅ **Interactive reporting** — self-contained HTML dashboard + JSON + JUnit XML  
 ✅ **Trend Tracking** — SQLite historical performance analysis  
 ✅ **Notifications** — Slack/Teams webhooks for alerts  
@@ -171,8 +172,21 @@ compass freshness --project my-kb --stale-after 30
 # chunks when present (Level 1) and the live KB (Level 2); the answer text + a reachable KB alone
 # still catches fabricated figures and phantom citations.
 compass grounding-audit --project my-kb --eval-result case.json --fail-on-fabrication
+
+# Eval oracle (deterministic, no LLM): enrich an eval with KB-grounded signals, then measure how
+# accurate the eval's OWN LLM judges are against that deterministic truth. LLM judges GUESS whether
+# a response is grounded; COMPASS CHECKS it against the KB - so it can tell you which judge is right.
+compass enrich --project my-kb --input eval.json --output enriched.json
+compass calibrate-judges --input enriched.json --metric groundedness
 ```
 
+`enrich` adds a `compass` block per case (a **citation grounding rate** over the verifiable
+citation+numeric subset, citation fidelity, chunk quality, context utilization, `COMPASS_*` signal
+codes). It is **schema-agnostic** — every field resolves through your `eval.gt_*` config, so it
+works whatever shape your eval JSON takes. `calibrate-judges` then reports each judge's error,
+bias, accuracy, and an evidence-based **recommended weight** — something no eval framework can
+compute, because it needs KB access.
+
 `migrate` and `coverage` also render a **self-contained HTML report** (same look as the
 validation dashboard) with `--html [path] --open` — a coverage heatmap, a migration
 stability distribution, verdicts, and the most-changed / dead-zone tables.

{compass_kb_validation-3.2.1 → compass_kb_validation-3.4.0}/compass/__init__.py

@@ -5,4 +5,4 @@ end-to-end: API upload -> object storage -> knowledge graph -> vector store ->
 retrieval quality, plus cross-store consistency.
 """
 
-__version__ = "3.2.1"
+__version__ = "3.4.0"

{compass_kb_validation-3.2.1 → compass_kb_validation-3.4.0}/compass/citation.py

@@ -377,7 +377,19 @@ def verify_level1(citations: list[Citation], eval_chunks: list[dict], cfg: Any)
                 v.terms_missing.append(cv.raw)
                 v._missing.append(cv)
         if not cit.values:
-            v.l1_verdict, v.severity = "UNVERIFIABLE", "INFO"   # no quantitative claim to check
+            # No numeric claim. If the claim carries distinctive entities (proper nouns / quoted
+            # phrases) and they are ALL present in the cited chunk, it is TERM_SUPPORTED - a
+            # weaker, lexical tier counted in a SEPARATE term-grounding rate, never the strict
+            # numeric one. Missing entities stay UNVERIFIABLE (lexical absence can be paraphrase).
+            ents = [] if cit.negated else _extract_entities(cit.claim)
+            if ents:
+                hay = _squash(_haystack(chunk, text_field))
+                v.entity_missing = [e for e in ents if _squash(e) not in hay]
+                v.entities_checked = True
+                v.l1_verdict, v.severity = (("TERM_SUPPORTED", "INFO") if not v.entity_missing
+                                            else ("UNVERIFIABLE", "INFO"))
+            else:
+                v.l1_verdict, v.severity = "UNVERIFIABLE", "INFO"   # no quantitative claim to check
             if cit.negated:
                 v.note = "claim is negated - lexical verification not applicable"
         elif not v.terms_missing:
@@ -539,6 +551,7 @@ _ATTRIB = {
     "citation_error": ("citation_error", "MEDIUM"),  # right fact, wrong cited location
     "fabricated_source": ("fabricated_source", "HIGH"),
     "none": ("none", "INFO"),
+    "term_supported": ("term_supported", "INFO"),  # distinctive entities present in cited chunk
     "unverifiable": ("unverifiable", "INFO"),      # no quantitative term; lexical check N/A
     "unknown": ("unknown", "MEDIUM"),              # a miss we could not attribute (no live KB)
 }
@@ -558,6 +571,8 @@ def attribute(verdicts: list[CitationVerdict], l1_ran: bool) -> None:
             key = "none"
         elif l2 == "FABRICATED_SOURCE":
             key = "fabricated_source"          # cited a document that isn't in the KB
+        elif l1 == "TERM_SUPPORTED":
+            key = "term_supported"             # distinctive entities present; weaker than numeric
         elif l2 in _L2_UNVERIFIABLE:
             key = "unverifiable"               # source unresolved / too large / no numeric claim
         elif l1 == "UNVERIFIABLE" and not l2:
@@ -650,6 +665,7 @@ def audit_citations(response: str, eval_chunks: Optional[list[dict]], adapter: A
     summary = summarize(verdicts, response, l1_ran, l2_ran)
     if l1_ran:
         summary.update(context_utilization(citations, eval_chunks or [], cfg))
+        summary.update(text_utilization(response, eval_chunks or [], cfg))
     # DEFECT-003 safety net: if the response clearly carries citation-like patterns but none
     # were extracted, say so rather than reporting a silent total_citations=0 that reads as
     # "no citations exist" (which was the DEFECT-001 failure mode).
@@ -680,6 +696,31 @@ def context_utilization(citations: list[Citation], eval_chunks: list[dict], cfg:
             "context_utilization_pct": round(100.0 * used / total, 1)}
 
 
+def _ngrams(text: str, n: int = 3) -> set:
+    toks = re.findall(r"[a-z0-9]+", (text or "").lower())
+    return {tuple(toks[i:i + n]) for i in range(len(toks) - n + 1)} if len(toks) >= n else set()
+
+
+def text_utilization(response: str, eval_chunks: list[dict], cfg: Any,
+                     n: int = 3, min_overlap: float = 0.1) -> dict:
+    """How much of the retrieved CONTENT actually appears in the response, by n-gram overlap -
+    distinct from ``context_utilization_pct`` (which measures whether a chunk's PAGE was cited).
+    A chunk counts as used when at least ``min_overlap`` of its n-grams recur in the response."""
+    text_field = getattr(cfg, "gt_chunk_text_field", "text")
+    total = len(eval_chunks or [])
+    if not total:
+        return {}
+    resp = _ngrams(response, n)
+    if not resp:
+        return {"text_utilization_pct": 0.0}
+    used = 0
+    for ch in eval_chunks:
+        cg = _ngrams(_haystack(ch, text_field), n)
+        if cg and len(cg & resp) / len(cg) >= min_overlap:
+            used += 1
+    return {"text_utilization_pct": round(100.0 * used / total, 1)}
+
+
 def summarize(verdicts: list[CitationVerdict], response: str, l1_ran: bool, l2_ran: bool) -> dict:
     """Aggregate signal for the audit: how many citations are sound, and the fault mix."""
     total = len(verdicts)
@@ -693,6 +734,11 @@ def summarize(verdicts: list[CitationVerdict], response: str, l1_ran: bool, l2_r
     checkable = sound + fault_total
     # Accuracy is over CHECKABLE citations only; unverifiable/unknown are excluded, not scored.
     accuracy = round(100.0 * sound / checkable, 1) if checkable else None
+    # Term grounding: a SEPARATE, weaker tier for qualitative citations whose distinctive entities
+    # are present in the cited chunk. Never folded into the strict numeric rate above.
+    term_supported = sum(1 for v in verdicts if v.attribution == "term_supported")
+    term_checkable = sum(1 for v in verdicts if v.entities_checked)
+    term_rate = round(100.0 * term_supported / term_checkable, 1) if term_checkable else None
     # Citation coverage: sentences with a checkable quantitative claim that carry NO citation.
     sentences = [s.strip() for s in re.split(r"(?<=[.!?])\s+", response or "") if s.strip()]
     numeric = [s for s in sentences if extract_terms(s)]
@@ -704,6 +750,9 @@ def summarize(verdicts: list[CitationVerdict], response: str, l1_ran: bool, l2_r
         "checkable": checkable,
         "unverifiable": unverifiable,
         "accuracy_pct": accuracy,
+        "term_supported": term_supported,
+        "term_checkable": term_checkable,
+        "term_grounding_rate_pct": term_rate,
         "faults": faults,
         "coverage_pct": coverage,
         "uncited_numeric_claims": len(uncited),

{compass_kb_validation-3.2.1 → compass_kb_validation-3.4.0}/compass/cli.py

@@ -1701,6 +1701,158 @@ def cmd_selftest(argv: list[str]) -> int:
 # --------------------------------------------------------------------------- #
 # Dispatcher
 # --------------------------------------------------------------------------- #
+def cmd_enrich(argv: list[str]) -> int:
+    p = argparse.ArgumentParser(
+        prog="compass enrich",
+        description="Add deterministic, KB-derived signals to an eval file: a citation grounding "
+                    "rate (over the verifiable citation+numeric subset), citation fidelity, chunk "
+                    "quality, context utilization, and COMPASS_* signal codes. Read-only, no LLM. "
+                    "Schema-agnostic - field locations come from your eval.gt_* config.")
+    p.add_argument("--project")
+    p.add_argument("--env", default="")
+    p.add_argument("--input", required=True, metavar="FILE",
+                   help="Eval file: a JSON array of cases, or a JSON object carrying a case array")
+    p.add_argument("--output", metavar="FILE", help="Write enriched JSON here (default: stdout)")
+    p.add_argument("--no-retrieval", action="store_true",
+                   help="Skip live-KB (Level 2) verification; use only the eval's own chunks "
+                        "(Level 1). No backend needed.")
+    p.add_argument("--json", action="store_true", help="(default) JSON output")
+    args = p.parse_args(argv)
+    _health_env(args)
+
+    import json as _json
+
+    from compass.config import load_settings, project_config_file
+    from compass.eval_enrich import discover_cases, enrich_cases
+
+    if args.project and project_config_file(args.project) is None:
+        print(f"  Project '{args.project}' not found (config/projects/{args.project}.yaml).", file=sys.stderr)
+        return 2
+    try:
+        settings = load_settings(project=args.project, env=args.env or None)
+    except Exception as exc:  # noqa: BLE001 - clean message, not a traceback
+        print(f"  Config error: {exc}", file=sys.stderr)
+        return 2
+    try:
+        with open(args.input, encoding="utf-8") as f:
+            data = _json.load(f)
+    except (OSError, ValueError) as exc:
+        print(f"  Cannot read {args.input}: {exc}", file=sys.stderr)
+        return 2
+    cases = discover_cases(data)
+    if not cases:
+        print(f"  No case array found in {args.input}.", file=sys.stderr)
+        return 2
+
+    # A reachable KB enables Level 2 (KB-verified) attribution; without it, Level 1 signals from
+    # the eval's own chunks still compute. Never fail the command over an unreachable KB.
+    adapter = None
+    if args.project and not args.no_retrieval:
+        try:
+            from compass.providers import get_adapter
+            adapter = get_adapter("vectorstore", settings.redis.provider, config=settings.redis)
+            if not getattr(adapter, "ping", lambda: True)():
+                adapter = None
+        except Exception:  # noqa: BLE001 - degrade to Level 1
+            adapter = None
+
+    print(f"  Enriching {len(cases)} case(s) with deterministic KB signals "
+          f"(level {'1+2' if adapter else '1'}) ...", file=sys.stderr)
+    enriched, summary = enrich_cases(cases, settings.eval, adapter)
+    if isinstance(data, dict):
+        data["compass_summary"] = summary
+        out_obj = data
+    else:
+        out_obj = enriched          # a list: each case now carries a "compass" block; shape preserved
+    body = _json.dumps(out_obj, indent=2, ensure_ascii=False)
+    mean = summary["mean_citation_grounding_rate"]
+    note = (f"{summary['cases_enriched']} enriched, {summary['cases_with_verifiable_claims']} "
+            f"with verifiable claims" + (f", mean grounding rate {mean:.0%}" if mean is not None else ""))
+    if args.output:
+        Path(args.output).parent.mkdir(parents=True, exist_ok=True)
+        Path(args.output).write_text(body + "\n", encoding="utf-8")
+        print(f"  Wrote {args.output} ({note})", file=sys.stderr)
+    else:
+        print(body)
+        print(f"  {note}", file=sys.stderr)
+    return 0
+
+
+def _format_calibration(report: dict, metric: str) -> str:
+    lines = [f"Judge calibration for '{metric}' vs COMPASS deterministic truth "
+             f"({report['cases_scored']} case(s), tolerance +/-{report['tolerance']}):", ""]
+    rows = sorted(report["judges"].items(),
+                  key=lambda kv: (kv[1]["avg_error"] is None, kv[1]["avg_error"] or 0))
+    lines.append(f"  {'judge':<18} {'n':>4} {'error':>8} {'bias':>8} {'acc':>7} {'weight':>8}")
+    for name, v in rows:
+        lean = "" if v["bias"] is None else ("  (lenient)" if v["bias"] > 0.03 else
+                                             "  (strict)" if v["bias"] < -0.03 else "")
+        lines.append(f"  {name:<18} {v['n']:>4} {v['avg_error']:>8.3f} {v['bias']:>+8.3f} "
+                     f"{v['accuracy']:>6.0%} {v['recommended_weight']:>8.3f}{lean}")
+    if report.get("most_accurate"):
+        lines += ["", f"  Most accurate: {report['most_accurate']}. Recommended weights are "
+                      "inverse-error (a more accurate judge is trusted more than equal weighting)."]
+    lines += ["", "  " + report["note"]]
+    return "\n".join(lines)
+
+
+def cmd_calibrate_judges(argv: list[str]) -> int:
+    p = argparse.ArgumentParser(
+        prog="compass calibrate-judges",
+        description="Measure how accurate each LLM judge is against COMPASS's deterministic "
+                    "grounding truth: per-judge error, bias, accuracy, and an inverse-error "
+                    "recommended weight. Input cases must carry both COMPASS truth (run "
+                    "'compass enrich' first) and the eval's per-judge scores. Schema-agnostic.")
+    p.add_argument("--input", required=True, metavar="FILE")
+    p.add_argument("--metric", required=True,
+                   help="Judge metric to calibrate (e.g. 'groundedness'); matched case-insensitively")
+    p.add_argument("--truth-path", default="compass.grounding.citation_grounding_rate",
+                   help="Dotted path to COMPASS truth per case (default: the 'enrich' output field)")
+    p.add_argument("--scores-path", default="",
+                   help="Dotted path to the per-case scores block (default: auto-discover)")
+    p.add_argument("--tolerance", type=float, default=0.2,
+                   help="A judge score within this of the truth counts as accurate (default 0.2)")
+    p.add_argument("--json", action="store_true")
+    args = p.parse_args(argv)
+
+    import json as _json
+
+    from compass.eval_enrich import calibrate_judges, discover_cases, extract_judge_scores
+    from compass.eval_interop import get_path
+
+    try:
+        with open(args.input, encoding="utf-8") as f:
+            data = _json.load(f)
+    except (OSError, ValueError) as exc:
+        print(f"  Cannot read {args.input}: {exc}", file=sys.stderr)
+        return 2
+    cases = discover_cases(data)
+    if not cases:
+        print(f"  No case array found in {args.input}.", file=sys.stderr)
+        return 2
+
+    pairs, truth_found, judges_found = [], 0, 0
+    for c in cases:
+        truth = get_path(c, args.truth_path)
+        judges = extract_judge_scores(c, args.metric, args.scores_path or None)
+        if isinstance(truth, (int, float)) and not isinstance(truth, bool):
+            truth_found += 1
+        if judges:
+            judges_found += 1
+        if isinstance(truth, (int, float)) and not isinstance(truth, bool) and judges:
+            pairs.append({"truth": float(truth), "judges": judges})
+    if not pairs:
+        print(f"  No comparable cases. COMPASS truth found in {truth_found}/{len(cases)} case(s) "
+              f"(path '{args.truth_path}'); judge scores for metric '{args.metric}' in "
+              f"{judges_found}/{len(cases)}. Run 'compass enrich' first, and check --metric, "
+              f"--scores-path, and --truth-path.", file=sys.stderr)
+        return 2
+    report = calibrate_judges(pairs, tolerance=args.tolerance)
+    print(_json.dumps(report, indent=2, ensure_ascii=False) if args.json
+          else _format_calibration(report, args.metric))
+    return 0
+
+
 _HELP = """compass - knowledge-base readiness validation
 
 Usage: compass <command> [options]
@@ -1716,6 +1868,8 @@ Commands:
   freshness        flag stale sources + orphan documents (source object gone)
   evidence         produce live, reproducible PROOF of flagged issues (chunks + queries) - no LLM
   grounding-audit  check an eval's grounding finding against the KB (count claims) - no LLM
+  enrich           add deterministic KB signals (grounding rate, chunk quality) to an eval file
+  calibrate-judges measure each LLM judge's accuracy/bias vs COMPASS's deterministic truth
   demo             generate a sample report from synthetic data (no backend needed)
   init [name]      scaffold a project (config + .env.example + marker). -i/--interactive wizard; --graph graph block; --ci workflow
   lint             statically check a project config for silent-failure traps (no backend)
@@ -1764,6 +1918,10 @@ def main(argv: list[str] | None = None) -> int:
         return cmd_evidence(rest)
     if cmd == "grounding-audit":
         return cmd_grounding_audit(rest)
+    if cmd == "enrich":
+        return cmd_enrich(rest)
+    if cmd == "calibrate-judges":
+        return cmd_calibrate_judges(rest)
     if cmd == "freshness":
         return cmd_freshness(rest)
     if cmd == "selftest":