compass-kb-validation 3.2.0__tar.gz → 3.3.0__tar.gz

{compass_kb_validation-3.2.0 → compass_kb_validation-3.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: compass-kb-validation
-Version: 3.2.0
+Version: 3.3.0
 Summary: Knowledge-base readiness validation for RAG & AI agents — config-driven, deterministic pipeline integrity checks.
 Author: COMPASS
 License: MIT
@@ -147,6 +147,7 @@ Two modes: **live** validation as documents are ingested, and **post-hoc discove
 ✅ **Pluggable steps** — register custom validators in a step registry; the report adapts automatically  
 ✅ **Deterministic & cheap** — structural/byte/schema checks, no LLM, CI-gateable  
 ✅ **Query & analyze the KB** — ask it questions, expose it to agents over MCP, and preview embedding-model migrations / map retrieval coverage — all deterministic, read-only (see *Beyond validation*)  
+✅ **Eval oracle** — enrich any eval with deterministic KB-grounding signals, then measure how accurate its LLM judges are against that ground truth (`compass enrich` / `compass calibrate-judges`) — the eval framework guesses, COMPASS checks the KB  
 ✅ **Interactive reporting** — self-contained HTML dashboard + JSON + JUnit XML  
 ✅ **Trend Tracking** — SQLite historical performance analysis  
 ✅ **Notifications** — Slack/Teams webhooks for alerts  
@@ -238,8 +239,21 @@ compass freshness --project my-kb --stale-after 30
 # chunks when present (Level 1) and the live KB (Level 2); the answer text + a reachable KB alone
 # still catches fabricated figures and phantom citations.
 compass grounding-audit --project my-kb --eval-result case.json --fail-on-fabrication
+
+# Eval oracle (deterministic, no LLM): enrich an eval with KB-grounded signals, then measure how
+# accurate the eval's OWN LLM judges are against that deterministic truth. LLM judges GUESS whether
+# a response is grounded; COMPASS CHECKS it against the KB - so it can tell you which judge is right.
+compass enrich --project my-kb --input eval.json --output enriched.json
+compass calibrate-judges --input enriched.json --metric groundedness
 ```
 
+`enrich` adds a `compass` block per case (a **citation grounding rate** over the verifiable
+citation+numeric subset, citation fidelity, chunk quality, context utilization, `COMPASS_*` signal
+codes). It is **schema-agnostic** — every field resolves through your `eval.gt_*` config, so it
+works whatever shape your eval JSON takes. `calibrate-judges` then reports each judge's error,
+bias, accuracy, and an evidence-based **recommended weight** — something no eval framework can
+compute, because it needs KB access.
+
 `migrate` and `coverage` also render a **self-contained HTML report** (same look as the
 validation dashboard) with `--html [path] --open` — a coverage heatmap, a migration
 stability distribution, verdicts, and the most-changed / dead-zone tables.

{compass_kb_validation-3.2.0 → compass_kb_validation-3.3.0}/README.md

@@ -80,6 +80,7 @@ Two modes: **live** validation as documents are ingested, and **post-hoc discove
 ✅ **Pluggable steps** — register custom validators in a step registry; the report adapts automatically  
 ✅ **Deterministic & cheap** — structural/byte/schema checks, no LLM, CI-gateable  
 ✅ **Query & analyze the KB** — ask it questions, expose it to agents over MCP, and preview embedding-model migrations / map retrieval coverage — all deterministic, read-only (see *Beyond validation*)  
+✅ **Eval oracle** — enrich any eval with deterministic KB-grounding signals, then measure how accurate its LLM judges are against that ground truth (`compass enrich` / `compass calibrate-judges`) — the eval framework guesses, COMPASS checks the KB  
 ✅ **Interactive reporting** — self-contained HTML dashboard + JSON + JUnit XML  
 ✅ **Trend Tracking** — SQLite historical performance analysis  
 ✅ **Notifications** — Slack/Teams webhooks for alerts  
@@ -171,8 +172,21 @@ compass freshness --project my-kb --stale-after 30
 # chunks when present (Level 1) and the live KB (Level 2); the answer text + a reachable KB alone
 # still catches fabricated figures and phantom citations.
 compass grounding-audit --project my-kb --eval-result case.json --fail-on-fabrication
+
+# Eval oracle (deterministic, no LLM): enrich an eval with KB-grounded signals, then measure how
+# accurate the eval's OWN LLM judges are against that deterministic truth. LLM judges GUESS whether
+# a response is grounded; COMPASS CHECKS it against the KB - so it can tell you which judge is right.
+compass enrich --project my-kb --input eval.json --output enriched.json
+compass calibrate-judges --input enriched.json --metric groundedness
 ```
 
+`enrich` adds a `compass` block per case (a **citation grounding rate** over the verifiable
+citation+numeric subset, citation fidelity, chunk quality, context utilization, `COMPASS_*` signal
+codes). It is **schema-agnostic** — every field resolves through your `eval.gt_*` config, so it
+works whatever shape your eval JSON takes. `calibrate-judges` then reports each judge's error,
+bias, accuracy, and an evidence-based **recommended weight** — something no eval framework can
+compute, because it needs KB access.
+
 `migrate` and `coverage` also render a **self-contained HTML report** (same look as the
 validation dashboard) with `--html [path] --open` — a coverage heatmap, a migration
 stability distribution, verdicts, and the most-changed / dead-zone tables.

{compass_kb_validation-3.2.0 → compass_kb_validation-3.3.0}/compass/__init__.py

@@ -5,4 +5,4 @@ end-to-end: API upload -> object storage -> knowledge graph -> vector store ->
 retrieval quality, plus cross-store consistency.
 """
 
-__version__ = "3.2.0"
+__version__ = "3.3.0"

{compass_kb_validation-3.2.0 → compass_kb_validation-3.3.0}/compass/citation.py

@@ -245,8 +245,16 @@ def _claim_before(text: str, idx: int) -> str:
     """The statement a citation supports: the text preceding it, back to the last real
     sentence break. Excludes the citation itself and anything after it."""
     start = 0
+    prev_start = 0
     for m in _SENT_BOUND.finditer(text[:idx]):
+        prev_start = start
         start = m.end()
+    # If the last sentence boundary sits right where the citation begins, that period is the
+    # TERMINATOR of the claim sentence (the standard "...claim. (SOURCE, p. N)" format), not a
+    # separator inside the claim - so text[start:idx] would be empty and the citation would be
+    # silently dropped. Fall back one boundary to capture the claim sentence itself.
+    if start >= idx - 2:
+        start = prev_start
     nl = text.rfind("\n", 0, idx)
     if nl + 1 > start:
         start = nl + 1
@@ -642,6 +650,15 @@ def audit_citations(response: str, eval_chunks: Optional[list[dict]], adapter: A
     summary = summarize(verdicts, response, l1_ran, l2_ran)
     if l1_ran:
         summary.update(context_utilization(citations, eval_chunks or [], cfg))
+    # DEFECT-003 safety net: if the response clearly carries citation-like patterns but none
+    # were extracted, say so rather than reporting a silent total_citations=0 that reads as
+    # "no citations exist" (which was the DEFECT-001 failure mode).
+    if not citations and response:
+        raw = len(_PAREN_CITE.findall(response))
+        if raw:
+            summary["diagnostic"] = (
+                f"{raw} citation-like pattern(s) matched by regex but 0 were extracted - "
+                "likely a claim-boundary parsing issue, not an absence of citations.")
     return {"verdicts": verdicts, "l1_ran": l1_ran, "l2_ran": l2_ran, "summary": summary}
 
 

{compass_kb_validation-3.2.0 → compass_kb_validation-3.3.0}/compass/cli.py

@@ -1203,6 +1203,22 @@ def cmd_compare(argv: list[str]) -> int:
     if not cases:
         print(f"  No usable cases in {args.ground_truth} "
               f"(check the eval.gt_* field paths / variant filter in your config).")
+        # Say WHICH paths were tried and what resolved on the first record, so the user can see
+        # the actual mismatch instead of guessing (DEFECT-007).
+        from compass.eval_interop import diagnose_paths
+        diag = diagnose_paths(args.ground_truth, {
+            "gt_question_path": ev.gt_question_path,
+            "gt_chunks_path": ev.gt_chunks_path,
+            "gt_filters_path": ev.gt_filters_path,
+            "gt_variant_path": ev.gt_variant_path,
+        })
+        if diag:
+            print("  Configured paths on the first record:")
+            for line in diag:
+                print(line)
+            if variant_filter:
+                print(f"  variant filter: {', '.join(variant_filter)} "
+                      f"(via gt_variant_path=\"{ev.gt_variant_path or '(unset)'}\")")
         return 2
 
     # Ground-truth cases carry production's own metadata filters. If the backend cannot
@@ -1214,7 +1230,9 @@ def cmd_compare(argv: list[str]) -> int:
               file=sys.stderr)
 
     vnote = f" (variant filter: {', '.join(variant_filter)})" if variant_filter else ""
-    print(f"  Comparing compass retrieval against {len(cases)} ground-truth case(s){vnote} ...")
+    # Progress goes to stderr under --json so stdout stays a clean, pipeable JSON document.
+    print(f"  Comparing compass retrieval against {len(cases)} ground-truth case(s){vnote} ...",
+          file=sys.stderr if args.json else sys.stdout)
     try:
         payload = comparison_report(settings, cases, top_k=args.top_k or None,
                                     skip_empty=not args.include_empty)
@@ -1312,7 +1330,8 @@ def cmd_evidence(argv: list[str]) -> int:
         return 2
     limit = 1 if args.quick else (args.limit or settings.evidence.sample_size)
 
-    print(f"  compass evidence - project={args.project or '(none)'} checks={checks} limit={limit}")
+    print(f"  compass evidence - project={args.project or '(none)'} checks={checks} limit={limit}",
+          file=sys.stderr if args.format == "json" else sys.stdout)
     try:
         payload = ev.gather_evidence(settings, checks, document_filter=args.document, limit=limit)
     except Exception as exc:  # noqa: BLE001 - unreachable/misconfigured backend -> clean message
@@ -1383,7 +1402,8 @@ def cmd_quality(argv: list[str]) -> int:
     except Exception as exc:  # noqa: BLE001 - malformed/empty project YAML -> clean message, not a traceback
         print(f"  Config error: {exc}")
         return 2
-    print(f"  Sampling up to {args.sample} chunks for quality signals ...")
+    print(f"  Sampling up to {args.sample} chunks for quality signals ...",
+          file=sys.stderr if args.json else sys.stdout)
     try:
         payload = quality_report(settings, sample=args.sample, min_chars=args.min_chars)
     except Exception as exc:  # noqa: BLE001 - clean message, never a traceback
@@ -1428,7 +1448,8 @@ def cmd_freshness(argv: list[str]) -> int:
     except Exception as exc:  # noqa: BLE001 - malformed/empty project YAML -> clean message, not a traceback
         print(f"  Config error: {exc}")
         return 2
-    print(f"  Checking source freshness for up to {args.sample} document(s) ...")
+    print(f"  Checking source freshness for up to {args.sample} document(s) ...",
+          file=sys.stderr if args.json else sys.stdout)
     try:
         payload = freshness_report(settings, sample=args.sample, stale_after_days=args.stale_after)
     except Exception as exc:  # noqa: BLE001 - clean message, never a traceback
@@ -1680,6 +1701,158 @@ def cmd_selftest(argv: list[str]) -> int:
 # --------------------------------------------------------------------------- #
 # Dispatcher
 # --------------------------------------------------------------------------- #
+def cmd_enrich(argv: list[str]) -> int:
+    p = argparse.ArgumentParser(
+        prog="compass enrich",
+        description="Add deterministic, KB-derived signals to an eval file: a citation grounding "
+                    "rate (over the verifiable citation+numeric subset), citation fidelity, chunk "
+                    "quality, context utilization, and COMPASS_* signal codes. Read-only, no LLM. "
+                    "Schema-agnostic - field locations come from your eval.gt_* config.")
+    p.add_argument("--project")
+    p.add_argument("--env", default="")
+    p.add_argument("--input", required=True, metavar="FILE",
+                   help="Eval file: a JSON array of cases, or a JSON object carrying a case array")
+    p.add_argument("--output", metavar="FILE", help="Write enriched JSON here (default: stdout)")
+    p.add_argument("--no-retrieval", action="store_true",
+                   help="Skip live-KB (Level 2) verification; use only the eval's own chunks "
+                        "(Level 1). No backend needed.")
+    p.add_argument("--json", action="store_true", help="(default) JSON output")
+    args = p.parse_args(argv)
+    _health_env(args)
+
+    import json as _json
+
+    from compass.config import load_settings, project_config_file
+    from compass.eval_enrich import discover_cases, enrich_cases
+
+    if args.project and project_config_file(args.project) is None:
+        print(f"  Project '{args.project}' not found (config/projects/{args.project}.yaml).", file=sys.stderr)
+        return 2
+    try:
+        settings = load_settings(project=args.project, env=args.env or None)
+    except Exception as exc:  # noqa: BLE001 - clean message, not a traceback
+        print(f"  Config error: {exc}", file=sys.stderr)
+        return 2
+    try:
+        with open(args.input, encoding="utf-8") as f:
+            data = _json.load(f)
+    except (OSError, ValueError) as exc:
+        print(f"  Cannot read {args.input}: {exc}", file=sys.stderr)
+        return 2
+    cases = discover_cases(data)
+    if not cases:
+        print(f"  No case array found in {args.input}.", file=sys.stderr)
+        return 2
+
+    # A reachable KB enables Level 2 (KB-verified) attribution; without it, Level 1 signals from
+    # the eval's own chunks still compute. Never fail the command over an unreachable KB.
+    adapter = None
+    if args.project and not args.no_retrieval:
+        try:
+            from compass.providers import get_adapter
+            adapter = get_adapter("vectorstore", settings.redis.provider, config=settings.redis)
+            if not getattr(adapter, "ping", lambda: True)():
+                adapter = None
+        except Exception:  # noqa: BLE001 - degrade to Level 1
+            adapter = None
+
+    print(f"  Enriching {len(cases)} case(s) with deterministic KB signals "
+          f"(level {'1+2' if adapter else '1'}) ...", file=sys.stderr)
+    enriched, summary = enrich_cases(cases, settings.eval, adapter)
+    if isinstance(data, dict):
+        data["compass_summary"] = summary
+        out_obj = data
+    else:
+        out_obj = enriched          # a list: each case now carries a "compass" block; shape preserved
+    body = _json.dumps(out_obj, indent=2, ensure_ascii=False)
+    mean = summary["mean_citation_grounding_rate"]
+    note = (f"{summary['cases_enriched']} enriched, {summary['cases_with_verifiable_claims']} "
+            f"with verifiable claims" + (f", mean grounding rate {mean:.0%}" if mean is not None else ""))
+    if args.output:
+        Path(args.output).parent.mkdir(parents=True, exist_ok=True)
+        Path(args.output).write_text(body + "\n", encoding="utf-8")
+        print(f"  Wrote {args.output} ({note})", file=sys.stderr)
+    else:
+        print(body)
+        print(f"  {note}", file=sys.stderr)
+    return 0
+
+
+def _format_calibration(report: dict, metric: str) -> str:
+    lines = [f"Judge calibration for '{metric}' vs COMPASS deterministic truth "
+             f"({report['cases_scored']} case(s), tolerance +/-{report['tolerance']}):", ""]
+    rows = sorted(report["judges"].items(),
+                  key=lambda kv: (kv[1]["avg_error"] is None, kv[1]["avg_error"] or 0))
+    lines.append(f"  {'judge':<18} {'n':>4} {'error':>8} {'bias':>8} {'acc':>7} {'weight':>8}")
+    for name, v in rows:
+        lean = "" if v["bias"] is None else ("  (lenient)" if v["bias"] > 0.03 else
+                                             "  (strict)" if v["bias"] < -0.03 else "")
+        lines.append(f"  {name:<18} {v['n']:>4} {v['avg_error']:>8.3f} {v['bias']:>+8.3f} "
+                     f"{v['accuracy']:>6.0%} {v['recommended_weight']:>8.3f}{lean}")
+    if report.get("most_accurate"):
+        lines += ["", f"  Most accurate: {report['most_accurate']}. Recommended weights are "
+                      "inverse-error (a more accurate judge is trusted more than equal weighting)."]
+    lines += ["", "  " + report["note"]]
+    return "\n".join(lines)
+
+
+def cmd_calibrate_judges(argv: list[str]) -> int:
+    p = argparse.ArgumentParser(
+        prog="compass calibrate-judges",
+        description="Measure how accurate each LLM judge is against COMPASS's deterministic "
+                    "grounding truth: per-judge error, bias, accuracy, and an inverse-error "
+                    "recommended weight. Input cases must carry both COMPASS truth (run "
+                    "'compass enrich' first) and the eval's per-judge scores. Schema-agnostic.")
+    p.add_argument("--input", required=True, metavar="FILE")
+    p.add_argument("--metric", required=True,
+                   help="Judge metric to calibrate (e.g. 'groundedness'); matched case-insensitively")
+    p.add_argument("--truth-path", default="compass.grounding.citation_grounding_rate",
+                   help="Dotted path to COMPASS truth per case (default: the 'enrich' output field)")
+    p.add_argument("--scores-path", default="",
+                   help="Dotted path to the per-case scores block (default: auto-discover)")
+    p.add_argument("--tolerance", type=float, default=0.2,
+                   help="A judge score within this of the truth counts as accurate (default 0.2)")
+    p.add_argument("--json", action="store_true")
+    args = p.parse_args(argv)
+
+    import json as _json
+
+    from compass.eval_enrich import calibrate_judges, discover_cases, extract_judge_scores
+    from compass.eval_interop import get_path
+
+    try:
+        with open(args.input, encoding="utf-8") as f:
+            data = _json.load(f)
+    except (OSError, ValueError) as exc:
+        print(f"  Cannot read {args.input}: {exc}", file=sys.stderr)
+        return 2
+    cases = discover_cases(data)
+    if not cases:
+        print(f"  No case array found in {args.input}.", file=sys.stderr)
+        return 2
+
+    pairs, truth_found, judges_found = [], 0, 0
+    for c in cases:
+        truth = get_path(c, args.truth_path)
+        judges = extract_judge_scores(c, args.metric, args.scores_path or None)
+        if isinstance(truth, (int, float)) and not isinstance(truth, bool):
+            truth_found += 1
+        if judges:
+            judges_found += 1
+        if isinstance(truth, (int, float)) and not isinstance(truth, bool) and judges:
+            pairs.append({"truth": float(truth), "judges": judges})
+    if not pairs:
+        print(f"  No comparable cases. COMPASS truth found in {truth_found}/{len(cases)} case(s) "
+              f"(path '{args.truth_path}'); judge scores for metric '{args.metric}' in "
+              f"{judges_found}/{len(cases)}. Run 'compass enrich' first, and check --metric, "
+              f"--scores-path, and --truth-path.", file=sys.stderr)
+        return 2
+    report = calibrate_judges(pairs, tolerance=args.tolerance)
+    print(_json.dumps(report, indent=2, ensure_ascii=False) if args.json
+          else _format_calibration(report, args.metric))
+    return 0
+
+
 _HELP = """compass - knowledge-base readiness validation
 
 Usage: compass <command> [options]
@@ -1695,6 +1868,8 @@ Commands:
   freshness        flag stale sources + orphan documents (source object gone)
   evidence         produce live, reproducible PROOF of flagged issues (chunks + queries) - no LLM
   grounding-audit  check an eval's grounding finding against the KB (count claims) - no LLM
+  enrich           add deterministic KB signals (grounding rate, chunk quality) to an eval file
+  calibrate-judges measure each LLM judge's accuracy/bias vs COMPASS's deterministic truth
   demo             generate a sample report from synthetic data (no backend needed)
   init [name]      scaffold a project (config + .env.example + marker). -i/--interactive wizard; --graph graph block; --ci workflow
   lint             statically check a project config for silent-failure traps (no backend)
@@ -1743,6 +1918,10 @@ def main(argv: list[str] | None = None) -> int:
         return cmd_evidence(rest)
     if cmd == "grounding-audit":
         return cmd_grounding_audit(rest)
+    if cmd == "enrich":
+        return cmd_enrich(rest)
+    if cmd == "calibrate-judges":
+        return cmd_calibrate_judges(rest)
     if cmd == "freshness":
         return cmd_freshness(rest)
     if cmd == "selftest":

{compass_kb_validation-3.2.0 → compass_kb_validation-3.3.0}/compass/corpus_health.py

@@ -261,6 +261,17 @@ def freshness_report(settings: Settings, *, sample: int = 500, stale_after_days:
             f"{without_key} of {len(seen_docs)} sampled document(s) carry no source key, so "
             f"their source could not be checked - likely a missing s3_key field (prefix/field "
             f"mismatch) or sources in a different store, not necessarily orphans.")
+    # A near-total orphan rate almost never means every source was deleted - it points at a
+    # systemic mismatch (the chunk s3_key does not match where objects actually live: wrong
+    # bucket, or a prefix baked into the key that the store does not use). Flag it so the user
+    # checks configuration before "reconciling" hundreds of false orphans (DEFECT-006).
+    if checked and len(orphans) / checked > 0.8:
+        result["warning"] = (
+            f"{len(orphans)}/{checked} sampled sources ({round(100 * len(orphans) / checked)}%) "
+            "look orphaned - a rate this high usually means a key mismatch (wrong bucket, or a "
+            "prefix in the chunk s3_key that the object store does not expect), not that the "
+            "sources are actually gone. Verify s3.bucket_name and the s3_key values before "
+            "reconciling.")
     return result