compass-kb-validation 2.7.0__tar.gz → 2.8.0__tar.gz

{compass_kb_validation-2.7.0 → compass_kb_validation-2.8.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: compass-kb-validation
-Version: 2.7.0
+Version: 2.8.0
 Summary: Knowledge-base readiness validation for RAG & AI agents — config-driven, deterministic pipeline integrity checks.
 Author: COMPASS
 License: MIT

{compass_kb_validation-2.7.0 → compass_kb_validation-2.8.0}/compass_kb_validation.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: compass-kb-validation
-Version: 2.7.0
+Version: 2.8.0
 Summary: Knowledge-base readiness validation for RAG & AI agents — config-driven, deterministic pipeline integrity checks.
 Author: COMPASS
 License: MIT

{compass_kb_validation-2.7.0 → compass_kb_validation-2.8.0}/compass_kb_validation.egg-info/SOURCES.txt

@@ -24,6 +24,7 @@ ingestion_validation/corpus_health.py
 ingestion_validation/diagnostics.py
 ingestion_validation/embedding_drift.py
 ingestion_validation/eval_interop.py
+ingestion_validation/evidence.py
 ingestion_validation/grounding.py
 ingestion_validation/init_wizard.py
 ingestion_validation/live_dashboard.py
@@ -33,6 +34,7 @@ ingestion_validation/profiler.py
 ingestion_validation/provenance.py
 ingestion_validation/remediation.py
 ingestion_validation/retrieval_analysis.py
+ingestion_validation/text_probes.py
 ingestion_validation/trend_tracker.py
 ingestion_validation/config/__init__.py
 ingestion_validation/models/__init__.py
@@ -82,6 +84,7 @@ tests/test_discover_cli.py
 tests/test_discovery_parallel.py
 tests/test_embedding_drift.py
 tests/test_eval_interop.py
+tests/test_evidence.py
 tests/test_gate.py
 tests/test_graph_transport.py
 tests/test_graph_validators.py

{compass_kb_validation-2.7.0 → compass_kb_validation-2.8.0}/ingestion_validation/__init__.py

@@ -5,4 +5,4 @@ end-to-end: API upload -> object storage -> knowledge graph -> vector store ->
 retrieval quality, plus cross-store consistency.
 """
 
-__version__ = "2.7.0"
+__version__ = "2.8.0"

{compass_kb_validation-2.7.0 → compass_kb_validation-2.8.0}/ingestion_validation/cli.py

@@ -1251,6 +1251,91 @@ def _health_env(args) -> None:
         pass
 
 
+def cmd_evidence(argv: list[str]) -> int:
+    p = argparse.ArgumentParser(
+        prog="compass evidence",
+        description="Produce live, reproducible PROOF of the issues a discovery report flags - the "
+                    "actual chunks, queries and scores to attach to a ticket. Read-only, no LLM, no "
+                    "live embedding: probes are built from the KB's own chunks and run the same "
+                    "checks as validation, so a CONFIRMED verdict is proof of what the report found.")
+    p.add_argument("--project")
+    p.add_argument("--env", default="")
+    p.add_argument("--checks", "-c", default=",".join(evidence_mod_all()),
+                   help="Comma-separated: paraphrase,boundary_spanning,exact_term,duplicate_chunks,"
+                        "near_duplicate,chunk_boundaries (default: all).")
+    p.add_argument("--document", "-d", default="", help="Only probe documents whose name contains this.")
+    p.add_argument("--limit", "-l", type=int, default=0, help="Max documents to probe (default: config sample_size).")
+    p.add_argument("--quick", action="store_true", help="Probe a single document.")
+    p.add_argument("--format", "-f", choices=["markdown", "json"], default="markdown")
+    p.add_argument("--output", "-o", help="Write to a file (default: stdout).")
+    args = p.parse_args(argv)
+
+    try:
+        from dotenv import load_dotenv
+        for fn in (".env", f".env.{args.env}" if args.env else None):
+            if fn and Path(fn).exists():
+                load_dotenv(fn, override=True)
+    except Exception:  # noqa: BLE001 - dotenv optional / file absent
+        pass
+
+    from ingestion_validation import evidence as ev
+    from ingestion_validation.config import load_settings, project_config_file
+
+    if args.project and project_config_file(args.project) is None:
+        print(f"  Project '{args.project}' not found (config/projects/{args.project}.yaml).")
+        return 2
+    try:
+        settings = load_settings(project=args.project, env=args.env or None)
+    except Exception as exc:  # noqa: BLE001
+        print(f"  Config error: {exc}")
+        return 2
+
+    valid = set(ev.ALL_CHECKS)
+    checks = [c.strip() for c in args.checks.split(",") if c.strip() in valid]
+    bad = [c.strip() for c in args.checks.split(",") if c.strip() and c.strip() not in valid]
+    if bad:
+        print(f"  Unknown check(s): {', '.join(bad)}. Valid: {', '.join(ev.ALL_CHECKS)}.")
+        return 2
+    if not checks:
+        print("  No valid checks selected.")
+        return 2
+    if args.limit < 0:
+        print("  --limit must be >= 0 (0 uses the configured sample_size).")
+        return 2
+    limit = 1 if args.quick else (args.limit or settings.evidence.sample_size)
+
+    print(f"  compass evidence - project={args.project or '(none)'} checks={checks} limit={limit}")
+    try:
+        payload = ev.gather_evidence(settings, checks, document_filter=args.document, limit=limit)
+    except Exception as exc:  # noqa: BLE001 - unreachable/misconfigured backend -> clean message
+        print(f"  Could not gather evidence: {exc}")
+        return 1
+    if payload.get("error"):
+        print(f"  {payload['error']}")
+        return 1
+
+    body = ev.format_json(payload, args.project or "") if args.format == "json" \
+        else ev.format_markdown(payload, args.project or "")
+    if args.output:
+        Path(args.output).parent.mkdir(parents=True, exist_ok=True)
+        Path(args.output).write_text(body + "\n", encoding="utf-8")
+        print(f"  Wrote {args.output}")
+    else:
+        print(body)
+    counts: dict[str, int] = {}
+    for r in payload["results"]:
+        counts[r.verdict] = counts.get(r.verdict, 0) + 1
+    print(f"  {payload['docs_probed']} doc(s) probed - "
+          f"CONFIRMED {counts.get('CONFIRMED', 0)}, WEAK {counts.get('WEAK', 0)}, "
+          f"not reproduced {counts.get('NOT_REPRODUCED', 0)}, errors {counts.get('ERROR', 0)}")
+    return 0
+
+
+def evidence_mod_all() -> tuple:
+    from ingestion_validation.evidence import ALL_CHECKS
+    return ALL_CHECKS
+
+
 def cmd_quality(argv: list[str]) -> int:
     p = argparse.ArgumentParser(
         prog="compass quality",
@@ -1475,25 +1560,55 @@ def cmd_grounding_audit(argv: list[str]) -> int:
     else:
         print(body)
 
-    # Gate on fabrications in EITHER path: a count that disagrees with the KB, or a cited
-    # document absent from it. (Previously only structured count mismatches counted, so a
-    # semantic fabricated-source case exited 0 despite being flagged.)
-    # Citation audit rollup across cases (deterministic; only counts checkable citations).
-    cit_sound = sum(r.citations.get("summary", {}).get("sound", 0) for r in results)
-    cit_checkable = sum(r.citations.get("summary", {}).get("checkable", 0) for r in results)
-    if cit_checkable:
-        faultmix: dict[str, int] = {}
+    if not results:
+        if args.format != "json":
+            print("\n  No eval records parsed from the input (empty file or no JSON objects).")
+        return 0
+
+    # Human-readable summary lines go to stdout only for the markdown format - in --format json
+    # they would corrupt the JSON body a downstream parser is reading.
+    if args.format != "json":
+        # If the answer field (or others) had to be auto-detected, say so once - a silent field
+        # mismatch is the #1 reason grounding-audit "finds nothing".
+        auto = {}
         for r in results:
-            for k, n in r.citations.get("summary", {}).get("faults", {}).items():
-                faultmix[k] = faultmix.get(k, 0) + n
-        acc = round(100.0 * cit_sound / cit_checkable, 1)
-        print(f"\n  Citation audit: {cit_sound}/{cit_checkable} checkable citations grounded ({acc}%).")
-        if faultmix:
-            print("    faults: " + ", ".join(f"{k} x{n}" for k, n in sorted(faultmix.items())))
+            for kind, info in (r.field_resolution or {}).items():
+                auto.setdefault(kind, info["used"])
+        if auto:
+            mapping = ", ".join(f"{k} <- '{p}'" for k, p in sorted(auto.items()))
+            print(f"\n  NOTE: auto-detected eval fields ({mapping}). "
+                  "Set eval.gt_question_path / gt_response_path / gt_context_path to pin them.")
+
+        # Citation audit rollup across cases (deterministic; only counts checkable citations).
+        total_cits = sum(r.citations.get("summary", {}).get("total_citations", 0) for r in results)
+        cit_sound = sum(r.citations.get("summary", {}).get("sound", 0) for r in results)
+        cit_checkable = sum(r.citations.get("summary", {}).get("checkable", 0) for r in results)
+        if cit_checkable:
+            faultmix: dict[str, int] = {}
+            for r in results:
+                for k, n in r.citations.get("summary", {}).get("faults", {}).items():
+                    faultmix[k] = faultmix.get(k, 0) + n
+            acc = round(100.0 * cit_sound / cit_checkable, 1)
+            print(f"\n  Citation audit: {cit_sound}/{cit_checkable} checkable citations grounded ({acc}%).")
+            if faultmix:
+                print("    faults: " + ", ".join(f"{k} x{n}" for k, n in sorted(faultmix.items())))
+        elif total_cits == 0:
+            # 0 citations is either a retrieval-only export (nothing to audit) or, far more often,
+            # the answer field is not mapped - say which, loudly, instead of a silent INFO.
+            if all(r.retrieval_only for r in results):
+                print("\n  NOTE: these are retrieval-only cases (no generated answer) - nothing to cite-audit.")
+            else:
+                print("\n  WARNING: 0 citations found across "
+                      f"{len(results)} case(s). If the answers DO cite sources, the answer field is "
+                      "probably not mapped - set eval.gt_response_path to your answer field (e.g. "
+                      "'output'/'answer'), and eval.citation_patterns if your citation style is custom.")
 
+    # The fabrication gate drives the exit code regardless of output format (the message is
+    # print-gated above; only the return value matters for --format json).
     fabricated = sum(1 for r in results if r.has_fabrication)
     if fabricated:
-        print(f"\n  {fabricated}/{len(results)} case(s) contain a claim or citation not backed by the KB.")
+        if args.format != "json":
+            print(f"\n  {fabricated}/{len(results)} case(s) contain a claim or citation not backed by the KB.")
         if args.fail_on_fabrication:
             return 1
     return 0
@@ -1552,6 +1667,7 @@ Commands:
   compare          compare retrieval against a ground-truth file (precision/recall/Jaccard)
   quality          score chunk quality (density, self-containedness, boilerplate) - no LLM
   freshness        flag stale sources + orphan documents (source object gone)
+  evidence         produce live, reproducible PROOF of flagged issues (chunks + queries) - no LLM
   grounding-audit  check an eval's grounding finding against the KB (count claims) - no LLM
   demo             generate a sample report from synthetic data (no backend needed)
   init [name]      scaffold a project (config + .env.example + marker). -i/--interactive wizard; --graph graph block; --ci workflow
@@ -1597,6 +1713,8 @@ def main(argv: list[str] | None = None) -> int:
         return cmd_compare(rest)
     if cmd == "quality":
         return cmd_quality(rest)
+    if cmd == "evidence":
+        return cmd_evidence(rest)
     if cmd == "grounding-audit":
         return cmd_grounding_audit(rest)
     if cmd == "freshness":

{compass_kb_validation-2.7.0 → compass_kb_validation-2.8.0}/ingestion_validation/config/__init__.py

@@ -569,6 +569,22 @@ class CorpusConfig:
     extra: dict[str, Any] = field(default_factory=dict)
 
 
+@dataclass
+class EvidenceConfig:
+    """`compass evidence`: how many documents to probe and how strict the confirm thresholds are.
+
+    Evidence is a read-only proof surface; these only tune sampling/sensitivity, never a
+    pass/fail. All deterministic - no LLM, no live embedding.
+    """
+
+    sample_size: int = 10                          # documents probed per run (--limit overrides)
+    output_dir: str = "reports/evidence"
+    # Below this MEAN overlap = paraphrase weakness CONFIRMED. Matches the retrieval-quality
+    # validator's own paraphrase threshold, so evidence reproduces what validation flags.
+    paraphrase_overlap_threshold: float = 0.3
+    near_duplicate_cosine: float = 0.97            # at/above this chunk-embedding cosine = near-duplicate
+
+
 @dataclass
 class EmbeddingConfig:
     """How to turn a natural-language query into a vector for `compass ask`.
@@ -728,6 +744,7 @@ class Settings:
     eval: EvalConfig = field(default_factory=EvalConfig)
     consistency: ConsistencyConfig = field(default_factory=ConsistencyConfig)
     corpus: CorpusConfig = field(default_factory=CorpusConfig)
+    evidence: EvidenceConfig = field(default_factory=EvidenceConfig)
     # Non-service metadata carried through from config files.
     project_name: str = ""
     display_name: str = ""
@@ -749,7 +766,8 @@ _META_KEYS = frozenset({"default_env", "display_name", "environments", "skip_ste
 # new block can never be added to one and forgotten in the other (which is exactly how
 # `consistency` came to be silently dropped at the top level).
 _SERVICE_BLOCKS = ("api", "s3", "graphdb", "redis", "neptune", "pipeline",
-                   "retrieval", "discovery", "embedding", "eval", "consistency", "corpus")
+                   "retrieval", "discovery", "embedding", "eval", "consistency", "corpus",
+                   "evidence")
 
 
 def _deep_merge(base: dict, overlay: dict) -> dict:
@@ -880,6 +898,7 @@ def load_settings(
         eval=_hydrate(EvalConfig, flat.get("eval")),
         consistency=_hydrate(ConsistencyConfig, flat.get("consistency")),
         corpus=_hydrate(CorpusConfig, flat.get("corpus")),
+        evidence=_hydrate(EvidenceConfig, flat.get("evidence")),
         project_name=project or "",
         display_name=merged.get("display_name", ""),
         environment=resolved_env,

{compass_kb_validation-2.7.0 → compass_kb_validation-2.8.0}/ingestion_validation/corpus_health.py

@@ -23,10 +23,24 @@ _SENTENCE_END = tuple(".!?\"')]}")
 # --------------------------------------------------------------------------- #
 # Chunk quality (ENH-6)
 # --------------------------------------------------------------------------- #
-def _chunk_quality(text: str, min_chars: int, boilerplate_lines: set[str]) -> dict:
+def _is_tabular(t: str, digit_ratio: float = 0.4) -> bool:
+    """Heuristic: the chunk is a table / structured data, where low lexical uniqueness and
+    repetition are EXPECTED (repeated column labels, codes, numbers) - not a quality problem.
+    ``digit_ratio`` is config-driven (redis.table_digit_ratio_threshold) so a numeric-but-prose
+    corpus is not misflagged; prose runs ~2-5% digits, tables far higher."""
+    lines = [ln for ln in t.splitlines() if ln.strip()]
+    if sum(1 for ln in lines if ln.count("|") >= 2) >= 2:
+        return True
+    digits = sum(c.isdigit() for c in t)
+    return len(t) > 60 and digits / len(t) > digit_ratio
+
+
+def _chunk_quality(text: str, min_chars: int, boilerplate_lines: set[str],
+                   digit_ratio: float = 0.4) -> dict:
     t = (text or "").strip()
     toks = _WORD.findall(t.lower())
     unique_ratio = len(set(toks)) / len(toks) if toks else 0.0
+    tabular = _is_tabular(t, digit_ratio)
     too_short = len(t) < min_chars
     starts_mid = bool(t[:1]) and t[:1].islower()          # begins mid-sentence
     ends_clean = bool(t) and t[-1] in _SENTENCE_END
@@ -43,7 +57,7 @@ def _chunk_quality(text: str, min_chars: int, boilerplate_lines: set[str]) -> di
     score = round(sum(signals) / len(signals), 3)
     return {"length": len(t), "unique_ratio": round(unique_ratio, 3), "too_short": too_short,
             "self_contained": self_contained, "boilerplate_ratio": round(boiler_ratio, 3),
-            "score": score}
+            "tabular": tabular, "score": score}
 
 
 def quality_report(settings: Settings, *, sample: int = 500, min_chars: int = 50,
@@ -62,9 +76,10 @@ def quality_report(settings: Settings, *, sample: int = 500, min_chars: int = 50
     floor = max(3, int(0.3 * len(chunks)))
     boilerplate_lines = {ln for ln, n in line_counts.items() if n >= floor}
 
+    digit_ratio = float(getattr(settings.redis, "table_digit_ratio_threshold", 0.4) or 0.4)
     per: list[dict] = []
     for c in chunks:
-        q = _chunk_quality(c.get("text", ""), min_chars, boilerplate_lines)
+        q = _chunk_quality(c.get("text", ""), min_chars, boilerplate_lines, digit_ratio)
         q["key"] = c.get("key", "")
         q["text"] = (c.get("text", "") or "")[:80]
         per.append(q)
@@ -77,13 +92,25 @@ def quality_report(settings: Settings, *, sample: int = 500, min_chars: int = 50
     verdict = ("HEALTHY - chunks are dense and self-contained" if mean_score >= 0.75
                else "MIXED - a meaningful share of chunks are short/boilerplate/fragmentary" if mean_score >= 0.5
                else "POOR - many chunks are low-value (short, boilerplate, or mid-sentence)")
+    pct_tabular = round(sum(p.get("tabular", False) for p in per) / n, 3)
+    # Content-type context: the density signal (unique-ratio) is naturally low for tables and
+    # structured data, so a MIXED verdict on a heavily tabular corpus can be expected, not a
+    # defect. Say so rather than letting the number read as poor quality.
+    content_note = ""
+    if pct_tabular >= 0.15:
+        content_note = (f"{pct_tabular:.0%} of sampled chunks look tabular/structured; low "
+                        "lexical uniqueness (information density) is expected for these, so the "
+                        "density component of the score understates their quality. Judge tabular "
+                        "content on retrievability, not density.")
     return {
         "sample": n, "min_chars": min_chars,
         "mean_score": mean_score, "mean_unique_ratio": _mean("unique_ratio"),
         "pct_too_short": round(sum(p["too_short"] for p in per) / n, 3),
         "pct_not_self_contained": round(sum(not p["self_contained"] for p in per) / n, 3),
+        "pct_tabular": pct_tabular,
         "boilerplate_lines": len(boilerplate_lines),
         "verdict": verdict,
+        "content_note": content_note,
         "worst": per[: min(15, n)],
         "per_chunk": per,
     }
@@ -98,8 +125,10 @@ def format_quality(payload: dict) -> str:
         f"too-short: {payload['pct_too_short']:.0%}   not self-contained: {payload['pct_not_self_contained']:.0%}",
         f"  recurring boilerplate lines: {payload['boilerplate_lines']}",
         f"  VERDICT: {payload['verdict']}",
-        "  lowest-quality chunks:",
     ]
+    if payload.get("content_note"):
+        lines.append(f"  NOTE: {payload['content_note']}")
+    lines.append("  lowest-quality chunks:")
     for p in payload["worst"]:
         flags = ",".join(f for f, on in (("short", p["too_short"]),
                          ("mid-sentence", not p["self_contained"]),
@@ -201,6 +230,22 @@ def freshness_report(settings: Settings, *, sample: int = 500, stale_after_days:
         if orphans:
             parts.append(f"{len(orphans)} orphan(s) (source object gone)")
         verdict = "STALE - " + ", ".join(parts)
+
+    # Actionable next steps, not just a count: "36/36 STALE" is alarming without a fix path.
+    remediation: list[str] = []
+    if drift:
+        remediation.append(
+            f"Re-ingest the {len(drift)} drifted document(s) so the KB reflects the current source "
+            "(the source changed after it was embedded).")
+    if stale:
+        remediation.append(
+            f"Refresh the {len(stale)} stale source(s): re-run the ingestion pipeline for their "
+            "prefix/bucket to re-embed the latest content.")
+    if orphans:
+        remediation.append(
+            f"Reconcile the {len(orphans)} orphan(s) (source object gone): if the deletions were "
+            "intentional, delete the corresponding chunks from the index; if not, restore the "
+            "source objects and re-ingest.")
     result = {
         "documents": checked, "sampled_documents": len(seen_docs),
         "without_source_key": without_key, "stale_after_days": stale_after_days,
@@ -209,6 +254,7 @@ def freshness_report(settings: Settings, *, sample: int = 500, stale_after_days:
         "stale": stale[: min(20, len(stale))], "orphans": orphans[: min(20, len(orphans))],
         "drift": drift[: min(20, len(drift))],
         "verdict": verdict,
+        "remediation": remediation,
     }
     if without_key:
         result["note"] = (
@@ -259,4 +305,8 @@ def format_freshness(payload: dict) -> str:
         lines.append("  stalest sources:")
         for s in payload["stale"]:
             lines.append(f"    {s['age_days']}d  {s['doc']}  ({s['s3_key']})")
+    if payload.get("remediation"):
+        lines.append("  REMEDIATION:")
+        for step in payload["remediation"]:
+            lines.append(f"    - {step}")
     return "\n".join(lines)