compass-kb-validation 2.2.0__tar.gz → 2.4.0__tar.gz

{compass_kb_validation-2.2.0 → compass_kb_validation-2.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: compass-kb-validation
-Version: 2.2.0
+Version: 2.4.0
 Summary: Knowledge-base readiness validation for RAG & AI agents — config-driven, deterministic pipeline integrity checks.
 Author: COMPASS
 License: MIT
@@ -228,6 +228,10 @@ compass ask --project my-kb --batch questions.jsonl --format eval --output eval_
 # Corpus health (deterministic, no LLM): score chunk quality / find stale + orphan sources
 compass quality   --project my-kb
 compass freshness --project my-kb --stale-after 30
+
+# Adjudicate an eval's grounding finding against the KB: did the bot's counts match reality?
+# Deterministic - attributes a failure to a layer (retrieval vs the model), never scores answer quality.
+compass grounding-audit --project my-kb --eval-result case.json --fail-on-fabrication
 ```
 
 `migrate` and `coverage` also render a **self-contained HTML report** (same look as the

{compass_kb_validation-2.2.0 → compass_kb_validation-2.4.0}/README.md

@@ -161,6 +161,10 @@ compass ask --project my-kb --batch questions.jsonl --format eval --output eval_
 # Corpus health (deterministic, no LLM): score chunk quality / find stale + orphan sources
 compass quality   --project my-kb
 compass freshness --project my-kb --stale-after 30
+
+# Adjudicate an eval's grounding finding against the KB: did the bot's counts match reality?
+# Deterministic - attributes a failure to a layer (retrieval vs the model), never scores answer quality.
+compass grounding-audit --project my-kb --eval-result case.json --fail-on-fabrication
 ```
 
 `migrate` and `coverage` also render a **self-contained HTML report** (same look as the

{compass_kb_validation-2.2.0 → compass_kb_validation-2.4.0}/compass_kb_validation.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: compass-kb-validation
-Version: 2.2.0
+Version: 2.4.0
 Summary: Knowledge-base readiness validation for RAG & AI agents — config-driven, deterministic pipeline integrity checks.
 Author: COMPASS
 License: MIT
@@ -228,6 +228,10 @@ compass ask --project my-kb --batch questions.jsonl --format eval --output eval_
 # Corpus health (deterministic, no LLM): score chunk quality / find stale + orphan sources
 compass quality   --project my-kb
 compass freshness --project my-kb --stale-after 30
+
+# Adjudicate an eval's grounding finding against the KB: did the bot's counts match reality?
+# Deterministic - attributes a failure to a layer (retrieval vs the model), never scores answer quality.
+compass grounding-audit --project my-kb --eval-result case.json --fail-on-fabrication
 ```
 
 `migrate` and `coverage` also render a **self-contained HTML report** (same look as the

{compass_kb_validation-2.2.0 → compass_kb_validation-2.4.0}/compass_kb_validation.egg-info/SOURCES.txt

@@ -23,6 +23,7 @@ ingestion_validation/corpus_health.py
 ingestion_validation/diagnostics.py
 ingestion_validation/embedding_drift.py
 ingestion_validation/eval_interop.py
+ingestion_validation/grounding.py
 ingestion_validation/init_wizard.py
 ingestion_validation/live_dashboard.py
 ingestion_validation/mcp_server.py
@@ -82,6 +83,7 @@ tests/test_eval_interop.py
 tests/test_gate.py
 tests/test_graph_transport.py
 tests/test_graph_validators.py
+tests/test_grounding.py
 tests/test_init_wizard.py
 tests/test_manifest.py
 tests/test_mcp.py

{compass_kb_validation-2.2.0 → compass_kb_validation-2.4.0}/discover_and_validate.py

@@ -67,6 +67,12 @@ def parse_args(argv=None) -> argparse.Namespace:
                    help="Also fail unless the KB Hygiene score is >= PCT (0-100). Hygiene is "
                         "the share of advisory data-quality checks that passed - data-plane "
                         "cleanliness, separate from the READY integrity gate.")
+    p.add_argument("--suggest-config", nargs="?", const="-", metavar="PATH",
+                   help="After the run, emit config refinements from what it observed "
+                        "(document count, chunk-size distribution, checks that never applied). "
+                        "Prints to stdout, or writes to PATH. Review before merging.")
+    p.add_argument("--no-tips", action="store_true",
+                   help="Suppress the end-of-run tips about relevant unused flags.")
     # --- config overrides (run scenarios without editing YAML; CLI wins over config) ---
     p.add_argument("--readiness-threshold", type=float, metavar="FRAC",
                    help="Override corpus readiness cutoff (0-1). READY when pass-rate >= this.")
@@ -136,8 +142,8 @@ def _resolve_skip_steps(settings, *, include_graphdb: bool, include_retrieval: b
         skip += ["GraphDB Metadata", "Neptune Metadata"]
     if not include_retrieval:
         skip.append("Retrieval Quality")
-    skip += _normalize_skips(list(getattr(settings, "skip_steps", None) or []))
-    skip += _normalize_skips(cli_skip)
+    skip += _normalize_skips(list(getattr(settings, "skip_steps", None) or []), source="config")
+    skip += _normalize_skips(cli_skip, source="cli")
 
     s3_auto_skipped = False
     if settings.discovery.provider in ("vectorstore", "index", "manifest"):
@@ -149,8 +155,14 @@ def _resolve_skip_steps(settings, *, include_graphdb: bool, include_retrieval: b
     return list(dict.fromkeys(skip)), s3_auto_skipped
 
 
-def _normalize_skips(tokens: list[str]) -> list[str]:
-    """Map --skip tokens (step keys or names) to the step names the pipeline skips by."""
+def _normalize_skips(tokens: list[str], source: str = "cli") -> list[str]:
+    """Map --skip tokens (step keys or names) to the step names the pipeline skips by.
+
+    An unknown entry from an interactive ``--skip`` is worth flagging (the user just typed
+    it and expects it to work). An unknown entry from the project's ``skip_steps`` is
+    config drift (e.g. a step removed in an upgrade) - silent at runtime because it is
+    harmless there and firing every run is noise; ``compass lint`` surfaces it once.
+    """
     from ingestion_validation.validators.registry import registered_steps
     specs = registered_steps()
     by_key = {s.key: s.name for s in specs}
@@ -161,7 +173,7 @@ def _normalize_skips(tokens: list[str]) -> list[str]:
             out.append(by_key[t])
         elif t in names:
             out.append(t)
-        else:
+        elif source == "cli":
             print(f"  (skip: unknown step '{t}'; valid keys: {', '.join(sorted(by_key))})")
     return out
 
@@ -240,6 +252,32 @@ _AUTH_HINTS = ("403", "forbidden", "expired", "credential", "token", "signatured
                "invalidclienttokenid", "accessdenied", "unauthor")
 
 
+def _print_tips(args, hygiene: dict | None, saved_baseline: bool) -> None:
+    """End-of-run hints about the most relevant unused flags.
+
+    Discoverability, not noise: at most three, only when the flag was NOT already used and
+    the run's own outcome makes it relevant. Suppress with --no-tips. This is why a partner
+    can run for weeks without finding --save-baseline / --unreachable-as / --suggest-config.
+    """
+    tips: list[str] = []
+    if not saved_baseline:
+        tips.append("Save this run as a baseline to catch regressions next time: "
+                    "--save-baseline baselines/{}.json".format(args.project or "kb"))
+    if hygiene and hygiene.get("score", 100) < 90 and args.quality_threshold is None:
+        tips.append("Gate CI on data-plane hygiene (not just integrity): "
+                    "--quality-threshold {}".format(int(hygiene["score"])))
+    if not getattr(args, "include_retrieval", False):
+        tips.append("Add retrieval-quality checks (can a stored vector be found via KNN?): "
+                    "--include-retrieval")
+    if not args.suggest_config:
+        tips.append("Refine your config from what this run observed: --suggest-config")
+    if not tips:
+        return
+    print("\n  Tips (--no-tips to silence):")
+    for t in tips[:3]:
+        print(f"    * {t}")
+
+
 def _resolve_expected_documents(raw, project: str):
     """Resolve the configured expected_documents, supporting ``auto``.
 
@@ -476,13 +514,18 @@ def main(argv=None) -> int:
     # Corpus-level knowledge-base readiness (completeness, dedup, coverage).
     from ingestion_validation.corpus import analyze_corpus
 
+    # Read corpus expectations from the canonical `corpus:` block, falling back to the
+    # legacy `discovery.extra` location so older configs keep working.
+    cc = settings.corpus
+    dx = settings.discovery.extra
     expected_documents = _resolve_expected_documents(
-        settings.discovery.extra.get("expected_documents"), args.project)
+        cc.expected_documents if cc.expected_documents is not None else dx.get("expected_documents"),
+        args.project)
     corpus = analyze_corpus(
         documents, results,
         expected_documents=expected_documents,
-        expected_folders=settings.discovery.extra.get("expected_folders"),
-        expected_document_names=settings.discovery.extra.get("expected_document_names"),
+        expected_folders=cc.expected_folders or dx.get("expected_folders"),
+        expected_document_names=cc.expected_document_names or dx.get("expected_document_names"),
         readiness_threshold=settings.pipeline.readiness_threshold,
     )
     print(f"  KB readiness: {corpus['verdict']} ({corpus['passed']}/{corpus['total']} passed, "
@@ -502,6 +545,16 @@ def main(argv=None) -> int:
     if args.output_format in ("junit", "all"):
         print(f"  JUnit report: {export_junit(results, base + '.xml')}")
 
+    if args.suggest_config:
+        from ingestion_validation.profiler import suggest_config_from_run
+        snippet = suggest_config_from_run(results, documents)
+        if args.suggest_config == "-":
+            print("\n" + snippet)
+        else:
+            from pathlib import Path as _P
+            _P(args.suggest_config).write_text(snippet, encoding="utf-8")
+            print(f"  Config suggestions written: {args.suggest_config} (review before merging)")
+
     # KB Hygiene: data-plane cleanliness (advisory checks), separate from the integrity
     # gate. Printed for every run; gates the build only when --quality-threshold is set.
     # Best-effort context - a failure to score it must not crash the gate decision.
@@ -539,6 +592,8 @@ def main(argv=None) -> int:
         reason = f"{len(documents)} documents < s3.min_documents ({min_docs})"
         failed_gate = True
     code = EXIT_GATE_FAILED if (failed_gate or (regression and args.fail_on_regression)) else EXIT_OK
+    if not args.no_tips:
+        _print_tips(args, hygiene, saved_baseline=args.save_baseline is not None or bool(args.against))
     print("=" * 60)
     print(f"  Results: {passed} passed, {failed} failed ({len(results)} total)")
     print(f"  Gate: {'FAILED' if failed_gate else 'PASSED'} - {reason}")

{compass_kb_validation-2.2.0 → compass_kb_validation-2.4.0}/ingestion_validation/__init__.py

@@ -5,4 +5,4 @@ end-to-end: API upload -> object storage -> knowledge graph -> vector store ->
 retrieval quality, plus cross-store consistency.
 """
 
-__version__ = "2.2.0"
+__version__ = "2.4.0"

{compass_kb_validation-2.2.0 → compass_kb_validation-2.4.0}/ingestion_validation/cli.py

@@ -1302,6 +1302,72 @@ def cmd_freshness(argv: list[str]) -> int:
     return 0
 
 
+def cmd_grounding_audit(argv: list[str]) -> int:
+    p = argparse.ArgumentParser(
+        prog="compass grounding-audit",
+        description="Adjudicate an eval's grounding finding against the live KB. Deterministic: "
+                    "verifies count/aggregation claims exactly against the KB and attributes the "
+                    "failure to a layer (retrieval vs the model). Does not judge answer quality.")
+    p.add_argument("--project")
+    p.add_argument("--env", default="")
+    g = p.add_mutually_exclusive_group(required=True)
+    g.add_argument("--eval-result", metavar="FILE", help="One eval record as JSON.")
+    g.add_argument("--eval-results", metavar="FILE", help="JSONL of eval records (one per line).")
+    p.add_argument("--format", choices=["markdown", "json"], default="markdown")
+    p.add_argument("--output", help="Write the report to a file (default: stdout).")
+    p.add_argument("--fail-on-fabrication", action="store_true",
+                   help="Exit non-zero if any verifiable claim disagrees with the KB.")
+    args = p.parse_args(argv)
+    _health_env(args)
+
+    import json as _json
+
+    from ingestion_validation.config import load_settings, project_config_file
+    from ingestion_validation.grounding import render_report, run_audit
+    from ingestion_validation.providers import get_adapter
+
+    if args.project and project_config_file(args.project) is None:
+        print(f"  Project '{args.project}' not found (config/projects/{args.project}.yaml).")
+        return 2
+    settings = load_settings(project=args.project, env=args.env or None)
+
+    # Load the eval record(s).
+    records: list[dict] = []
+    try:
+        if args.eval_result:
+            records = [_json.loads(Path(args.eval_result).read_text(encoding="utf-8"))]
+        else:
+            for line in Path(args.eval_results).read_text(encoding="utf-8").splitlines():
+                if line.strip():
+                    records.append(_json.loads(line))
+    except Exception as exc:  # noqa: BLE001 - clean message, never a traceback
+        print(f"  Could not read eval input: {exc}")
+        return 2
+
+    adapter = get_adapter("vectorstore", settings.redis.provider, config=settings.redis)
+    results = [run_audit(rec, adapter, settings.eval) for rec in records]
+
+    if args.format == "json":
+        from dataclasses import asdict
+        body = _json.dumps([asdict(r) for r in results], indent=2, default=str)
+    else:
+        body = ("\n\n".join(render_report(r, project=args.project or "") for r in results))
+
+    if args.output:
+        Path(args.output).parent.mkdir(parents=True, exist_ok=True)
+        Path(args.output).write_text(body + "\n", encoding="utf-8")
+        print(f"  Wrote {args.output}")
+    else:
+        print(body)
+
+    fabricated = sum(1 for r in results if r.wrong_claims)
+    if fabricated:
+        print(f"\n  {fabricated}/{len(results)} case(s) contain claims that disagree with the KB.")
+        if args.fail_on_fabrication:
+            return 1
+    return 0
+
+
 # --------------------------------------------------------------------------- #
 # compass selftest - prove exit-code propagation in THIS shell (BUG-14/15 diagnosis)
 # --------------------------------------------------------------------------- #
@@ -1355,6 +1421,7 @@ Commands:
   compare          compare retrieval against a ground-truth file (precision/recall/Jaccard)
   quality          score chunk quality (density, self-containedness, boilerplate) - no LLM
   freshness        flag stale sources + orphan documents (source object gone)
+  grounding-audit  check an eval's grounding finding against the KB (count claims) - no LLM
   demo             generate a sample report from synthetic data (no backend needed)
   init [name]      scaffold a project (config + .env.example + marker). -i/--interactive wizard; --graph graph block; --ci workflow
   lint             statically check a project config for silent-failure traps (no backend)
@@ -1399,6 +1466,8 @@ def main(argv: list[str] | None = None) -> int:
         return cmd_compare(rest)
     if cmd == "quality":
         return cmd_quality(rest)
+    if cmd == "grounding-audit":
+        return cmd_grounding_audit(rest)
     if cmd == "freshness":
         return cmd_freshness(rest)
     if cmd == "selftest":

{compass_kb_validation-2.2.0 → compass_kb_validation-2.4.0}/ingestion_validation/config/__init__.py

@@ -534,6 +534,22 @@ class DiscoveryConfig:
     extra: dict[str, Any] = field(default_factory=dict)
 
 
+@dataclass
+class CorpusConfig:
+    """Corpus-level expectations: what SHOULD be in the knowledge base.
+
+    The natural home for the expected document set, so completeness alerts on drift. This
+    is the block users reach for (`corpus:`); previously it was neither recognized nor read
+    (the values had to live under `discovery.extra`, which few discover), so a
+    `corpus.expected_documents` was silently ignored. Now first-class and read directly.
+    """
+
+    expected_documents: Any = None                 # int, or "auto" (baseline from last run)
+    expected_folders: list[str] = field(default_factory=list)
+    expected_document_names: list[str] = field(default_factory=list)
+    extra: dict[str, Any] = field(default_factory=dict)
+
+
 @dataclass
 class EmbeddingConfig:
     """How to turn a natural-language query into a vector for `compass ask`.
@@ -582,6 +598,21 @@ class EvalConfig:
     gt_variant_path: str = ""
     gt_variant_filter: list[str] = field(default_factory=list)
 
+    # --- Grounding audit (compass grounding-audit) ------------------------------------
+    # Verify an eval's grounding finding against the live KB. Every field name here comes
+    # from config, never assumed - a KB whose ingestion named things differently only
+    # changes these values, not code.
+    gt_response_path: str = "response"                    # the answer text in the eval record
+    gt_context_path: str = "context.knowledge_retrieval"  # the retrieved context the eval saw
+    gt_scores_path: str = "scores"                        # eval scores (shown in the report)
+    # Structured-query verification: which fields the KB can aggregate to check a count
+    # claim. Each entry: {field: <group-by field>, distinct_by: <doc-identity field>,
+    # label: <optional human label>}. Empty = count claims are not verified.
+    aggregatable_fields: list[dict] = field(default_factory=list)
+    # Extra query-intent regexes, merged with the built-in defaults (how many / count / by
+    # type / list all / most recent ...). Lets a domain add its own phrasings.
+    structured_query_patterns: list[str] = field(default_factory=list)
+
 
 @dataclass
 class ConsistencyConfig:
@@ -640,6 +671,7 @@ class Settings:
     embedding: EmbeddingConfig = field(default_factory=EmbeddingConfig)
     eval: EvalConfig = field(default_factory=EvalConfig)
     consistency: ConsistencyConfig = field(default_factory=ConsistencyConfig)
+    corpus: CorpusConfig = field(default_factory=CorpusConfig)
     # Non-service metadata carried through from config files.
     project_name: str = ""
     display_name: str = ""
@@ -661,7 +693,7 @@ _META_KEYS = frozenset({"default_env", "display_name", "environments", "skip_ste
 # new block can never be added to one and forgotten in the other (which is exactly how
 # `consistency` came to be silently dropped at the top level).
 _SERVICE_BLOCKS = ("api", "s3", "graphdb", "redis", "neptune", "pipeline",
-                   "retrieval", "discovery", "embedding", "eval", "consistency")
+                   "retrieval", "discovery", "embedding", "eval", "consistency", "corpus")
 
 
 def _deep_merge(base: dict, overlay: dict) -> dict:
@@ -787,6 +819,7 @@ def load_settings(
         embedding=_hydrate(EmbeddingConfig, flat.get("embedding")),
         eval=_hydrate(EvalConfig, flat.get("eval")),
         consistency=_hydrate(ConsistencyConfig, flat.get("consistency")),
+        corpus=_hydrate(CorpusConfig, flat.get("corpus")),
         project_name=project or "",
         display_name=merged.get("display_name", ""),
         environment=resolved_env,