compass-kb-validation 2.3.0__tar.gz → 2.5.0__tar.gz

{compass_kb_validation-2.3.0 → compass_kb_validation-2.5.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: compass-kb-validation
-Version: 2.3.0
+Version: 2.5.0
 Summary: Knowledge-base readiness validation for RAG & AI agents — config-driven, deterministic pipeline integrity checks.
 Author: COMPASS
 License: MIT
@@ -228,6 +228,10 @@ compass ask --project my-kb --batch questions.jsonl --format eval --output eval_
 # Corpus health (deterministic, no LLM): score chunk quality / find stale + orphan sources
 compass quality   --project my-kb
 compass freshness --project my-kb --stale-after 30
+
+# Adjudicate an eval's grounding finding against the KB: did the bot's counts match reality?
+# Deterministic - attributes a failure to a layer (retrieval vs the model), never scores answer quality.
+compass grounding-audit --project my-kb --eval-result case.json --fail-on-fabrication
 ```
 
 `migrate` and `coverage` also render a **self-contained HTML report** (same look as the

{compass_kb_validation-2.3.0 → compass_kb_validation-2.5.0}/README.md

@@ -161,6 +161,10 @@ compass ask --project my-kb --batch questions.jsonl --format eval --output eval_
 # Corpus health (deterministic, no LLM): score chunk quality / find stale + orphan sources
 compass quality   --project my-kb
 compass freshness --project my-kb --stale-after 30
+
+# Adjudicate an eval's grounding finding against the KB: did the bot's counts match reality?
+# Deterministic - attributes a failure to a layer (retrieval vs the model), never scores answer quality.
+compass grounding-audit --project my-kb --eval-result case.json --fail-on-fabrication
 ```
 
 `migrate` and `coverage` also render a **self-contained HTML report** (same look as the

{compass_kb_validation-2.3.0 → compass_kb_validation-2.5.0}/compass_kb_validation.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: compass-kb-validation
-Version: 2.3.0
+Version: 2.5.0
 Summary: Knowledge-base readiness validation for RAG & AI agents — config-driven, deterministic pipeline integrity checks.
 Author: COMPASS
 License: MIT
@@ -228,6 +228,10 @@ compass ask --project my-kb --batch questions.jsonl --format eval --output eval_
 # Corpus health (deterministic, no LLM): score chunk quality / find stale + orphan sources
 compass quality   --project my-kb
 compass freshness --project my-kb --stale-after 30
+
+# Adjudicate an eval's grounding finding against the KB: did the bot's counts match reality?
+# Deterministic - attributes a failure to a layer (retrieval vs the model), never scores answer quality.
+compass grounding-audit --project my-kb --eval-result case.json --fail-on-fabrication
 ```
 
 `migrate` and `coverage` also render a **self-contained HTML report** (same look as the

{compass_kb_validation-2.3.0 → compass_kb_validation-2.5.0}/compass_kb_validation.egg-info/SOURCES.txt

@@ -23,6 +23,7 @@ ingestion_validation/corpus_health.py
 ingestion_validation/diagnostics.py
 ingestion_validation/embedding_drift.py
 ingestion_validation/eval_interop.py
+ingestion_validation/grounding.py
 ingestion_validation/init_wizard.py
 ingestion_validation/live_dashboard.py
 ingestion_validation/mcp_server.py
@@ -82,6 +83,7 @@ tests/test_eval_interop.py
 tests/test_gate.py
 tests/test_graph_transport.py
 tests/test_graph_validators.py
+tests/test_grounding.py
 tests/test_init_wizard.py
 tests/test_manifest.py
 tests/test_mcp.py

{compass_kb_validation-2.3.0 → compass_kb_validation-2.5.0}/discover_and_validate.py

@@ -142,8 +142,8 @@ def _resolve_skip_steps(settings, *, include_graphdb: bool, include_retrieval: b
         skip += ["GraphDB Metadata", "Neptune Metadata"]
     if not include_retrieval:
         skip.append("Retrieval Quality")
-    skip += _normalize_skips(list(getattr(settings, "skip_steps", None) or []))
-    skip += _normalize_skips(cli_skip)
+    skip += _normalize_skips(list(getattr(settings, "skip_steps", None) or []), source="config")
+    skip += _normalize_skips(cli_skip, source="cli")
 
     s3_auto_skipped = False
     if settings.discovery.provider in ("vectorstore", "index", "manifest"):
@@ -155,8 +155,14 @@ def _resolve_skip_steps(settings, *, include_graphdb: bool, include_retrieval: b
     return list(dict.fromkeys(skip)), s3_auto_skipped
 
 
-def _normalize_skips(tokens: list[str]) -> list[str]:
-    """Map --skip tokens (step keys or names) to the step names the pipeline skips by."""
+def _normalize_skips(tokens: list[str], source: str = "cli") -> list[str]:
+    """Map --skip tokens (step keys or names) to the step names the pipeline skips by.
+
+    An unknown entry from an interactive ``--skip`` is worth flagging (the user just typed
+    it and expects it to work). An unknown entry from the project's ``skip_steps`` is
+    config drift (e.g. a step removed in an upgrade) - silent at runtime because it is
+    harmless there and firing every run is noise; ``compass lint`` surfaces it once.
+    """
     from ingestion_validation.validators.registry import registered_steps
     specs = registered_steps()
     by_key = {s.key: s.name for s in specs}
@@ -167,7 +173,7 @@ def _normalize_skips(tokens: list[str]) -> list[str]:
             out.append(by_key[t])
         elif t in names:
             out.append(t)
-        else:
+        elif source == "cli":
             print(f"  (skip: unknown step '{t}'; valid keys: {', '.join(sorted(by_key))})")
     return out
 
@@ -508,13 +514,18 @@ def main(argv=None) -> int:
     # Corpus-level knowledge-base readiness (completeness, dedup, coverage).
     from ingestion_validation.corpus import analyze_corpus
 
+    # Read corpus expectations from the canonical `corpus:` block, falling back to the
+    # legacy `discovery.extra` location so older configs keep working.
+    cc = settings.corpus
+    dx = settings.discovery.extra
     expected_documents = _resolve_expected_documents(
-        settings.discovery.extra.get("expected_documents"), args.project)
+        cc.expected_documents if cc.expected_documents is not None else dx.get("expected_documents"),
+        args.project)
     corpus = analyze_corpus(
         documents, results,
         expected_documents=expected_documents,
-        expected_folders=settings.discovery.extra.get("expected_folders"),
-        expected_document_names=settings.discovery.extra.get("expected_document_names"),
+        expected_folders=cc.expected_folders or dx.get("expected_folders"),
+        expected_document_names=cc.expected_document_names or dx.get("expected_document_names"),
         readiness_threshold=settings.pipeline.readiness_threshold,
     )
     print(f"  KB readiness: {corpus['verdict']} ({corpus['passed']}/{corpus['total']} passed, "

{compass_kb_validation-2.3.0 → compass_kb_validation-2.5.0}/ingestion_validation/__init__.py

@@ -5,4 +5,4 @@ end-to-end: API upload -> object storage -> knowledge graph -> vector store ->
 retrieval quality, plus cross-store consistency.
 """
 
-__version__ = "2.3.0"
+__version__ = "2.5.0"

{compass_kb_validation-2.3.0 → compass_kb_validation-2.5.0}/ingestion_validation/cli.py

@@ -1302,6 +1302,72 @@ def cmd_freshness(argv: list[str]) -> int:
     return 0
 
 
+def cmd_grounding_audit(argv: list[str]) -> int:
+    p = argparse.ArgumentParser(
+        prog="compass grounding-audit",
+        description="Adjudicate an eval's grounding finding against the live KB. Deterministic: "
+                    "verifies count/aggregation claims exactly against the KB and attributes the "
+                    "failure to a layer (retrieval vs the model). Does not judge answer quality.")
+    p.add_argument("--project")
+    p.add_argument("--env", default="")
+    g = p.add_mutually_exclusive_group(required=True)
+    g.add_argument("--eval-result", metavar="FILE", help="One eval record as JSON.")
+    g.add_argument("--eval-results", metavar="FILE", help="JSONL of eval records (one per line).")
+    p.add_argument("--format", choices=["markdown", "json"], default="markdown")
+    p.add_argument("--output", help="Write the report to a file (default: stdout).")
+    p.add_argument("--fail-on-fabrication", action="store_true",
+                   help="Exit non-zero if any verifiable claim disagrees with the KB.")
+    args = p.parse_args(argv)
+    _health_env(args)
+
+    import json as _json
+
+    from ingestion_validation.config import load_settings, project_config_file
+    from ingestion_validation.grounding import render_report, run_audit
+    from ingestion_validation.providers import get_adapter
+
+    if args.project and project_config_file(args.project) is None:
+        print(f"  Project '{args.project}' not found (config/projects/{args.project}.yaml).")
+        return 2
+    settings = load_settings(project=args.project, env=args.env or None)
+
+    # Load the eval record(s).
+    records: list[dict] = []
+    try:
+        if args.eval_result:
+            records = [_json.loads(Path(args.eval_result).read_text(encoding="utf-8"))]
+        else:
+            for line in Path(args.eval_results).read_text(encoding="utf-8").splitlines():
+                if line.strip():
+                    records.append(_json.loads(line))
+    except Exception as exc:  # noqa: BLE001 - clean message, never a traceback
+        print(f"  Could not read eval input: {exc}")
+        return 2
+
+    adapter = get_adapter("vectorstore", settings.redis.provider, config=settings.redis)
+    results = [run_audit(rec, adapter, settings.eval) for rec in records]
+
+    if args.format == "json":
+        from dataclasses import asdict
+        body = _json.dumps([asdict(r) for r in results], indent=2, default=str)
+    else:
+        body = ("\n\n".join(render_report(r, project=args.project or "") for r in results))
+
+    if args.output:
+        Path(args.output).parent.mkdir(parents=True, exist_ok=True)
+        Path(args.output).write_text(body + "\n", encoding="utf-8")
+        print(f"  Wrote {args.output}")
+    else:
+        print(body)
+
+    fabricated = sum(1 for r in results if r.wrong_claims)
+    if fabricated:
+        print(f"\n  {fabricated}/{len(results)} case(s) contain claims that disagree with the KB.")
+        if args.fail_on_fabrication:
+            return 1
+    return 0
+
+
 # --------------------------------------------------------------------------- #
 # compass selftest - prove exit-code propagation in THIS shell (BUG-14/15 diagnosis)
 # --------------------------------------------------------------------------- #
@@ -1355,6 +1421,7 @@ Commands:
   compare          compare retrieval against a ground-truth file (precision/recall/Jaccard)
   quality          score chunk quality (density, self-containedness, boilerplate) - no LLM
   freshness        flag stale sources + orphan documents (source object gone)
+  grounding-audit  check an eval's grounding finding against the KB (count claims) - no LLM
   demo             generate a sample report from synthetic data (no backend needed)
   init [name]      scaffold a project (config + .env.example + marker). -i/--interactive wizard; --graph graph block; --ci workflow
   lint             statically check a project config for silent-failure traps (no backend)
@@ -1399,6 +1466,8 @@ def main(argv: list[str] | None = None) -> int:
         return cmd_compare(rest)
     if cmd == "quality":
         return cmd_quality(rest)
+    if cmd == "grounding-audit":
+        return cmd_grounding_audit(rest)
     if cmd == "freshness":
         return cmd_freshness(rest)
     if cmd == "selftest":

{compass_kb_validation-2.3.0 → compass_kb_validation-2.5.0}/ingestion_validation/config/__init__.py

@@ -534,6 +534,22 @@ class DiscoveryConfig:
     extra: dict[str, Any] = field(default_factory=dict)
 
 
+@dataclass
+class CorpusConfig:
+    """Corpus-level expectations: what SHOULD be in the knowledge base.
+
+    The natural home for the expected document set, so completeness alerts on drift. This
+    is the block users reach for (`corpus:`); previously it was neither recognized nor read
+    (the values had to live under `discovery.extra`, which few discover), so a
+    `corpus.expected_documents` was silently ignored. Now first-class and read directly.
+    """
+
+    expected_documents: Any = None                 # int, or "auto" (baseline from last run)
+    expected_folders: list[str] = field(default_factory=list)
+    expected_document_names: list[str] = field(default_factory=list)
+    extra: dict[str, Any] = field(default_factory=dict)
+
+
 @dataclass
 class EmbeddingConfig:
     """How to turn a natural-language query into a vector for `compass ask`.
@@ -582,6 +598,21 @@ class EvalConfig:
     gt_variant_path: str = ""
     gt_variant_filter: list[str] = field(default_factory=list)
 
+    # --- Grounding audit (compass grounding-audit) ------------------------------------
+    # Verify an eval's grounding finding against the live KB. Every field name here comes
+    # from config, never assumed - a KB whose ingestion named things differently only
+    # changes these values, not code.
+    gt_response_path: str = "response"                    # the answer text in the eval record
+    gt_context_path: str = "context.knowledge_retrieval"  # the retrieved context the eval saw
+    gt_scores_path: str = "scores"                        # eval scores (shown in the report)
+    # Structured-query verification: which fields the KB can aggregate to check a count
+    # claim. Each entry: {field: <group-by field>, distinct_by: <doc-identity field>,
+    # label: <optional human label>}. Empty = count claims are not verified.
+    aggregatable_fields: list[dict] = field(default_factory=list)
+    # Extra query-intent regexes, merged with the built-in defaults (how many / count / by
+    # type / list all / most recent ...). Lets a domain add its own phrasings.
+    structured_query_patterns: list[str] = field(default_factory=list)
+
 
 @dataclass
 class ConsistencyConfig:
@@ -640,6 +671,7 @@ class Settings:
     embedding: EmbeddingConfig = field(default_factory=EmbeddingConfig)
     eval: EvalConfig = field(default_factory=EvalConfig)
     consistency: ConsistencyConfig = field(default_factory=ConsistencyConfig)
+    corpus: CorpusConfig = field(default_factory=CorpusConfig)
     # Non-service metadata carried through from config files.
     project_name: str = ""
     display_name: str = ""
@@ -661,7 +693,7 @@ _META_KEYS = frozenset({"default_env", "display_name", "environments", "skip_ste
 # new block can never be added to one and forgotten in the other (which is exactly how
 # `consistency` came to be silently dropped at the top level).
 _SERVICE_BLOCKS = ("api", "s3", "graphdb", "redis", "neptune", "pipeline",
-                   "retrieval", "discovery", "embedding", "eval", "consistency")
+                   "retrieval", "discovery", "embedding", "eval", "consistency", "corpus")
 
 
 def _deep_merge(base: dict, overlay: dict) -> dict:
@@ -787,6 +819,7 @@ def load_settings(
         embedding=_hydrate(EmbeddingConfig, flat.get("embedding")),
         eval=_hydrate(EvalConfig, flat.get("eval")),
         consistency=_hydrate(ConsistencyConfig, flat.get("consistency")),
+        corpus=_hydrate(CorpusConfig, flat.get("corpus")),
         project_name=project or "",
         display_name=merged.get("display_name", ""),
         environment=resolved_env,

compass_kb_validation-2.5.0/ingestion_validation/grounding.py

@@ -0,0 +1,341 @@
+"""Grounding audit: adjudicate an eval's grounding finding against the live KB.
+
+An eval (often an LLM judge) flags "this answer looks ungrounded". That verdict can be
+wrong, so this module supplies the DETERMINISTIC ground truth the verdict is checked
+against: it queries the KB COMPASS already knows how to reach and asks factual, checkable
+questions - is this claim actually in the KB, and did retrieval surface it? - then
+attributes the failure to a layer (retrieval vs the model). It does NOT score answer
+quality; that stays in the eval layer. No LLM in any verdict path.
+
+Phase 1 (this module) implements the STRUCTURED path - count/aggregation claims verified
+exactly against the KB via the adapter's ``aggregate_count`` (e.g. "the bot said 24
+documents; the KB has 127"). It is deterministic and unambiguous. The semantic-claim path
+(free-text claim extraction) is a separate, measurement-gated effort; see ``run_audit``.
+
+Backend-agnostic and schema-agnostic by construction: field names come from config, and
+aggregation goes through the adapter - a KB whose ingestion is shaped differently only
+changes config values, never this code. Backends that cannot aggregate exactly return
+None and the report says so, rather than asserting a wrong actual count.
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass, field
+from typing import Any, Optional
+
+# Built-in query-intent patterns. A KB can add its own via eval.structured_query_patterns;
+# these are merged, never replaced, so the defaults always hold.
+_STRUCTURED_PATTERNS: tuple[tuple[str, str], ...] = (
+    (r"\bhow many\b", "count"),
+    (r"\bcount\b", "count"),
+    (r"\bnumber of\b", "count"),
+    (r"\btotal\b.*\b(document|doc|record|item|file)", "count"),
+    (r"\bby type\b", "groupby"),
+    (r"\bbreakdown\b", "groupby"),
+    (r"\bwhat types\b", "groupby"),
+    (r"\blist all\b", "enumerate"),
+    (r"\bmost recent\b", "sort"),
+    (r"\blatest\b", "sort"),
+    (r"\boldest\b", "sort"),
+)
+
+
+def classify_query(query: str, extra_patterns: Optional[list[str]] = None) -> tuple[str, list[str]]:
+    """Classify a query as 'structured' (aggregation/count/list) or 'semantic' (topic).
+
+    Deterministic regex intent detection. Returns (kind, sorted-unique-intents).
+    """
+    q = (query or "").lower()
+    intents: set[str] = set()
+    for pattern, intent in _STRUCTURED_PATTERNS:
+        if re.search(pattern, q):
+            intents.add(intent)
+    for pattern in extra_patterns or []:
+        try:
+            if re.search(pattern, q):
+                intents.add("count")   # user-declared structured phrasing
+        except re.error:
+            continue
+    return ("structured" if intents else "semantic"), sorted(intents)
+
+
+@dataclass
+class NumericClaim:
+    label: str          # what the count is about (a group value, or "total")
+    count: int          # the number the answer asserted
+    source: str         # how it was extracted (table | inline)
+
+
+def extract_numeric_claims(response: str) -> list[NumericClaim]:
+    """Extract (label, count) claims from an answer - generically, no hardcoded labels.
+
+    Labels are whatever the answer names (a document type, a category, ...); they are
+    matched against real KB values later, never against a fixed set. Catches markdown
+    table rows and inline "N documents" phrasings.
+    """
+    text = response or ""
+    claims: list[NumericClaim] = []
+    seen: set[tuple[str, int]] = set()
+
+    def _add(label: str, count: int, source: str) -> None:
+        label = label.strip().strip("*").strip()
+        key = (label.lower(), count)
+        if label and key not in seen:
+            seen.add(key)
+            claims.append(NumericClaim(label=label, count=count, source=source))
+
+    # Markdown table rows: first cell is the label, the last standalone integer cell is
+    # the count. Header/separator rows (label in a known non-data set) are skipped.
+    _skip = {"type", "total", "metric", "count", "category", "documents", "document type", ""}
+    for row in re.findall(r"^\s*\|(.+)\|\s*$", text, flags=re.MULTILINE):
+        cells = [c.strip().strip("*").strip() for c in row.split("|")]
+        if not cells or all(set(c) <= set("-: ") for c in cells):
+            continue   # separator row
+        label = cells[0]
+        if label.lower() in _skip:
+            continue
+        nums = [int(c.replace(",", "")) for c in cells[1:] if re.fullmatch(r"[\d,]+", c)]
+        if nums:
+            _add(label, nums[-1], "table")
+
+    # Inline "N documents / docs / files / records / PDFs".
+    for m in re.finditer(r"\*{0,2}([\d,]+)\*{0,2}\s+(?:total\s+)?(?:documents?|docs|files?|records?|pdfs?)\b",
+                         text, flags=re.IGNORECASE):
+        _add("total", int(m.group(1).replace(",", "")), "inline")
+
+    return claims
+
+
+@dataclass
+class ClaimVerdict:
+    label: str
+    claimed: int
+    actual: Optional[int]      # None = not verifiable (backend can't aggregate / field absent)
+    verifiable: bool
+    correct: bool
+    detail: str
+
+
+def verify_structured_claims(adapter: Any, claims: list[NumericClaim],
+                             aggregatable_fields: list[dict]) -> tuple[list[ClaimVerdict], dict]:
+    """Verify count claims against exact KB aggregations via the adapter.
+
+    Returns (verdicts, meta). ``meta`` carries why verification could not run (e.g. the
+    backend does not support aggregation) so the report degrades honestly instead of
+    asserting a wrong actual value.
+    """
+    meta: dict = {"aggregated": False, "reason": ""}
+    if not getattr(adapter, "supports_aggregation", False):
+        meta["reason"] = (f"{type(adapter).__name__} does not support exact aggregation - "
+                          "count claims cannot be verified on this backend.")
+        return [ClaimVerdict(c.label, c.count, None, False, False, "backend cannot aggregate")
+                for c in claims], meta
+    if not aggregatable_fields:
+        meta["reason"] = ("no eval.aggregatable_fields configured - declare the group-by and "
+                          "doc-identity fields to verify count claims.")
+        return [ClaimVerdict(c.label, c.count, None, False, False, "no aggregatable fields configured")
+                for c in claims], meta
+
+    # Build a value -> count map across every configured aggregation, plus an overall
+    # total (distinct documents) for "total" claims.
+    value_counts: dict[str, int] = {}
+    total_docs = 0
+    for spec in aggregatable_fields:
+        gfield = spec.get("field", "")
+        dfield = spec.get("distinct_by") or None
+        counts = adapter.aggregate_count(gfield, dfield) if gfield else None
+        if counts:
+            meta["aggregated"] = True
+            for k, v in counts.items():
+                value_counts[str(k).lower()] = value_counts.get(str(k).lower(), 0) + int(v)
+                total_docs += int(v)
+    if not meta["aggregated"]:
+        meta["reason"] = "aggregation returned nothing (empty index, or the configured fields are absent)."
+
+    verdicts: list[ClaimVerdict] = []
+    for c in claims:
+        if c.label.lower() == "total":
+            actual = total_docs if meta["aggregated"] else None
+        else:
+            actual = value_counts.get(c.label.lower())
+        if actual is None:
+            verdicts.append(ClaimVerdict(c.label, c.count, None, False, False,
+                                         "no matching KB value" if meta["aggregated"] else meta["reason"]))
+            continue
+        correct = actual == c.count
+        if correct:
+            detail = "matches KB"
+        else:
+            ratio = (actual / c.count) if c.count else float("inf")
+            direction = "undercount" if actual > c.count else "overcount"
+            detail = f"off by {abs(actual - c.count)} ({ratio:.1f}x {direction})"
+        verdicts.append(ClaimVerdict(c.label, c.count, actual, True, correct, detail))
+    return verdicts, meta
+
+
+def _is_empty_context(ctx: Any) -> bool:
+    """True when the eval's retrieved context is effectively empty (null/blank fields)."""
+    if not ctx:
+        return True
+    s = str(ctx).strip()
+    return s in ("", "{}", "[]", "null", "None") or bool(re.fullmatch(r"[\s{}\[\]:,'\"]*", s))
+
+
+@dataclass
+class GroundingResult:
+    test_id: str
+    query: str
+    query_type: str
+    intents: list[str]
+    claim_verdicts: list[ClaimVerdict] = field(default_factory=list)
+    context_empty: bool = True
+    eval_scores: dict = field(default_factory=dict)
+    meta: dict = field(default_factory=dict)
+
+    @property
+    def wrong_claims(self) -> list[ClaimVerdict]:
+        return [v for v in self.claim_verdicts if v.verifiable and not v.correct]
+
+    @property
+    def verified_claims(self) -> list[ClaimVerdict]:
+        return [v for v in self.claim_verdicts if v.verifiable]
+
+    def layer_diagnosis(self) -> str:
+        """Deterministic layer attribution - which stage is implicated, no quality score."""
+        if self.query_type == "structured" and self.wrong_claims:
+            if self.context_empty:
+                return ("HALLUCINATION + QUERY-ROUTING FAILURE: retrieval returned nothing and the "
+                        "model fabricated counts. An aggregation query was answered by semantic search.")
+            return "FABRICATION: the model's counts disagree with the KB despite retrieved context."
+        if self.query_type == "structured" and self.verified_claims and not self.wrong_claims:
+            return "GROUNDED: every verifiable count matches the KB."
+        if not self.verified_claims:
+            return "UNVERIFIED: no count claim could be checked (see notes)."
+        return "MIXED: some counts match, others do not."
+
+    def severity(self) -> str:
+        if self.wrong_claims:
+            return "HIGH"      # confident wrong data reaching users
+        if not self.verified_claims:
+            return "LOW"
+        return "INFO"
+
+
+def get_path(obj: Any, dotted: str) -> Any:
+    """Read a dotted path (e.g. 'context.knowledge_retrieval') from a nested dict."""
+    cur = obj
+    for part in (dotted or "").split("."):
+        if isinstance(cur, dict) and part in cur:
+            cur = cur[part]
+        else:
+            return None
+    return cur
+
+
+def run_audit(eval_record: dict, adapter: Any, eval_cfg: Any) -> GroundingResult:
+    """Adjudicate one eval record against the KB. Deterministic; the structured path only.
+
+    ``eval_cfg`` supplies the dotted paths (gt_question_path, gt_response_path,
+    gt_context_path, gt_scores_path) and ``aggregatable_fields`` - all schema-agnostic.
+    """
+    query = str(get_path(eval_record, eval_cfg.gt_question_path) or "")
+    response = str(get_path(eval_record, getattr(eval_cfg, "gt_response_path", "response")) or "")
+    context = get_path(eval_record, getattr(eval_cfg, "gt_context_path", "context.knowledge_retrieval"))
+    scores = get_path(eval_record, getattr(eval_cfg, "gt_scores_path", "scores")) or {}
+    test_id = str(eval_record.get("test_id") or eval_record.get("id") or "")
+
+    qtype, intents = classify_query(query, getattr(eval_cfg, "structured_query_patterns", []))
+    result = GroundingResult(test_id=test_id, query=query, query_type=qtype, intents=intents,
+                             context_empty=_is_empty_context(context),
+                             eval_scores=scores if isinstance(scores, dict) else {})
+    if qtype == "structured":
+        claims = extract_numeric_claims(response)
+        result.claim_verdicts, result.meta = verify_structured_claims(
+            adapter, claims, getattr(eval_cfg, "aggregatable_fields", []) or [])
+    else:
+        # Semantic path (free-text claim verification) is a separate, measurement-gated
+        # mode; for now say so rather than run a fragile extractor and imply a verdict.
+        result.meta = {"semantic": True,
+                       "note": "Semantic claim verification is not yet enabled; only the "
+                               "retrieved-context check ran."}
+    return result
+
+
+def render_report(result: GroundingResult, *, project: str = "") -> str:
+    """Render a portable, evidence-first defect report (issue tracker / wiki / chat ready).
+
+    COMPASS's authoritative content is the evidence (claimed vs actual, from the KB) and
+    the deterministic layer attribution. The recommendations are clearly-labelled generic
+    hints, never presented as bespoke analysis or an answer-quality score.
+    """
+    L: list[str] = []
+    diagnosis = result.layer_diagnosis()
+    L.append(f"## Grounding Audit: {result.test_id or 'case'} - \"{result.query}\"")
+    L.append("")
+    L.append(f"**Severity:** {result.severity()}  ")
+    L.append(f"**Query type:** {result.query_type}"
+             + (f" ({', '.join(result.intents)})" if result.intents else "") + "  ")
+    L.append(f"**Verdict (deterministic, KB-verified):** {diagnosis}")
+    L.append("")
+
+    if result.eval_scores:
+        L.append("### Eval scores (as reported by the eval - shown for context)")
+        L.append("")
+        L.append("| Metric | Score | Target | Status |")
+        L.append("|---|---|---|---|")
+        for metric, data in result.eval_scores.items():
+            if isinstance(data, dict):
+                score, target = data.get("score", 0), data.get("target", 0)
+            else:
+                score, target = data, 0
+            status = "PASS" if (isinstance(score, (int, float)) and score >= target) else "FAIL"
+            L.append(f"| {metric} | {score} | {target} | {status} |")
+        L.append("")
+
+    if result.claim_verdicts:
+        L.append("### Claim verification against the KB")
+        L.append("")
+        L.append("| Claim | Bot said | KB actual | Result |")
+        L.append("|---|---|---|---|")
+        for v in result.claim_verdicts:
+            actual = v.actual if v.actual is not None else "unverifiable"
+            mark = "OK" if v.correct else ("FAIL" if v.verifiable else "N/A")
+            L.append(f"| {v.label} | {v.claimed} | {actual} | {mark} - {v.detail} |")
+        L.append("")
+        n_ok = sum(1 for v in result.claim_verdicts if v.correct)
+        n_ver = len(result.verified_claims)
+        if n_ver:
+            L.append(f"**{n_ok}/{n_ver} verifiable claim(s) match the KB.**")
+        if result.meta.get("reason"):
+            L.append(f"> Note: {result.meta['reason']}")
+        L.append("")
+
+    L.append("### Retrieved context")
+    L.append("")
+    L.append("`[EMPTY - retrieval returned nothing]`" if result.context_empty
+             else "_(non-empty; see the eval record)_")
+    L.append("")
+
+    # Recommendations: generic, template-selected by the diagnosis. Labelled as hints.
+    L.append("### Suggested next steps (generic guidance, not KB-specific analysis)")
+    L.append("")
+    if result.query_type == "structured" and result.wrong_claims:
+        L += [
+            "- Route count/aggregation questions to a **direct aggregation** (the KB can answer",
+            "  these exactly), not semantic search.",
+            "- On **empty retrieval, refuse or flag uncertainty** rather than letting the model fill the gap.",
+        ]
+    elif result.query_type == "semantic":
+        L += [
+            "- This is a topic query; run the semantic grounding mode (when enabled) to verify",
+            "  individual claims against the KB.",
+        ]
+    else:
+        L.append("- Counts verified against the KB; no action indicated for this case.")
+    L.append("")
+    L.append("---")
+    L.append("_Generated by `compass grounding-audit`"
+             + (f" (project {project})" if project else "") + ". "
+             "Evidence is deterministic and KB-verified; it does not judge answer quality._")
+    return "\n".join(L) + "\n"

{compass_kb_validation-2.3.0 → compass_kb_validation-2.5.0}/ingestion_validation/profiler.py

@@ -581,9 +581,8 @@ def suggest_config_from_run(results: list, documents: list) -> str:
         "",
         "# Documents discovered this run. Pin it to alert on drift (or use 'auto' to",
         "# baseline against the previous run automatically):",
-        "discovery:",
-        "  extra:",
-        f"    expected_documents: {n_docs}",
+        "corpus:",
+        f"  expected_documents: {n_docs}",
     ]
     if avg_chars:
         median_size = int(statistics.median(avg_chars))