compass-kb-validation 3.3.0__tar.gz → 3.4.0__tar.gz

{compass_kb_validation-3.3.0 → compass_kb_validation-3.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: compass-kb-validation
-Version: 3.3.0
+Version: 3.4.0
 Summary: Knowledge-base readiness validation for RAG & AI agents — config-driven, deterministic pipeline integrity checks.
 Author: COMPASS
 License: MIT

{compass_kb_validation-3.3.0 → compass_kb_validation-3.4.0}/compass/__init__.py

@@ -5,4 +5,4 @@ end-to-end: API upload -> object storage -> knowledge graph -> vector store ->
 retrieval quality, plus cross-store consistency.
 """
 
-__version__ = "3.3.0"
+__version__ = "3.4.0"

{compass_kb_validation-3.3.0 → compass_kb_validation-3.4.0}/compass/citation.py

@@ -377,7 +377,19 @@ def verify_level1(citations: list[Citation], eval_chunks: list[dict], cfg: Any)
                 v.terms_missing.append(cv.raw)
                 v._missing.append(cv)
         if not cit.values:
-            v.l1_verdict, v.severity = "UNVERIFIABLE", "INFO"   # no quantitative claim to check
+            # No numeric claim. If the claim carries distinctive entities (proper nouns / quoted
+            # phrases) and they are ALL present in the cited chunk, it is TERM_SUPPORTED - a
+            # weaker, lexical tier counted in a SEPARATE term-grounding rate, never the strict
+            # numeric one. Missing entities stay UNVERIFIABLE (lexical absence can be paraphrase).
+            ents = [] if cit.negated else _extract_entities(cit.claim)
+            if ents:
+                hay = _squash(_haystack(chunk, text_field))
+                v.entity_missing = [e for e in ents if _squash(e) not in hay]
+                v.entities_checked = True
+                v.l1_verdict, v.severity = (("TERM_SUPPORTED", "INFO") if not v.entity_missing
+                                            else ("UNVERIFIABLE", "INFO"))
+            else:
+                v.l1_verdict, v.severity = "UNVERIFIABLE", "INFO"   # no quantitative claim to check
             if cit.negated:
                 v.note = "claim is negated - lexical verification not applicable"
         elif not v.terms_missing:
@@ -539,6 +551,7 @@ _ATTRIB = {
     "citation_error": ("citation_error", "MEDIUM"),  # right fact, wrong cited location
     "fabricated_source": ("fabricated_source", "HIGH"),
     "none": ("none", "INFO"),
+    "term_supported": ("term_supported", "INFO"),  # distinctive entities present in cited chunk
     "unverifiable": ("unverifiable", "INFO"),      # no quantitative term; lexical check N/A
     "unknown": ("unknown", "MEDIUM"),              # a miss we could not attribute (no live KB)
 }
@@ -558,6 +571,8 @@ def attribute(verdicts: list[CitationVerdict], l1_ran: bool) -> None:
             key = "none"
         elif l2 == "FABRICATED_SOURCE":
             key = "fabricated_source"          # cited a document that isn't in the KB
+        elif l1 == "TERM_SUPPORTED":
+            key = "term_supported"             # distinctive entities present; weaker than numeric
         elif l2 in _L2_UNVERIFIABLE:
             key = "unverifiable"               # source unresolved / too large / no numeric claim
         elif l1 == "UNVERIFIABLE" and not l2:
@@ -650,6 +665,7 @@ def audit_citations(response: str, eval_chunks: Optional[list[dict]], adapter: A
     summary = summarize(verdicts, response, l1_ran, l2_ran)
     if l1_ran:
         summary.update(context_utilization(citations, eval_chunks or [], cfg))
+        summary.update(text_utilization(response, eval_chunks or [], cfg))
     # DEFECT-003 safety net: if the response clearly carries citation-like patterns but none
     # were extracted, say so rather than reporting a silent total_citations=0 that reads as
     # "no citations exist" (which was the DEFECT-001 failure mode).
@@ -680,6 +696,31 @@ def context_utilization(citations: list[Citation], eval_chunks: list[dict], cfg:
             "context_utilization_pct": round(100.0 * used / total, 1)}
 
 
+def _ngrams(text: str, n: int = 3) -> set:
+    toks = re.findall(r"[a-z0-9]+", (text or "").lower())
+    return {tuple(toks[i:i + n]) for i in range(len(toks) - n + 1)} if len(toks) >= n else set()
+
+
+def text_utilization(response: str, eval_chunks: list[dict], cfg: Any,
+                     n: int = 3, min_overlap: float = 0.1) -> dict:
+    """How much of the retrieved CONTENT actually appears in the response, by n-gram overlap -
+    distinct from ``context_utilization_pct`` (which measures whether a chunk's PAGE was cited).
+    A chunk counts as used when at least ``min_overlap`` of its n-grams recur in the response."""
+    text_field = getattr(cfg, "gt_chunk_text_field", "text")
+    total = len(eval_chunks or [])
+    if not total:
+        return {}
+    resp = _ngrams(response, n)
+    if not resp:
+        return {"text_utilization_pct": 0.0}
+    used = 0
+    for ch in eval_chunks:
+        cg = _ngrams(_haystack(ch, text_field), n)
+        if cg and len(cg & resp) / len(cg) >= min_overlap:
+            used += 1
+    return {"text_utilization_pct": round(100.0 * used / total, 1)}
+
+
 def summarize(verdicts: list[CitationVerdict], response: str, l1_ran: bool, l2_ran: bool) -> dict:
     """Aggregate signal for the audit: how many citations are sound, and the fault mix."""
     total = len(verdicts)
@@ -693,6 +734,11 @@ def summarize(verdicts: list[CitationVerdict], response: str, l1_ran: bool, l2_r
     checkable = sound + fault_total
     # Accuracy is over CHECKABLE citations only; unverifiable/unknown are excluded, not scored.
     accuracy = round(100.0 * sound / checkable, 1) if checkable else None
+    # Term grounding: a SEPARATE, weaker tier for qualitative citations whose distinctive entities
+    # are present in the cited chunk. Never folded into the strict numeric rate above.
+    term_supported = sum(1 for v in verdicts if v.attribution == "term_supported")
+    term_checkable = sum(1 for v in verdicts if v.entities_checked)
+    term_rate = round(100.0 * term_supported / term_checkable, 1) if term_checkable else None
     # Citation coverage: sentences with a checkable quantitative claim that carry NO citation.
     sentences = [s.strip() for s in re.split(r"(?<=[.!?])\s+", response or "") if s.strip()]
     numeric = [s for s in sentences if extract_terms(s)]
@@ -704,6 +750,9 @@ def summarize(verdicts: list[CitationVerdict], response: str, l1_ran: bool, l2_r
         "checkable": checkable,
         "unverifiable": unverifiable,
         "accuracy_pct": accuracy,
+        "term_supported": term_supported,
+        "term_checkable": term_checkable,
+        "term_grounding_rate_pct": term_rate,
         "faults": faults,
         "coverage_pct": coverage,
         "uncited_numeric_claims": len(uncited),

{compass_kb_validation-3.3.0 → compass_kb_validation-3.4.0}/compass/eval_enrich.py

@@ -21,10 +21,34 @@ Two capabilities:
 
 from __future__ import annotations
 
+import re
 from typing import Any, Optional
 
 from compass.eval_interop import get_path
 
+# Minimal stop-word set for the lexical context-support signal (deterministic, no NLP lib).
+_STOP = frozenset(
+    "the a an and or of to in for on with at by from is are was were be been being it its this that "
+    "these those as but if then than so we you they he she i our your their his her not no do does did "
+    "have has had will would can could may might should must about into over under between out up down".split())
+_WORD_RE = re.compile(r"[a-z0-9][a-z0-9\-]{2,}")
+
+
+def _resolve_chunks(record: dict, cfg: Any) -> list:
+    """The case's retrieved chunks, via the configured path then common nested fallbacks."""
+    chunks = get_path(record, getattr(cfg, "gt_chunks_path", "context.chunks"))
+    if isinstance(chunks, list) and chunks:
+        return chunks
+    for alt in ("context.chunks", "chunks"):
+        v = get_path(record, alt)
+        if isinstance(v, list) and v:
+            return v
+    return []
+
+
+def _content_terms(text: str) -> set:
+    return {t for t in _WORD_RE.findall((text or "").lower()) if t not in _STOP}
+
 # Deterministic signal codes COMPASS owns. KB-derived only - no answer-quality / guardrail / SQL.
 GROUNDED, PARTIAL, UNGROUNDED = "COMPASS_GROUNDED", "COMPASS_PARTIAL_GROUND", "COMPASS_UNGROUNDED"
 _FAULT_CODES = {
@@ -40,6 +64,7 @@ DEFAULTS = {
     "context_waste_below_pct": 20.0,
     "chunk_quality_floor": 0.5,
     "uncited_numeric_max": 3,
+    "context_support_floor": 0.4,     # lexical (weaker) proxy floor, only used when no citations
 }
 
 
@@ -56,14 +81,8 @@ def _case_chunk_quality(record: dict, cfg: Any, min_chars: int) -> Optional[dict
     """Score the chunks THIS case was given, deterministically. None if no chunks are present."""
     from compass.corpus_health import _chunk_quality
 
-    chunks = get_path(record, getattr(cfg, "gt_chunks_path", "context.chunks"))
-    if not isinstance(chunks, list) or not chunks:
-        for alt in ("context.chunks", "chunks"):
-            v = get_path(record, alt)
-            if isinstance(v, list) and v:
-                chunks = v
-                break
-    if not isinstance(chunks, list) or not chunks:
+    chunks = _resolve_chunks(record, cfg)
+    if not chunks:
         return None
     text_field = getattr(cfg, "gt_chunk_text_field", "text") or "text"
     scores, not_self_contained = [], 0
@@ -83,7 +102,46 @@ def _case_chunk_quality(record: dict, cfg: Any, min_chars: int) -> Optional[dict
     }
 
 
-def _derive_signals(grounding: dict, chunk_quality: Optional[dict], thr: dict) -> list[dict]:
+def _context_support(record: dict, cfg: Any, *, min_overlap: float = 0.25) -> Optional[dict]:
+    """A SEPARATE, weaker lexical signal - never the citation grounding rate. The share of
+    response sentences whose content terms are echoed by the retrieved context chunks. It
+    estimates support for responses that may carry NO citations, but it is only a lexical proxy,
+    and is computed ONLY when the case actually has retrieved chunks (a refusal / non-retrieval
+    case has none, so it never fabricates a misleading 'grounding' number)."""
+    chunks = _resolve_chunks(record, cfg)
+    if not chunks:
+        return None
+    from compass.grounding import _resolve_field
+
+    r_val, _, _ = _resolve_field(record, getattr(cfg, "gt_response_path", "response"), "response")
+    response = str(r_val or "").strip()
+    if not response:
+        return None
+    text_field = getattr(cfg, "gt_chunk_text_field", "text") or "text"
+    ctx_terms: set = set()
+    for c in chunks:
+        if isinstance(c, dict):
+            ctx_terms |= _content_terms(c.get(text_field) or c.get("text") or "")
+    if not ctx_terms:
+        return None
+    scored = supported = 0
+    for sent in re.split(r"(?<=[.!?])\s+", response):
+        terms = _content_terms(sent)
+        if len(terms) < 3:                       # too short to judge overlap meaningfully
+            continue
+        scored += 1
+        if len(terms & ctx_terms) / len(terms) >= min_overlap:
+            supported += 1
+    if not scored:
+        return None
+    return {"context_support_rate": round(supported / scored, 3),
+            "sentences_scored": scored, "supported": supported,
+            "method": "lexical_term_overlap",
+            "note": "Weaker lexical proxy for responses without citations - NOT the citation grounding rate."}
+
+
+def _derive_signals(grounding: dict, chunk_quality: Optional[dict],
+                    context_support: Optional[dict], thr: dict) -> list[dict]:
     """Turn the deterministic measurements into COMPASS_* signal codes. Every code is
     check-backed; there is no heuristic guessing here."""
     signals: list[dict] = []
@@ -113,6 +171,14 @@ def _derive_signals(grounding: dict, chunk_quality: Optional[dict], thr: dict) -
     if (grounding.get("uncited_numeric_claims") or 0) > thr["uncited_numeric_max"]:
         signals.append({"code": "COMPASS_UNCITED_NUMERIC", "severity": "MEDIUM",
                         "detail": f"{grounding['uncited_numeric_claims']} numeric claim(s) carry no citation"})
+    # Separate, weaker lexical signal - only when there is no strong citation grounding to rely on,
+    # so it complements (never overrides) the citation grounding rate.
+    if context_support and rate is None:
+        csr = context_support["context_support_rate"]
+        if csr < thr["context_support_floor"]:
+            signals.append({"code": "COMPASS_LOW_CONTEXT_SUPPORT", "severity": "LOW",
+                            "detail": f"only {csr:.0%} of response sentences are lexically echoed "
+                                      f"by the retrieved context (weak proxy; no verifiable citations)"})
     return signals
 
 
@@ -128,13 +194,19 @@ def enrich_case(record: dict, cfg: Any, adapter: Any = None, *,
     thr = {**DEFAULTS, **(thresholds or {})}
     res = run_audit(record, adapter, cfg)
     summary = res.citations.get("summary", {}) if isinstance(res.citations, dict) else {}
+    tr = summary.get("term_grounding_rate_pct")
     grounding = {
         "citation_grounding_rate": _citation_grounding_rate(summary),
         "total_citations": summary.get("total_citations", 0),
         "checkable": summary.get("checkable", 0),
         "sound": summary.get("sound", 0),
+        # Separate, weaker tier: distinctive entities present in the cited chunk (not numeric).
+        "term_grounding_rate": (tr / 100.0 if tr is not None else None),
+        "term_supported": summary.get("term_supported", 0),
+        "term_checkable": summary.get("term_checkable", 0),
         "faults": summary.get("faults", {}),
-        "context_utilization_pct": summary.get("context_utilization_pct"),
+        "context_utilization_pct": summary.get("context_utilization_pct"),   # cited-page coverage
+        "text_utilization_pct": summary.get("text_utilization_pct"),         # actual content overlap
         "uncited_numeric_claims": summary.get("uncited_numeric_claims", 0),
         "context_empty": res.context_empty,
         "levels": summary.get("levels", "none"),
@@ -142,10 +214,13 @@ def enrich_case(record: dict, cfg: Any, adapter: Any = None, *,
     if summary.get("diagnostic"):
         grounding["diagnostic"] = summary["diagnostic"]
     chunk_quality = _case_chunk_quality(record, cfg, min_chars)
+    context_support = _context_support(record, cfg)
     block: dict = {"grounding": grounding, "method": "compass_deterministic"}
     if chunk_quality:
         block["chunk_quality"] = chunk_quality
-    block["signals"] = _derive_signals(grounding, chunk_quality, thr)
+    if context_support:
+        block["context_support"] = context_support
+    block["signals"] = _derive_signals(grounding, chunk_quality, context_support, thr)
     return block
 
 

{compass_kb_validation-3.3.0 → compass_kb_validation-3.4.0}/compass_kb_validation.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: compass-kb-validation
-Version: 3.3.0
+Version: 3.4.0
 Summary: Knowledge-base readiness validation for RAG & AI agents — config-driven, deterministic pipeline integrity checks.
 Author: COMPASS
 License: MIT

{compass_kb_validation-3.3.0 → compass_kb_validation-3.4.0}/tests/test_citation.py

@@ -472,3 +472,32 @@ def test_diagnostic_when_pattern_matches_but_nothing_extracted():
     res = citation.audit_citations("(SOURCE, 2026, p. 1)", None, None, cfg)
     assert res["summary"]["total_citations"] == 0
     assert "matched by regex but 0 were extracted" in res["summary"].get("diagnostic", "")
+
+
+def test_term_supported_is_a_separate_tier_not_the_strict_sound_rate():
+    # Problem 1: a qualitative citation whose distinctive entity is present in the cited chunk is
+    # TERM_SUPPORTED - counted in a SEPARATE term rate, never inflating the strict numeric rate.
+    from types import SimpleNamespace
+    cfg = SimpleNamespace(citation_patterns=[], citation_page_tolerance=0, citation_verify_entities=False,
+                          gt_chunk_doc_field="document_title", gt_chunk_page_field="page_start",
+                          gt_chunk_text_field="text")
+    resp = "The Federal Reserve signaled patience. (Doc, 2026, p. 1) Markets calmed."
+    chunks = [{"text": "The Federal Reserve signaled patience on rates.", "document_title": "Doc", "page_start": 1}]
+    s = citation.audit_citations(resp, chunks, None, cfg)["summary"]
+    assert s["term_supported"] == 1 and s["term_grounding_rate_pct"] == 100.0
+    assert s["sound"] == 0 and s["checkable"] == 0        # strict numeric rate untouched
+    # entity absent -> stays unverifiable, never a fault (lexical absence can be paraphrase)
+    s2 = citation.audit_citations(resp, [{"text": "Unrelated weather note.", "document_title": "Doc",
+                                          "page_start": 1}], None, cfg)["summary"]
+    assert s2["term_supported"] == 0 and not s2["faults"]
+
+
+def test_text_utilization_measures_content_overlap_distinct_from_page_coverage():
+    # Problem 4: text_utilization_pct is n-gram content overlap, separate from context_utilization
+    # (which is cited-page coverage).
+    from types import SimpleNamespace
+    cfg = SimpleNamespace(gt_chunk_text_field="text")
+    chunks = [{"text": "quarterly revenue increased sharply on strong product demand"}]
+    hi = citation.text_utilization("quarterly revenue increased sharply on strong product demand", chunks, cfg)
+    lo = citation.text_utilization("completely unrelated sentence about the weather today", chunks, cfg)
+    assert hi["text_utilization_pct"] == 100.0 and lo["text_utilization_pct"] == 0.0

{compass_kb_validation-3.3.0 → compass_kb_validation-3.4.0}/tests/test_eval_enrich.py

@@ -107,3 +107,29 @@ def test_extract_judge_scores_handles_varied_shapes():
     assert extract_judge_scores(c3, "Answer Relevance") == {"judge_x": 0.9}
     # metric absent -> empty
     assert extract_judge_scores({"scores": {"other": {"j": 0.5}}}, "groundedness") == {}
+
+
+def test_context_support_is_separate_weak_signal_and_scoped_to_retrieval_cases():
+    # Problem 3: a citation-free response gets a SEPARATE lexical context_support (never the
+    # citation grounding rate), and only when the case actually carries retrieved chunks.
+    cfg = Settings().eval
+    echoed = [{"input": "q", "output": "The economy is slowing and inflation is easing gradually.",
+               "context": {"chunks": [{"text": "The economy is slowing while inflation eases gradually.",
+                                       "document_title": "D", "page_start": 1}]}}]
+    b = enrich_cases(echoed, cfg)[0][0]["compass"]
+    assert b["grounding"]["citation_grounding_rate"] is None          # no citation -> strict rate stays None
+    assert b["context_support"]["context_support_rate"] == 1.0        # separate lexical proxy
+    assert "NOT the citation grounding rate" in b["context_support"]["note"]
+    # a refusal / non-retrieval case (no chunks) gets NO context_support - no misleading score
+    refusal = [{"input": "do X", "output": "I can't help with that.", "context": {}}]
+    assert "context_support" not in enrich_cases(refusal, cfg)[0][0]["compass"]
+
+
+def test_term_grounding_rate_is_separate_from_citation_rate():
+    cfg = Settings().eval
+    recs = [{"input": "q", "output": "The Federal Reserve held rates steady. (Doc, 2026, p. 1) End.",
+             "context": {"chunks": [{"text": "The Federal Reserve held rates steady this month.",
+                                     "document_title": "Doc", "page_start": 1}]}}]
+    g = enrich_cases(recs, cfg)[0][0]["compass"]["grounding"]
+    assert g["citation_grounding_rate"] is None       # no numeric claim -> strict rate None
+    assert g["term_grounding_rate"] == 1.0            # but the entity is supported (separate tier)