compass-kb-validation 3.2.0__tar.gz → 3.2.1__tar.gz

{compass_kb_validation-3.2.0 → compass_kb_validation-3.2.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: compass-kb-validation
-Version: 3.2.0
+Version: 3.2.1
 Summary: Knowledge-base readiness validation for RAG & AI agents — config-driven, deterministic pipeline integrity checks.
 Author: COMPASS
 License: MIT

{compass_kb_validation-3.2.0 → compass_kb_validation-3.2.1}/compass/__init__.py

@@ -5,4 +5,4 @@ end-to-end: API upload -> object storage -> knowledge graph -> vector store ->
 retrieval quality, plus cross-store consistency.
 """
 
-__version__ = "3.2.0"
+__version__ = "3.2.1"

{compass_kb_validation-3.2.0 → compass_kb_validation-3.2.1}/compass/citation.py

@@ -245,8 +245,16 @@ def _claim_before(text: str, idx: int) -> str:
     """The statement a citation supports: the text preceding it, back to the last real
     sentence break. Excludes the citation itself and anything after it."""
     start = 0
+    prev_start = 0
     for m in _SENT_BOUND.finditer(text[:idx]):
+        prev_start = start
         start = m.end()
+    # If the last sentence boundary sits right where the citation begins, that period is the
+    # TERMINATOR of the claim sentence (the standard "...claim. (SOURCE, p. N)" format), not a
+    # separator inside the claim - so text[start:idx] would be empty and the citation would be
+    # silently dropped. Fall back one boundary to capture the claim sentence itself.
+    if start >= idx - 2:
+        start = prev_start
     nl = text.rfind("\n", 0, idx)
     if nl + 1 > start:
         start = nl + 1
@@ -642,6 +650,15 @@ def audit_citations(response: str, eval_chunks: Optional[list[dict]], adapter: A
     summary = summarize(verdicts, response, l1_ran, l2_ran)
     if l1_ran:
         summary.update(context_utilization(citations, eval_chunks or [], cfg))
+    # DEFECT-003 safety net: if the response clearly carries citation-like patterns but none
+    # were extracted, say so rather than reporting a silent total_citations=0 that reads as
+    # "no citations exist" (which was the DEFECT-001 failure mode).
+    if not citations and response:
+        raw = len(_PAREN_CITE.findall(response))
+        if raw:
+            summary["diagnostic"] = (
+                f"{raw} citation-like pattern(s) matched by regex but 0 were extracted - "
+                "likely a claim-boundary parsing issue, not an absence of citations.")
     return {"verdicts": verdicts, "l1_ran": l1_ran, "l2_ran": l2_ran, "summary": summary}
 
 

{compass_kb_validation-3.2.0 → compass_kb_validation-3.2.1}/compass/cli.py

@@ -1203,6 +1203,22 @@ def cmd_compare(argv: list[str]) -> int:
     if not cases:
         print(f"  No usable cases in {args.ground_truth} "
               f"(check the eval.gt_* field paths / variant filter in your config).")
+        # Say WHICH paths were tried and what resolved on the first record, so the user can see
+        # the actual mismatch instead of guessing (DEFECT-007).
+        from compass.eval_interop import diagnose_paths
+        diag = diagnose_paths(args.ground_truth, {
+            "gt_question_path": ev.gt_question_path,
+            "gt_chunks_path": ev.gt_chunks_path,
+            "gt_filters_path": ev.gt_filters_path,
+            "gt_variant_path": ev.gt_variant_path,
+        })
+        if diag:
+            print("  Configured paths on the first record:")
+            for line in diag:
+                print(line)
+            if variant_filter:
+                print(f"  variant filter: {', '.join(variant_filter)} "
+                      f"(via gt_variant_path=\"{ev.gt_variant_path or '(unset)'}\")")
         return 2
 
     # Ground-truth cases carry production's own metadata filters. If the backend cannot
@@ -1214,7 +1230,9 @@ def cmd_compare(argv: list[str]) -> int:
               file=sys.stderr)
 
     vnote = f" (variant filter: {', '.join(variant_filter)})" if variant_filter else ""
-    print(f"  Comparing compass retrieval against {len(cases)} ground-truth case(s){vnote} ...")
+    # Progress goes to stderr under --json so stdout stays a clean, pipeable JSON document.
+    print(f"  Comparing compass retrieval against {len(cases)} ground-truth case(s){vnote} ...",
+          file=sys.stderr if args.json else sys.stdout)
     try:
         payload = comparison_report(settings, cases, top_k=args.top_k or None,
                                     skip_empty=not args.include_empty)
@@ -1312,7 +1330,8 @@ def cmd_evidence(argv: list[str]) -> int:
         return 2
     limit = 1 if args.quick else (args.limit or settings.evidence.sample_size)
 
-    print(f"  compass evidence - project={args.project or '(none)'} checks={checks} limit={limit}")
+    print(f"  compass evidence - project={args.project or '(none)'} checks={checks} limit={limit}",
+          file=sys.stderr if args.format == "json" else sys.stdout)
     try:
         payload = ev.gather_evidence(settings, checks, document_filter=args.document, limit=limit)
     except Exception as exc:  # noqa: BLE001 - unreachable/misconfigured backend -> clean message
@@ -1383,7 +1402,8 @@ def cmd_quality(argv: list[str]) -> int:
     except Exception as exc:  # noqa: BLE001 - malformed/empty project YAML -> clean message, not a traceback
         print(f"  Config error: {exc}")
         return 2
-    print(f"  Sampling up to {args.sample} chunks for quality signals ...")
+    print(f"  Sampling up to {args.sample} chunks for quality signals ...",
+          file=sys.stderr if args.json else sys.stdout)
     try:
         payload = quality_report(settings, sample=args.sample, min_chars=args.min_chars)
     except Exception as exc:  # noqa: BLE001 - clean message, never a traceback
@@ -1428,7 +1448,8 @@ def cmd_freshness(argv: list[str]) -> int:
     except Exception as exc:  # noqa: BLE001 - malformed/empty project YAML -> clean message, not a traceback
         print(f"  Config error: {exc}")
         return 2
-    print(f"  Checking source freshness for up to {args.sample} document(s) ...")
+    print(f"  Checking source freshness for up to {args.sample} document(s) ...",
+          file=sys.stderr if args.json else sys.stdout)
     try:
         payload = freshness_report(settings, sample=args.sample, stale_after_days=args.stale_after)
     except Exception as exc:  # noqa: BLE001 - clean message, never a traceback

{compass_kb_validation-3.2.0 → compass_kb_validation-3.2.1}/compass/corpus_health.py

@@ -261,6 +261,17 @@ def freshness_report(settings: Settings, *, sample: int = 500, stale_after_days:
             f"{without_key} of {len(seen_docs)} sampled document(s) carry no source key, so "
             f"their source could not be checked - likely a missing s3_key field (prefix/field "
             f"mismatch) or sources in a different store, not necessarily orphans.")
+    # A near-total orphan rate almost never means every source was deleted - it points at a
+    # systemic mismatch (the chunk s3_key does not match where objects actually live: wrong
+    # bucket, or a prefix baked into the key that the store does not use). Flag it so the user
+    # checks configuration before "reconciling" hundreds of false orphans (DEFECT-006).
+    if checked and len(orphans) / checked > 0.8:
+        result["warning"] = (
+            f"{len(orphans)}/{checked} sampled sources ({round(100 * len(orphans) / checked)}%) "
+            "look orphaned - a rate this high usually means a key mismatch (wrong bucket, or a "
+            "prefix in the chunk s3_key that the object store does not expect), not that the "
+            "sources are actually gone. Verify s3.bucket_name and the s3_key values before "
+            "reconciling.")
     return result
 
 

{compass_kb_validation-3.2.0 → compass_kb_validation-3.2.1}/compass/eval_interop.py

@@ -32,6 +32,42 @@ def get_path(record: Any, dotted: str) -> Any:
     return cur
 
 
+def diagnose_paths(path: str, gt_paths: dict) -> list[str]:
+    """Explain why a ground-truth file yielded no usable cases: for each configured gt_* path,
+    report whether it resolves on the first record and what type/size was found. Deterministic,
+    read-only, best-effort - returns human-readable lines (empty list if the file is unreadable).
+    """
+    import json
+
+    try:
+        with open(path, encoding="utf-8") as f:
+            head = f.read(2_000_000)
+        data = json.loads(head)
+    except (OSError, ValueError):
+        # Maybe JSONL: try the first line.
+        try:
+            with open(path, encoding="utf-8") as f:
+                data = [json.loads(f.readline())]
+        except (OSError, ValueError):
+            return []
+    rec = data[0] if isinstance(data, list) and data else (data if isinstance(data, dict) else None)
+    if not isinstance(rec, dict):
+        return ["first record is not an object - expected a JSON array (or JSONL) of case objects"]
+    lines = []
+    for label, dotted in gt_paths.items():
+        if not dotted:
+            lines.append(f"  {label}: (unset)")
+            continue
+        val = get_path(rec, dotted)
+        if val is None:
+            lines.append(f"  {label}: \"{dotted}\" -> NOT FOUND")
+        elif isinstance(val, list):
+            lines.append(f"  {label}: \"{dotted}\" -> found (list, {len(val)} item(s))")
+        else:
+            lines.append(f"  {label}: \"{dotted}\" -> found ({type(val).__name__})")
+    return lines
+
+
 def normalize_filters(raw: Any) -> dict:
     """Parse a filters value into ``{field: "a|b"}`` (compass filter form).
 

{compass_kb_validation-3.2.0 → compass_kb_validation-3.2.1}/compass/grounding.py

@@ -411,9 +411,21 @@ def run_audit(eval_record: dict, adapter: Any, eval_cfg: Any, doc_exists: Any =
     scores = get_path(eval_record, getattr(eval_cfg, "gt_scores_path", "scores")) or {}
     test_id = str(eval_record.get("test_id") or eval_record.get("id") or "")
 
+    # Retrieval is empty only if BOTH the configured context path AND the retrieved chunks are
+    # empty. The default gt_context_path (and gt_chunks_path) do not fit every eval schema, so
+    # keying "empty retrieval" off one path alone raised a false HIGH signal on records that
+    # clearly had chunks (DEFECT-002). Cross-check the configured chunk path plus the common
+    # nested locations so the emptiness verdict is schema-agnostic.
+    eval_chunks = get_path(eval_record, getattr(eval_cfg, "gt_chunks_path", "context.chunks"))
+    _chunk_candidates = (eval_chunks,
+                         get_path(eval_record, "context.chunks"),
+                         get_path(eval_record, "chunks"))
+    has_chunks = any(isinstance(c, list) and len(c) > 0 for c in _chunk_candidates)
+    context_empty = _is_empty_context(context) and not has_chunks
+
     qtype, intents = classify_query(query, getattr(eval_cfg, "structured_query_patterns", []))
     result = GroundingResult(test_id=test_id, query=query, query_type=qtype, intents=intents,
-                             context_empty=_is_empty_context(context),
+                             context_empty=context_empty,
                              eval_scores=scores if isinstance(scores, dict) else {})
     # Record which fields were auto-detected (vs configured), and whether this is a
     # retrieval-only case (no generated answer) - so the CLI can explain "0 citations" honestly.
@@ -429,7 +441,7 @@ def run_audit(eval_record: dict, adapter: Any, eval_cfg: Any, doc_exists: Any =
     # Citation audit: verify each cited claim at its cited location (Level 1 against the eval's
     # retrieved chunks when present, Level 2 against the live KB). The "cited document absent"
     # signal is derived from it, so citation extraction lives in exactly one place.
-    eval_chunks = get_path(eval_record, getattr(eval_cfg, "gt_chunks_path", "context.chunks"))
+    # (eval_chunks was resolved above for the empty-retrieval cross-check.)
     result.citations = citation.audit_citations(
         response, eval_chunks if isinstance(eval_chunks, list) else None, adapter, eval_cfg,
         extra_patterns=getattr(eval_cfg, "citation_patterns", None), doc_exists=doc_exists)

{compass_kb_validation-3.2.0 → compass_kb_validation-3.2.1}/compass/providers/vectorstore.py

@@ -205,6 +205,15 @@ class RedisVectorStoreAdapter(VectorStoreAdapter):
         special = r"""\.+*?[]{}<>()@!#$%^&-=~'"`:;,/| """
         return "".join(f"\\{ch}" if ch in special else ch for ch in value)
 
+    @staticmethod
+    def escape_text(value: str) -> str:
+        """Escape RediSearch query-parser metacharacters for a TEXT-field query while
+        PRESERVING spaces as token separators. TAG escaping (escape_tag) backslash-escapes
+        spaces, which collapses a multi-word value into a single unmatchable token, so every
+        multi-word TEXT search silently returned nothing (DEFECT-008)."""
+        special = r"""\+*?[]{}<>()@!#$%^&-=~'"`:;,/|"""
+        return "".join(f"\\{ch}" if ch in special else ch for ch in value)
+
     def _id_query(self, search_value: str) -> str:
         field = self.config.document_id_field
         if self.config.document_id_field_type == "TAG":
@@ -305,7 +314,9 @@ class RedisVectorStoreAdapter(VectorStoreAdapter):
         return self._parse_search(raw, keep_binary_fields=(emb,))
 
     def text_search(self, field: str, value: str, limit: int = 10) -> list[dict]:
-        escaped = self.escape_tag(value)
+        # TEXT-field query: escape parser metacharacters but keep spaces as token separators
+        # so a multi-word value matches its tokens (was TAG-escaped -> matched nothing).
+        escaped = self.escape_text(value)
         query = f"@{field}:({escaped})"
         try:
             raw = self.client.execute_command(

{compass_kb_validation-3.2.0 → compass_kb_validation-3.2.1}/compass_kb_validation.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: compass-kb-validation
-Version: 3.2.0
+Version: 3.2.1
 Summary: Knowledge-base readiness validation for RAG & AI agents — config-driven, deterministic pipeline integrity checks.
 Author: COMPASS
 License: MIT

{compass_kb_validation-3.2.0 → compass_kb_validation-3.2.1}/discover_and_validate.py

@@ -26,7 +26,8 @@ EXIT_OK, EXIT_GATE_FAILED, EXIT_CONFIG = 0, 1, 2
 
 
 def parse_args(argv=None) -> argparse.Namespace:
-    p = argparse.ArgumentParser(description="Discover storage documents and validate vector chunks")
+    p = argparse.ArgumentParser(prog="compass discover",
+                                description="Discover storage documents and validate vector chunks")
     p.add_argument("--project", help="Project name (required unless --list-checks)")
     p.add_argument("--list-checks", action="store_true",
                    help="Print the vector-store check catalogue (number, name, default severity tier) and exit.")

{compass_kb_validation-3.2.0 → compass_kb_validation-3.2.1}/profile_kb.py

@@ -27,7 +27,11 @@ EXIT_OK, EXIT_NONE_REACHABLE, EXIT_CONFIG = 0, 1, 2
 
 
 def parse_args(argv=None) -> argparse.Namespace:
-    p = argparse.ArgumentParser(description="Profile pipeline stages and suggest a COMPASS config")
+    # Explicit prog: when dispatched via `compass profile` / `python -m compass profile`, the
+    # command name is stripped and sys.argv[0] can be None, which argparse's default
+    # prog=basename(sys.argv[0]) chokes on (TypeError on None). Naming it avoids that.
+    p = argparse.ArgumentParser(prog="compass profile",
+                                description="Profile pipeline stages and suggest a COMPASS config")
     p.add_argument("--project", required=True, help="Project name")
     p.add_argument("--env", default="dev", help="Environment")
     p.add_argument("--stage", choices=["storage", "graph", "vector", "all"], default="all",

{compass_kb_validation-3.2.0 → compass_kb_validation-3.2.1}/run_validation.py

@@ -29,7 +29,8 @@ EXIT_OK, EXIT_FAILURES, EXIT_CONFIG = 0, 1, 2
 
 
 def parse_args(argv=None) -> argparse.Namespace:
-    p = argparse.ArgumentParser(description="COMPASS — AI knowledge base validation")
+    p = argparse.ArgumentParser(prog="compass validate",
+                                description="COMPASS - AI knowledge base validation")
     p.add_argument("--project", required=True, help="Project name (loads config/projects/<name>.yaml)")
     p.add_argument("--env", default=None, help="Environment: dev, staging, prod")
     p.add_argument("--file", help="Validate a single file (overrides test_data)")

{compass_kb_validation-3.2.0 → compass_kb_validation-3.2.1}/tests/test_citation.py

@@ -443,3 +443,32 @@ def test_report_renders_citation_section():
     assert "Citation audit" in md and "Generation hallucination" in md
     assert "lexical" in md.lower()                    # honest-boundary note present
     assert "Where to fix" in md and "**Generation**" in md   # per-fault remediation present
+
+
+def test_citation_after_sentence_period_is_extracted():
+    # DEFECT-001: the standard "...claim. (SOURCE, p. N)" format (citation right after the
+    # sentence-ending period) must extract the claim, not silently drop the citation.
+    text = ("Growth is expected to stay robust, supported by resilient demand. "
+            "(SOURCE, June 2026, p. 1) Earnings are projected to grow.")
+    cites = citation.extract_citations(text)
+    assert len(cites) == 1
+    assert cites[0].claim.endswith("resilient demand.")     # the claim sentence, not empty
+    # multiple sentences: the claim is the LAST sentence before the citation, not the whole text
+    multi = citation.extract_citations("First point here. Second claim with detail. "
+                                       "(SOURCE, 2026, p. 2) Next.")
+    assert multi and multi[0].claim == "Second claim with detail."
+    # a mid-sentence citation still resolves to the preceding text
+    mid = citation.extract_citations("Earnings grew 12% (SOURCE, 2026, p. 3) which beat.")
+    assert mid and mid[0].claim == "Earnings grew 12%"
+
+
+def test_diagnostic_when_pattern_matches_but_nothing_extracted():
+    # DEFECT-003: a citation-like pattern with no supporting claim (extraction yields 0) must
+    # surface a diagnostic, not a silent total_citations=0 that reads as "no citations exist".
+    from types import SimpleNamespace
+    cfg = SimpleNamespace(citation_patterns=[], citation_page_tolerance=0,
+                          citation_verify_entities=False,
+                          gt_chunk_doc_field="document_title", gt_chunk_page_field="page_start")
+    res = citation.audit_citations("(SOURCE, 2026, p. 1)", None, None, cfg)
+    assert res["summary"]["total_citations"] == 0
+    assert "matched by regex but 0 were extracted" in res["summary"].get("diagnostic", "")

{compass_kb_validation-3.2.0 → compass_kb_validation-3.2.1}/tests/test_corpus_health.py

@@ -102,3 +102,15 @@ def test_cmd_freshness_end_to_end(monkeypatch, capsys):
                         if cat == "vectorstore" else store)
     assert cli.cmd_freshness([]) == 0
     assert "Freshness" in capsys.readouterr().out
+
+
+def test_freshness_warns_on_near_total_orphan_rate():
+    # DEFECT-006: a ~100% orphan rate almost always means a key/bucket mismatch, not that every
+    # source was deleted. Flag it so the user checks config before "reconciling" false orphans.
+    now = datetime(2026, 7, 10, tzinfo=timezone.utc)
+    chunks = [make_chunk(f"doc{i}", 0, s3_key=f"missing{i}.pdf") for i in range(5)]
+    store = FakeStorageAdapter(objects={})   # nothing found -> all orphaned
+    rep = freshness_report(Settings(), sample=10, now=now,
+                           vectorstore=FakeVectorStoreAdapter(chunks=chunks), storage=store)
+    assert rep["orphan_count"] == 5
+    assert "warning" in rep and "key mismatch" in rep["warning"]

{compass_kb_validation-3.2.0 → compass_kb_validation-3.2.1}/tests/test_eval_interop.py

@@ -206,3 +206,19 @@ def test_cmd_compare_end_to_end(monkeypatch, tmp_path, capsys):
     assert cli.cmd_compare(["--ground-truth", str(gt)]) == 0
     out = capsys.readouterr().out
     assert "Retrieval comparison" in out and "VERDICT" in out
+
+
+def test_diagnose_paths_reports_found_and_missing(tmp_path):
+    # DEFECT-007: when compare finds no usable cases, say which configured paths resolved on the
+    # first record and which did not, instead of a generic "check your config".
+    from compass.eval_interop import diagnose_paths
+    p = tmp_path / "gt.json"
+    p.write_text(json.dumps([{"input": "q", "context": {"chunks": [{"text": "a"}, {"text": "b"}]}}]))
+    lines = "\n".join(diagnose_paths(str(p), {
+        "gt_question_path": "question",          # absent
+        "gt_chunks_path": "context.chunks",      # present, 2 items
+        "gt_variant_path": "",                   # unset
+    }))
+    assert 'gt_question_path: "question" -> NOT FOUND' in lines
+    assert 'gt_chunks_path: "context.chunks" -> found (list, 2 item(s))' in lines
+    assert "gt_variant_path: (unset)" in lines

{compass_kb_validation-3.2.0 → compass_kb_validation-3.2.1}/tests/test_grounding.py

@@ -263,3 +263,22 @@ def test_fabrication_gate_covers_semantic_fabricated_source():
     r2 = run_audit({"input": "outlook?", "response": "Cautious.", "context.retrieved": "cautious given rates",
                     "scores": {"Groundedness": {"score": 0.9, "target": 0.7}}}, KB(), _cfg(gt_context_path="context.retrieved"))
     assert r2.has_fabrication is False
+
+
+def test_context_not_empty_when_chunks_present_defect_002():
+    # DEFECT-002: chunks nested under context.chunks must not read as "empty retrieval" just
+    # because the configured gt_context_path does not resolve for this eval schema.
+    from compass.config import Settings
+
+    class _Ad:
+        def text_search(self, *a, **k):
+            return []
+        def get_by_id(self, *a, **k):
+            return None
+
+    ev = Settings().eval
+    with_chunks = {"input": "q", "output": "answer",
+                   "context": {"type": "knowledge_retrieval", "chunks": [{"text": "a"}, {"text": "b"}]}}
+    assert run_audit(with_chunks, _Ad(), ev).context_empty is False
+    truly_empty = {"input": "q", "output": "answer", "context": {}}
+    assert run_audit(truly_empty, _Ad(), ev).context_empty is True

{compass_kb_validation-3.2.0 → compass_kb_validation-3.2.1}/tests/test_vectorstore_filters.py

@@ -83,6 +83,20 @@ def test_redis_builds_hybrid_tag_prefilter():
     assert "=>[KNN 5 @text_embedding $vec AS __score]" in client.query
 
 
+def test_text_search_preserves_spaces_for_multiword_queries():
+    # DEFECT-008: a multi-word TEXT search must keep spaces as token separators. TAG-escaping
+    # backslash-escaped them, collapsing the value into one unmatchable token so every
+    # multi-word search silently returned nothing.
+    client = _FakeRedis()
+    a = RedisVectorStoreAdapter(_cfg("redis"), client=client)
+    a.text_search("text", "Growth is robust", 5)
+    assert client.query == "@text:(Growth is robust)"      # spaces intact
+    assert "\\ " not in client.query                        # NOT the broken TAG-escaped form
+    # single words are unchanged; query-parser metacharacters are still escaped
+    assert RedisVectorStoreAdapter.escape_text("ACWI") == "ACWI"
+    assert "\\(" in RedisVectorStoreAdapter.escape_text("S&P (est)")
+
+
 # --------------------------------------------------------------------------- #
 # Qdrant - Filter(must=[FieldCondition(MatchValue|MatchAny)])
 # --------------------------------------------------------------------------- #