compass-kb-validation 2.6.0__tar.gz → 2.6.1__tar.gz

{compass_kb_validation-2.6.0 → compass_kb_validation-2.6.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: compass-kb-validation
-Version: 2.6.0
+Version: 2.6.1
 Summary: Knowledge-base readiness validation for RAG & AI agents — config-driven, deterministic pipeline integrity checks.
 Author: COMPASS
 License: MIT

{compass_kb_validation-2.6.0 → compass_kb_validation-2.6.1}/compass_kb_validation.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: compass-kb-validation
-Version: 2.6.0
+Version: 2.6.1
 Summary: Knowledge-base readiness validation for RAG & AI agents — config-driven, deterministic pipeline integrity checks.
 Author: COMPASS
 License: MIT

{compass_kb_validation-2.6.0 → compass_kb_validation-2.6.1}/discover_and_validate.py

@@ -138,14 +138,14 @@ def _resolve_skip_steps(settings, *, include_graphdb: bool, include_retrieval: b
     (BUG-22). ``--keep s3_storage`` overrides the auto-skip.
     """
     skip = ["API Upload"]
-    # Graph validation runs by default when a graph block is actually configured (a
-    # sparql_query or an endpoint) - a configured neptune:/graphdb: block should not be
-    # silently ignored just because --include-graphdb was not passed. Opt out with
-    # --skip neptune_metadata (or graphdb_metadata). --include-graphdb still forces it on.
+    # Graph validation runs by default when THIS PROJECT declares a per-document graph
+    # query (sparql_query) - the signal that graph validation is actually intended. It must
+    # NOT key off `neptune.endpoint`, which resolves from a shared NEPTUNE_ENDPOINT env var:
+    # doing so turned graph on for every project whenever that env var was set, so a project
+    # with no graph block inherited another project's Neptune and failed all 19 graph checks
+    # (regression). Opt out with --skip neptune_metadata; --include-graphdb still forces on.
     nep, gdb = settings.neptune, settings.graphdb
-    graph_configured = bool(
-        getattr(nep, "sparql_query", "") or getattr(nep, "endpoint", "")
-        or getattr(gdb, "sparql_query", "") or getattr(gdb, "base_url", ""))
+    graph_configured = bool(getattr(nep, "sparql_query", "") or getattr(gdb, "sparql_query", ""))
     if not (include_graphdb or graph_configured):
         skip += ["GraphDB Metadata", "Neptune Metadata"]
     if not include_retrieval:

{compass_kb_validation-2.6.0 → compass_kb_validation-2.6.1}/ingestion_validation/__init__.py

@@ -5,4 +5,4 @@ end-to-end: API upload -> object storage -> knowledge graph -> vector store ->
 retrieval quality, plus cross-store consistency.
 """
 
-__version__ = "2.6.0"
+__version__ = "2.6.1"

{compass_kb_validation-2.6.0 → compass_kb_validation-2.6.1}/ingestion_validation/cli.py

@@ -1318,7 +1318,7 @@ def cmd_grounding_audit(argv: list[str]) -> int:
     g = p.add_mutually_exclusive_group(required=True)
     g.add_argument("--eval-result", metavar="FILE", help="One eval record as a single JSON object.")
     g.add_argument("--eval-results", metavar="FILE",
-                   help="Batch: a JSONL file (one JSON object per LINE, not a JSON array). "
+                   help="Batch: JSONL (one object per line) or a JSON array - both accepted. "
                         "`compass ask --format eval` emits a compatible file.")
     p.add_argument("--format", choices=["markdown", "json"], default="markdown")
     p.add_argument("--output", help="Write the report to a file (default: stdout).")
@@ -1352,15 +1352,23 @@ def cmd_grounding_audit(argv: list[str]) -> int:
                 return 2
             records = [one]
         else:
-            for n, line in enumerate(Path(args.eval_results).read_text(encoding="utf-8").splitlines(), 1):
-                if not line.strip():
-                    continue
-                try:
-                    records.append(_json.loads(line))
-                except Exception:  # noqa: BLE001 - point at the offending line, not char 1
-                    print(f"  --eval-results is JSONL (one JSON object per line); line {n} is not valid JSON. "
-                          "A JSON array is not accepted here - see `compass ask --format eval`.")
-                    return 2
+            raw = Path(args.eval_results).read_text(encoding="utf-8")
+            # Accept BOTH a JSON array and JSONL (one object per line) - a whole-file array
+            # is a natural thing to pass, so handle it instead of erroring.
+            stripped = raw.lstrip()
+            if stripped.startswith("["):
+                arr = _json.loads(raw)
+                records = [r for r in arr if isinstance(r, dict)]
+            else:
+                for n, line in enumerate(raw.splitlines(), 1):
+                    if not line.strip():
+                        continue
+                    try:
+                        records.append(_json.loads(line))
+                    except Exception:  # noqa: BLE001 - point at the offending line, not char 1
+                        print(f"  --eval-results: line {n} is not valid JSON. Expected JSONL "
+                              "(one object per line) or a single JSON array.")
+                        return 2
     except Exception as exc:  # noqa: BLE001 - clean message, never a traceback
         print(f"  Could not read eval input: {exc}")
         return 2
@@ -1381,9 +1389,12 @@ def cmd_grounding_audit(argv: list[str]) -> int:
     else:
         print(body)
 
-    fabricated = sum(1 for r in results if r.wrong_claims)
+    # Gate on fabrications in EITHER path: a count that disagrees with the KB, or a cited
+    # document absent from it. (Previously only structured count mismatches counted, so a
+    # semantic fabricated-source case exited 0 despite being flagged.)
+    fabricated = sum(1 for r in results if r.has_fabrication)
     if fabricated:
-        print(f"\n  {fabricated}/{len(results)} case(s) contain claims that disagree with the KB.")
+        print(f"\n  {fabricated}/{len(results)} case(s) contain a claim or citation not backed by the KB.")
         if args.fail_on_fabrication:
             return 1
     return 0

{compass_kb_validation-2.6.0 → compass_kb_validation-2.6.1}/ingestion_validation/grounding.py

@@ -309,6 +309,14 @@ class GroundingResult:
     def verified_claims(self) -> list[ClaimVerdict]:
         return [v for v in self.claim_verdicts if v.verifiable]
 
+    @property
+    def has_fabrication(self) -> bool:
+        """A count that disagrees with the KB, OR a cited document absent from the KB -
+        the two deterministic 'the model asserted something not in the KB' signals that
+        --fail-on-fabrication gates on (semantic fabrications were previously missed)."""
+        return bool(self.wrong_claims) or any(
+            "not found in KB" in s.get("label", "") for s in self.signals)
+
     def layer_diagnosis(self) -> str:
         """Deterministic layer attribution - which stage is implicated, no quality score."""
         if self.query_type == "structured" and self.wrong_claims:

{compass_kb_validation-2.6.0 → compass_kb_validation-2.6.1}/ingestion_validation/profiler.py

@@ -242,11 +242,15 @@ def _configured_graph(settings: Any) -> tuple[Any, str]:
     configured Neptune endpoint. Returns ``(None, "")`` when neither is set.
     """
     gdb = getattr(settings, "graphdb", None)
-    if gdb is not None and (getattr(gdb, "endpoint", "")
-                            or (getattr(gdb, "base_url", "") and getattr(gdb, "repository", ""))):
+    if gdb is not None and getattr(gdb, "sparql_query", "") and (
+            getattr(gdb, "endpoint", "") or (getattr(gdb, "base_url", "") and getattr(gdb, "repository", ""))):
         return gdb, gdb.provider
     nep = getattr(settings, "neptune", None)
-    if nep is not None and getattr(nep, "endpoint", ""):
+    # Require the project to declare a per-document sparql_query, NOT just a reachable
+    # endpoint: neptune.endpoint resolves from the shared NEPTUNE_ENDPOINT env var, so keying
+    # only off it made an unrelated project profile (and then validate) another project's
+    # graph when the env var was set.
+    if nep is not None and getattr(nep, "sparql_query", "") and getattr(nep, "endpoint", ""):
         return nep, "neptune"
     return None, ""
 

{compass_kb_validation-2.6.0 → compass_kb_validation-2.6.1}/ingestion_validation/remediation.py

@@ -116,14 +116,19 @@ _HINTS: list[tuple[str, str]] = [
      "Stored vectors were embedded with a different model than the one declared/serving: retrieval is "
      "silently wrong. Re-embed the corpus with the declared model, or correct the declared model."),
     (r"exact-term retrieval|keyword-index completeness|findable lexically",
-     "Chunks are not findable by exact keyword: the text field is not indexed for lexical search. Enable "
-     "hybrid (keyword + vector) retrieval, or index the text field for full-text search."),
+     "Exact-keyword retrieval is weak. Either the text field is not indexed for lexical search, OR - in a "
+     "large corpus of similar documents - a common term returns many near-identical hits and the right one "
+     "does not rank first. Confirm the text field is indexed; if it is, add hybrid (keyword+vector) or a "
+     "metadata filter, or raise the similarity threshold."),
     (r"negative retrieval|off-topic",
-     "An off-topic query still scored high: embeddings are too generic or the index is polluted by "
-     "boilerplate. Improve chunk quality and strip boilerplate so unrelated queries do not match."),
+     "An off-topic query still scored high: embeddings may be too generic, or the corpus is dense with "
+     "similar/boilerplate content so unrelated queries still match. Strip boilerplate and consider a "
+     "higher similarity threshold or metadata pre-filtering."),
     (r"recall|mrr|paraphrase|boundary|empty query",
-     "The document is not retrievable as expected for real queries: embeddings may not match the query "
-     "path, or the filter field is not indexed. Verify the retrieval/query configuration."),
+     "Retrieval is weak for real queries. This can be a config problem (embeddings not matching the query "
+     "path, or an unindexed filter field) OR corpus-specific: a large set of highly similar documents makes "
+     "paraphrase/boundary disambiguation genuinely hard. Verify the retrieval config first; if it is sound, "
+     "adjust the similarity threshold or filter by metadata."),
 
     # ---- Ingestion API / storage / connectivity --------------------------------------
     (r"http status|graphql|success status|valid json|api ",

{compass_kb_validation-2.6.0 → compass_kb_validation-2.6.1}/ingestion_validation/report/__init__.py

@@ -46,6 +46,10 @@ def _step_icons() -> dict:
 _BAND = {"PASSED": "ok", "FAILED": "bad", "ERROR": "bad", "PARTIAL": "warn", "SKIPPED": "na", "EMPTY": "na"}
 # Cap RDF triples embedded per step for the knowledge-graph view (bounds payload size).
 _KG_TRIPLE_CAP = 80
+# At scale, a passed document keeps its chunk previews if it has at most this many - small
+# docs are cheap and their preview is what you drill into. Only large docs (heavy previews)
+# are trimmed, per document, rather than a corpus-wide blanket that emptied even 1-chunk docs.
+_CHUNK_ITEMS_KEEP = 10
 # Max folder/category rows in the readiness roll-up (worst pass-rate first); keeps it
 # a signal rather than one row per document when folders are fine-grained.
 _FOLDER_DISPLAY_CAP = 20
@@ -1875,8 +1879,13 @@ def generate_batch_dashboard(
                     ]
             if not drillable:
                 trimmed += 1
-                if c.get("chunks"):
-                    c["chunks"] = {**c["chunks"], "items": []}
+                # Drop chunk previews PER DOCUMENT by its own size, not a corpus-wide
+                # blanket: a doc with a handful of chunks costs almost nothing and its
+                # preview is exactly what you drill into, so keep it. Only large docs
+                # (where the previews are actually heavy) are trimmed.
+                ch = c.get("chunks")
+                if ch and len(ch.get("items") or []) > _CHUNK_ITEMS_KEEP:
+                    c["chunks"] = {**ch, "items": []}
     trim_note = (
         f'<div class="meta-item"><span class="meta-key">Detail:</span> triples + verbose fields '
         f'trimmed for {trimmed} passed doc(s); check names + status kept</div>'

{compass_kb_validation-2.6.0 → compass_kb_validation-2.6.1}/tests/test_grounding.py

@@ -248,3 +248,18 @@ def test_emit_config_suggests_aggregatable_fields():
     out = emit_project_config(profiles, project="demo")
     assert "aggregatable_fields:" in out
     assert "field: document_type" in out and "count_by: file_name" in out
+
+
+def test_fabrication_gate_covers_semantic_fabricated_source():
+    # --fail-on-fabrication must fire on a cited document absent from the KB (semantic),
+    # not only structured count mismatches - previously it exited 0 on a fabricated source.
+    class KB:
+        supports_aggregation = True
+        def text_search(self, f, v, limit=1): return [] if "fake" in v.lower() else [{"x": 1}]
+    r = run_audit({"input": "summary?", "response": "Per **Fake Report.pdf**, x happened.",
+                   "context.retrieved": "c", "scores": {}}, KB(), _cfg(gt_context_path="context.retrieved"))
+    assert r.has_fabrication is True
+    # a clean case does not trip the gate
+    r2 = run_audit({"input": "outlook?", "response": "Cautious.", "context.retrieved": "cautious given rates",
+                    "scores": {"Groundedness": {"score": 0.9, "target": 0.7}}}, KB(), _cfg(gt_context_path="context.retrieved"))
+    assert r2.has_fabrication is False

{compass_kb_validation-2.6.0 → compass_kb_validation-2.6.1}/tests/test_neptune_bugfixes2.py

@@ -51,6 +51,7 @@ def test_configured_graph_prefers_graphdb():
     s = Settings()
     s.graphdb.base_url = "http://gdb"
     s.graphdb.repository = "repo"
+    s.graphdb.sparql_query = "SELECT ?s WHERE { ?s ?p ?o {filter_clause} }"
     cfg, name = _configured_graph(s)
     assert cfg is s.graphdb and name == s.graphdb.provider
 
@@ -58,6 +59,7 @@ def test_configured_graph_prefers_graphdb():
 def test_configured_graph_falls_back_to_neptune():
     s = Settings()
     s.neptune.endpoint_default = "cluster.example.neptune.amazonaws.com"
+    s.neptune.sparql_query = "SELECT ?s WHERE { ?s ?p ?o {filter_clause} }"
     cfg, name = _configured_graph(s)
     assert cfg is s.neptune and name == "neptune"
 
@@ -67,17 +69,27 @@ def test_configured_graph_none_when_unset():
     assert cfg is None and name == ""
 
 
+def test_configured_graph_ignores_env_only_endpoint(monkeypatch):
+    # Regression (#18): neptune.endpoint resolves from the shared NEPTUNE_ENDPOINT env var,
+    # so a project with NO graph query (no graph block declared) must NOT be treated as
+    # graph-configured just because that env var is set for another project.
+    monkeypatch.setenv("NEPTUNE_ENDPOINT", "https://someone-elses-cluster.example:8182")
+    cfg, name = _configured_graph(Settings())   # no sparql_query declared
+    assert cfg is None and name == ""
+
+
 def test_profile_graph_reports_neptune_not_missing(monkeypatch):
     from ingestion_validation import profiler
     s = Settings()
     s.neptune.endpoint_default = "cluster.example.neptune.amazonaws.com"
+    s.neptune.sparql_query = "SELECT ?s WHERE { ?s ?p ?o {filter_clause} }"
 
     class _FakeGraph:
         def reachable(self):
             return False
     monkeypatch.setattr(profiler, "get_adapter", lambda *a, **k: _FakeGraph())
     prof = profiler.profile_graph(s)
-    # It must not claim "no graph configured" - it now sees the neptune block.
+    # It must not claim "no graph configured" - it now sees the neptune block (with a query).
     assert prof["provider"] == "neptune"
     assert not any("No graph endpoint" in n for n in prof["notes"])
 

{compass_kb_validation-2.6.0 → compass_kb_validation-2.6.1}/tests/test_profiler.py

@@ -172,7 +172,7 @@ def test_profile_storage_no_bucket_is_neutral():
 def test_profile_graph_collects_predicates(monkeypatch):
     rows = [{"p": "http://ex/eTag"}, {"p": "http://ex/name"}]
     _patch(monkeypatch, FakeGraphAdapter(rows=rows, reachable=True))
-    s = Settings(graphdb=GraphDBConfig(base_url="http://g", repository="repo"))
+    s = Settings(graphdb=GraphDBConfig(base_url="http://g", repository="repo", sparql_query="SELECT ?s WHERE { ?s ?p ?o {filter_clause} }"))
     p = profiler.profile_graph(s)
     assert p["reachable"] is True
     assert "http://ex/eTag" in p["predicates"]
@@ -190,7 +190,7 @@ def test_profile_graph_predicate_coverage_suggests_required(monkeypatch):
     # can be computed. eTag 10/10, name 9/10 -> required; rare 1/10 -> optional.
     rows = [{"p": "http://ex/eTag", "c": "10"}, {"p": "http://ex/name", "c": "9"}, {"p": "http://ex/rare", "c": "1"}]
     _patch(monkeypatch, FakeGraphAdapter(rows=rows, reachable=True))
-    s = Settings(graphdb=GraphDBConfig(base_url="http://g", repository="repo"))
+    s = Settings(graphdb=GraphDBConfig(base_url="http://g", repository="repo", sparql_query="SELECT ?s WHERE { ?s ?p ?o {filter_clause} }"))
     p = profiler.profile_graph(s)
     assert p["subjects"] == 10
     assert len(p["coverage"]) == 3

{compass_kb_validation-2.6.0 → compass_kb_validation-2.6.1}/tests/test_reporting.py

@@ -265,8 +265,9 @@ def test_scale_guard_trims_passed_docs(tmp_path):
         r.add_step(StepResult(
             "Redis Chunks", status, "x",
             checks=[{"label": "Chunks exist", "passed": status is StepStatus.PASSED, "detail": "9 chunks, verbose"}],
-            details={"chunk_view": {"count": 9, "total_chars": 100, "avg_chars": 11,
-                                    "items": [{"key": "k", "length": 11, "preview": "p", "fields": {}}]}},
+            details={"chunk_view": {"count": 15, "total_chars": 100, "avg_chars": 11,
+                                    "items": [{"key": f"k{j}", "length": 11, "preview": "p", "fields": {}}
+                                              for j in range(15)]}},  # >_CHUNK_ITEMS_KEEP -> trimmed at scale
         ))
         r.add_step(StepResult(
             "Neptune Metadata", StepStatus.PASSED, "graph",
@@ -685,3 +686,26 @@ def test_recommendations_section_only_with_fixable_issues():
     # clean run -> nothing rendered (follows "only when issues exist")
     clean_agg = report_mod._aggregate([case(i, clean=True) for i in range(10)])
     assert report_mod._recommendations_section(clean_agg) == ""
+
+
+def test_chunk_previews_kept_per_document_not_blanket_at_scale():
+    # A small doc's chunk previews are cheap and are what you drill into; only large docs
+    # are trimmed at scale - previously the whole corpus was blanked, emptying 1-chunk docs.
+    def case(name, n_chunks):
+        r = PipelineResult(document_id=name, file_name=f"{name}.pdf")
+        items = [{"key": f"c{j}", "length": 10, "preview": "text", "fields": {}} for j in range(n_chunks)]
+        r.add_step(StepResult("Redis Chunks", StepStatus.PASSED, "ok",
+                              checks=[{"label": "Chunks exist", "passed": True}],
+                              details={"chunk_view": {"count": n_chunks, "total_chars": 10 * n_chunks,
+                                                      "avg_chars": 10, "items": items}}))
+        r.finalize()
+        return {"name": name, "result": r}
+    cases = [case("small", 2)] + [case(f"big{i}", 20) for i in range(250)]  # > embed_detail_cap
+    html = open(generate_batch_dashboard(cases, __import__("tempfile").mktemp(suffix=".html"),
+                                         embed_detail_cap=200), encoding="utf-8").read()
+    m = re.search(r"const REPORT = (\{.*?\});", html, re.S)
+    assert m is not None
+    payload = json.loads(m.group(1))
+    by = {c["name"]: c for c in payload["cases"]}
+    assert by["small"]["chunks"]["items"]          # 2-chunk doc keeps its previews
+    assert by["big0"]["chunks"]["items"] == []     # 20-chunk doc is trimmed