compass-kb-validation 2.2.0__tar.gz → 2.3.0__tar.gz

{compass_kb_validation-2.2.0 → compass_kb_validation-2.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: compass-kb-validation
-Version: 2.2.0
+Version: 2.3.0
 Summary: Knowledge-base readiness validation for RAG & AI agents — config-driven, deterministic pipeline integrity checks.
 Author: COMPASS
 License: MIT

{compass_kb_validation-2.2.0 → compass_kb_validation-2.3.0}/compass_kb_validation.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: compass-kb-validation
-Version: 2.2.0
+Version: 2.3.0
 Summary: Knowledge-base readiness validation for RAG & AI agents — config-driven, deterministic pipeline integrity checks.
 Author: COMPASS
 License: MIT

{compass_kb_validation-2.2.0 → compass_kb_validation-2.3.0}/discover_and_validate.py

@@ -67,6 +67,12 @@ def parse_args(argv=None) -> argparse.Namespace:
                    help="Also fail unless the KB Hygiene score is >= PCT (0-100). Hygiene is "
                         "the share of advisory data-quality checks that passed - data-plane "
                         "cleanliness, separate from the READY integrity gate.")
+    p.add_argument("--suggest-config", nargs="?", const="-", metavar="PATH",
+                   help="After the run, emit config refinements from what it observed "
+                        "(document count, chunk-size distribution, checks that never applied). "
+                        "Prints to stdout, or writes to PATH. Review before merging.")
+    p.add_argument("--no-tips", action="store_true",
+                   help="Suppress the end-of-run tips about relevant unused flags.")
     # --- config overrides (run scenarios without editing YAML; CLI wins over config) ---
     p.add_argument("--readiness-threshold", type=float, metavar="FRAC",
                    help="Override corpus readiness cutoff (0-1). READY when pass-rate >= this.")
@@ -240,6 +246,32 @@ _AUTH_HINTS = ("403", "forbidden", "expired", "credential", "token", "signatured
                "invalidclienttokenid", "accessdenied", "unauthor")
 
 
+def _print_tips(args, hygiene: dict | None, saved_baseline: bool) -> None:
+    """End-of-run hints about the most relevant unused flags.
+
+    Discoverability, not noise: at most three, only when the flag was NOT already used and
+    the run's own outcome makes it relevant. Suppress with --no-tips. This is why a partner
+    can run for weeks without finding --save-baseline / --unreachable-as / --suggest-config.
+    """
+    tips: list[str] = []
+    if not saved_baseline:
+        tips.append("Save this run as a baseline to catch regressions next time: "
+                    "--save-baseline baselines/{}.json".format(args.project or "kb"))
+    if hygiene and hygiene.get("score", 100) < 90 and args.quality_threshold is None:
+        tips.append("Gate CI on data-plane hygiene (not just integrity): "
+                    "--quality-threshold {}".format(int(hygiene["score"])))
+    if not getattr(args, "include_retrieval", False):
+        tips.append("Add retrieval-quality checks (can a stored vector be found via KNN?): "
+                    "--include-retrieval")
+    if not args.suggest_config:
+        tips.append("Refine your config from what this run observed: --suggest-config")
+    if not tips:
+        return
+    print("\n  Tips (--no-tips to silence):")
+    for t in tips[:3]:
+        print(f"    * {t}")
+
+
 def _resolve_expected_documents(raw, project: str):
     """Resolve the configured expected_documents, supporting ``auto``.
 
@@ -502,6 +534,16 @@ def main(argv=None) -> int:
     if args.output_format in ("junit", "all"):
         print(f"  JUnit report: {export_junit(results, base + '.xml')}")
 
+    if args.suggest_config:
+        from ingestion_validation.profiler import suggest_config_from_run
+        snippet = suggest_config_from_run(results, documents)
+        if args.suggest_config == "-":
+            print("\n" + snippet)
+        else:
+            from pathlib import Path as _P
+            _P(args.suggest_config).write_text(snippet, encoding="utf-8")
+            print(f"  Config suggestions written: {args.suggest_config} (review before merging)")
+
     # KB Hygiene: data-plane cleanliness (advisory checks), separate from the integrity
     # gate. Printed for every run; gates the build only when --quality-threshold is set.
     # Best-effort context - a failure to score it must not crash the gate decision.
@@ -539,6 +581,8 @@ def main(argv=None) -> int:
         reason = f"{len(documents)} documents < s3.min_documents ({min_docs})"
         failed_gate = True
     code = EXIT_GATE_FAILED if (failed_gate or (regression and args.fail_on_regression)) else EXIT_OK
+    if not args.no_tips:
+        _print_tips(args, hygiene, saved_baseline=args.save_baseline is not None or bool(args.against))
     print("=" * 60)
     print(f"  Results: {passed} passed, {failed} failed ({len(results)} total)")
     print(f"  Gate: {'FAILED' if failed_gate else 'PASSED'} - {reason}")

{compass_kb_validation-2.2.0 → compass_kb_validation-2.3.0}/ingestion_validation/__init__.py

@@ -5,4 +5,4 @@ end-to-end: API upload -> object storage -> knowledge graph -> vector store ->
 retrieval quality, plus cross-store consistency.
 """
 
-__version__ = "2.2.0"
+__version__ = "2.3.0"

{compass_kb_validation-2.2.0 → compass_kb_validation-2.3.0}/ingestion_validation/profiler.py

@@ -537,7 +537,79 @@ def apply_profile_to_settings(settings: Any, profiles: dict) -> list[str]:
     return applied
 
 
+def suggest_config_from_run(results: list, documents: list) -> str:
+    """Suggest config refinements from what a validation RUN actually observed.
+
+    Complements ``emit_project_config`` (pre-run backend introspection) with the things
+    only a run reveals: the real document count, the observed chunk-size distribution, and
+    the checks that were NEUTRAL on every document - checks that never apply to this KB and
+    are pure noise (e.g. section_metadata when nothing carries sections). Returns a review-
+    and-merge snippet, never a file to apply blind - a wrong config key would be exactly the
+    silent-failure trap COMPASS exists to catch, so these are suggestions with the evidence.
+    """
+    import statistics
+
+    n_docs = len(documents)
+    avg_chars: list[int] = []
+    # label -> [evaluated_count, neutral_count]; a check neutral everywhere never applies.
+    support: dict[str, list[int]] = {}
+    for item in results:
+        for st in getattr(item.get("result"), "steps", []) or []:
+            details = getattr(st, "details", {}) or {}
+            cv = details.get("chunk_view")
+            if isinstance(cv, dict) and cv.get("avg_chars"):
+                avg_chars.append(int(cv["avg_chars"]))
+            status = getattr(getattr(st, "status", None), "value", "")
+            if status == "SKIPPED":
+                continue   # a step that did not run says nothing about applicability
+            for ch in getattr(st, "checks", []) or []:
+                lbl = ch.get("label", "")
+                if not lbl:
+                    continue
+                rec = support.setdefault(lbl, [0, 0])
+                if ch.get("supported", True) is False:
+                    rec[1] += 1
+                else:
+                    rec[0] += 1
+
+    never_applies = sorted(lbl for lbl, (ev, neu) in support.items() if ev == 0 and neu > 0)
+
+    lines = [
+        "# --- compass discover --suggest-config -------------------------------------",
+        "# Refinements from THIS run's observations. Review each, then merge into your",
+        "# project YAML - nothing here is applied automatically.",
+        "",
+        "# Documents discovered this run. Pin it to alert on drift (or use 'auto' to",
+        "# baseline against the previous run automatically):",
+        "discovery:",
+        "  extra:",
+        f"    expected_documents: {n_docs}",
+    ]
+    if avg_chars:
+        median_size = int(statistics.median(avg_chars))
+        lines += [
+            "",
+            f"# Observed chunk size: median {median_size} chars across {len(avg_chars)} document(s)",
+            f"# (range {min(avg_chars)}-{max(avg_chars)}). Set expected bounds if you gate on size:",
+            "# redis:",
+            f"#   chunk_size: {median_size}",
+        ]
+    if never_applies:
+        lines += [
+            "",
+            "# These checks were NOT APPLICABLE to any document this run (neutral every time) -",
+            "# the backend/data cannot answer them. Disabling removes them from the tally and",
+            "# the report noise. Verify each is genuinely irrelevant before disabling:",
+        ]
+        for lbl in never_applies:
+            lines.append(f"#   - {lbl}")
+    else:
+        lines += ["", "# Every check applied to at least one document - nothing to disable."]
+    return "\n".join(lines) + "\n"
+
+
 __all__ = [
     "profile_vectorstore", "profile_storage", "profile_graph", "profile_kb",
-    "apply_profile_to_settings", "suggest_config",
+    "apply_profile_to_settings", "suggest_config", "emit_project_config",
+    "suggest_config_from_run",
 ]

{compass_kb_validation-2.2.0 → compass_kb_validation-2.3.0}/ingestion_validation/report/__init__.py

@@ -224,9 +224,14 @@ def _result_payload(name: str, r: PipelineResult) -> dict:
             continue
         order.append(s.step_name)
         metrics = {k: v for k, v in s.details.items() if k not in ("screenshot", "chunk_view", "rdf_triples") and isinstance(v, (int, float, str))}
-        # Graph metadata steps export RDF triples for the knowledge-graph view.
+        # Graph metadata steps export RDF triples for the knowledge-graph view, which
+        # renders ONLY predicate + value. The raw triples also carry `subject` and
+        # `predicate_uri` - both full IRIs, repeated on every triple - that nothing reads;
+        # they were ~83% of a graph-heavy report's payload. Keep only the rendered fields.
         raw_triples = s.details.get("rdf_triples")
-        triples = raw_triples[:_KG_TRIPLE_CAP] if isinstance(raw_triples, list) and raw_triples else None
+        triples = ([{"predicate": t.get("predicate", ""), "value": t.get("value", "")}
+                    for t in raw_triples[:_KG_TRIPLE_CAP]]
+                   if isinstance(raw_triples, list) and raw_triples else None)
         # Attach a remediation hint to each failed, evaluable check (advisory only).
         checks_out = [
             ({**c, "hint": hint_for(c.get("label", ""))}

{compass_kb_validation-2.2.0 → compass_kb_validation-2.3.0}/tests/test_discover_cli.py

@@ -144,3 +144,49 @@ def test_quality_threshold_gates_on_hygiene():
     assert not ok
     # no threshold set -> hygiene never gates
     assert not dav.gate_failed(10, 10, "READY", None, False, hygiene_score=10.0)[0]
+
+
+def _run_case(name, avg_chars, section_supported):
+    from ingestion_validation.models import PipelineResult, StepResult, StepStatus
+    r = PipelineResult(document_id=name, file_name=f"{name}.pdf")
+    checks = [{"label": "Chunks exist", "passed": True},
+              {"label": "Section metadata present", "passed": False, "supported": section_supported}]
+    r.add_step(StepResult("Redis Chunks", StepStatus.PASSED, "ok", checks=checks,
+                          details={"chunk_view": {"count": 10, "total_chars": avg_chars * 10, "avg_chars": avg_chars}}))
+    r.finalize()
+    return {"name": name, "result": r}
+
+
+def test_suggest_config_from_run_reports_count_size_and_dead_checks():
+    from ingestion_validation.profiler import suggest_config_from_run
+    docs = [{"file_name": f"d{i}.pdf"} for i in range(12)]
+    # section metadata is neutral (unsupported) on every doc -> flagged as never-applicable
+    results = [_run_case(f"d{i}", 800 + i * 20, section_supported=False) for i in range(12)]
+    out = suggest_config_from_run(results, docs)
+    assert "expected_documents: 12" in out
+    assert "median 910 chars" in out                      # observed size distribution
+    assert "Section metadata present" in out              # dead check surfaced for disabling
+
+
+def test_suggest_config_does_not_flag_a_check_that_applied_somewhere():
+    from ingestion_validation.profiler import suggest_config_from_run
+    docs = [{"file_name": "d0.pdf"}, {"file_name": "d1.pdf"}]
+    # section metadata is supported (applies) on d1 -> must NOT be suggested for disabling
+    results = [_run_case("d0", 900, section_supported=False),
+               _run_case("d1", 900, section_supported=True)]
+    out = suggest_config_from_run(results, docs)
+    assert "Section metadata present" not in out
+    assert "nothing to disable" in out
+
+
+def test_post_run_tips_are_contextual_and_suppressible(capsys):
+    class _A:
+        project = "wp-kb"; quality_threshold = None; include_retrieval = False; suggest_config = None
+    dav._print_tips(_A(), {"score": 56.1, "tier": "POOR"}, saved_baseline=False)
+    out = capsys.readouterr().out
+    assert "--save-baseline" in out and "--quality-threshold 56" in out and "--include-retrieval" in out
+
+    class _B:
+        project = "wp-kb"; quality_threshold = 80; include_retrieval = True; suggest_config = "-"
+    dav._print_tips(_B(), {"score": 98}, saved_baseline=True)   # everything used, clean run
+    assert capsys.readouterr().out == ""                        # -> no tips

{compass_kb_validation-2.2.0 → compass_kb_validation-2.3.0}/tests/test_reporting.py

@@ -545,3 +545,18 @@ def test_hygiene_gauge_shows_score_and_worst_beside_the_verdict():
     assert "KB Hygiene" in gauge and "POOR" in gauge
     assert "worst:" in gauge and "Chunk Structure" in gauge
     assert report_mod._hygiene_gauge(None) == ""             # no advisory -> no gauge
+
+
+def test_embedded_triples_keep_only_rendered_fields():
+    # The KG view renders predicate + value only; subject and predicate_uri (full IRIs,
+    # repeated per triple) were ~83% of a graph-heavy payload and are never read.
+    r = PipelineResult(document_id="d", file_name="d.pdf")
+    r.add_step(StepResult("Neptune Metadata", StepStatus.PASSED, "ok",
+        checks=[{"label": "Found", "passed": True}],
+        details={"rdf_triples": [
+            {"subject": "http://ns/doc/1", "predicate": "documentId",
+             "predicate_uri": "http://ns/documentId", "value": "DOC-1"}]}))
+    r.finalize()
+    triples = report_mod._result_payload("d.pdf", r)["steps"]["Neptune Metadata"]["triples"]
+    assert triples == [{"predicate": "documentId", "value": "DOC-1"}]
+    assert "subject" not in triples[0] and "predicate_uri" not in triples[0]