compass-kb-validation 3.1.0__tar.gz → 3.2.0__tar.gz

{compass_kb_validation-3.1.0 → compass_kb_validation-3.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: compass-kb-validation
-Version: 3.1.0
+Version: 3.2.0
 Summary: Knowledge-base readiness validation for RAG & AI agents — config-driven, deterministic pipeline integrity checks.
 Author: COMPASS
 License: MIT

{compass_kb_validation-3.1.0 → compass_kb_validation-3.2.0}/compass/__init__.py

@@ -5,4 +5,4 @@ end-to-end: API upload -> object storage -> knowledge graph -> vector store ->
 retrieval quality, plus cross-store consistency.
 """
 
-__version__ = "3.1.0"
+__version__ = "3.2.0"

{compass_kb_validation-3.1.0 → compass_kb_validation-3.2.0}/compass/cli.py

@@ -36,6 +36,13 @@ pipeline:
   critical_steps: []
   readiness_threshold: 1.0       # READY when document pass-rate >= this (e.g. 0.99)
 
+# Advisory checks you have ACCEPTED as expected for this corpus (e.g. an insurance KB where
+# near-duplicate boilerplate is normal). They drop out of the hygiene / Platform Health scores
+# but are still counted and shown ("N suppressed"). Only advisory checks - a hard gating failure
+# is never suppressed. Ad-hoc equivalent: `compass discover --suppress "Near-duplicate embeddings"`.
+# suppressions:
+#   - "Near-duplicate embeddings"
+
 environments:
   dev:
     redis:

{compass_kb_validation-3.1.0 → compass_kb_validation-3.2.0}/compass/config/__init__.py

@@ -750,6 +750,12 @@ class Settings:
     display_name: str = ""
     environment: str = "dev"
     skip_steps: list[str] = field(default_factory=list)
+    # Advisory check labels the project has ACCEPTED as expected for this corpus (e.g. an
+    # insurance KB where near-duplicate boilerplate is normal). Suppressed advisories are
+    # excluded from the hygiene / Platform Health scores but still counted and shown ("N
+    # suppressed"). Only advisory (warn) checks can be suppressed - a hard gating failure
+    # always counts, so a known-issue can never mask a real failure.
+    suppressions: list[str] = field(default_factory=list)
     # Advisories raised while loading (e.g. an unrecognized top-level block that was
     # ignored). Surfaced by validate_config so a silently-dropped key is never invisible.
     config_warnings: list[str] = field(default_factory=list)
@@ -760,7 +766,7 @@ class Settings:
 # --------------------------------------------------------------------------- #
 # Non-service top-level keys that are legitimately not a service block.
 _META_KEYS = frozenset({"default_env", "display_name", "environments", "skip_steps",
-                        "pipeline_dependencies"})
+                        "pipeline_dependencies", "suppressions"})
 # The service-config blocks that may appear at the top level or under an environment.
 # Single source of truth: the flatten loop AND the unknown-key check both read this, so a
 # new block can never be added to one and forgotten in the other (which is exactly how
@@ -903,6 +909,7 @@ def load_settings(
         display_name=merged.get("display_name", ""),
         environment=resolved_env,
         skip_steps=list(merged.get("skip_steps", [])),
+        suppressions=list(merged.get("suppressions", [])),
     )
     # The embedding block's declared dimension IS the index's expected dimension. Cross-fill
     # it into the vector-store check when that check has no explicit value, so a project

{compass_kb_validation-3.1.0 → compass_kb_validation-3.2.0}/compass/report/__init__.py

@@ -317,7 +317,8 @@ def _result_payload(name: str, r: PipelineResult) -> dict:
     }
 
 
-def _aggregate(cases: list[dict], hide_skipped: bool = True) -> dict:
+def _aggregate(cases: list[dict], hide_skipped: bool = True,
+               suppress: set[str] | None = None) -> dict:
     total = len(cases)
     passed = sum(1 for c in cases if c["overall"] == "PASSED")
     failed = sum(1 for c in cases if c["overall"] == "FAILED")
@@ -330,6 +331,12 @@ def _aggregate(cases: list[dict], hide_skipped: bool = True) -> dict:
     # check is either a HARD failure (fails the step) or an ADVISORY warning (does not).
     # Neutral checks (the backend/config can't evaluate them) are counted separately.
     checks_hard = checks_advisory = checks_neutral = 0
+    # Advisory checks a project has explicitly accepted (config `suppressions`). They are
+    # excluded from scoring - like neutral checks - but tallied and shown, never hidden. HARD
+    # (gating) checks can NEVER be suppressed, so an accepted known-issue can't mask a failure.
+    checks_suppressed = 0
+    suppressed: Counter = Counter()
+    suppress = suppress or set()
     cat = {name: {"total": 0, "passed": 0, "adv_total": 0, "adv_passed": 0} for name, _ in _CAT_ORDER}
     # Which checks fail across the MOST documents. One systemic cause usually shows up as
     # one label repeated over many documents, so ranking by document count points at the
@@ -369,10 +376,19 @@ def _aggregate(cases: list[dict], hide_skipped: bool = True) -> dict:
                 if ch.get("supported", True) is False:
                     checks_neutral += 1  # neutral — backend can't evaluate this check
                     continue
-                k = _categorise(ch.get("label", ""), step_name)
+                _label = ch.get("label", "")
+                _sev = "warn" if str(ch.get("severity", "hard")).lower() == "warn" else "hard"
+                # Suppressed = an ADVISORY check the project accepted as expected (e.g. known
+                # boilerplate near-duplicates). Excluded from scoring but counted + shown.
+                # Only advisory can be suppressed - a hard/gating check ignores the list.
+                if _sev == "warn" and _label in suppress:
+                    checks_suppressed += 1
+                    suppressed[_label] += 1
+                    continue
+                k = _categorise(_label, step_name)
                 cat[k]["total"] += 1
                 ch_passed = bool(ch.get("passed"))
-                sev = "warn" if str(ch.get("severity", "hard")).lower() == "warn" else "hard"
+                sev = _sev
                 # Hygiene tracks ADVISORY (WARN) checks only - the quality BEYOND the
                 # integrity gate. HARD checks passing is what READY already reports, so
                 # counting them would just pull hygiene toward 100 and hide the signal.
@@ -387,9 +403,17 @@ def _aggregate(cases: list[dict], hide_skipped: bool = True) -> dict:
                     checks_advisory += 1
                 else:
                     checks_hard += 1
-                label = ch.get("label", "")
-                rec = offenders.setdefault(label, {"label": label, "docs": 0, "severity": sev, "step": step_name})
+                label = _label
+                rec = offenders.setdefault(
+                    label, {"label": label, "docs": 0, "severity": sev, "step": step_name, "detail": ""})
                 rec["docs"] += 1
+                # Keep one concrete, measured example (the check's own detail, e.g.
+                # "4200 (~3000 expected)") so the Action Plan can show a specific number
+                # instead of a generic cause. First non-empty wins - deterministic.
+                if not rec["detail"]:
+                    _dv = str(ch.get("detail") or "").strip()
+                    if _dv:
+                        rec["detail"] = _dv
                 # a label is HARD overall if it ever gates
                 if sev == "hard":
                     rec["severity"] = "hard"
@@ -433,6 +457,11 @@ def _aggregate(cases: list[dict], hide_skipped: bool = True) -> dict:
         # rests on less than the full pipeline.
         "docs_incomplete": docs_incomplete,
         "hygiene": hygiene,
+        # Accepted advisory issues (config `suppressions`): count + per-label tally, shown
+        # so a suppressed warning is disclosed, never silently dropped.
+        "checks_suppressed": checks_suppressed,
+        "suppressed": [{"label": lbl, "docs": n} for lbl, n in
+                       sorted(suppressed.items(), key=lambda kv: (-kv[1], kv[0]))],
     }
 
 
@@ -444,15 +473,16 @@ _HYGIENE_TIERS = (
 )
 
 
-def hygiene_from_results(results: list[dict]) -> dict | None:
+def hygiene_from_results(results: list[dict], suppress: list[str] | None = None) -> dict | None:
     """Public: compute the KB Hygiene score from a list of ``{name, result}`` items.
 
     Same deterministic scoring the report uses, exposed so the CLI can gate on it
-    (``--quality-threshold``) without rendering HTML. Returns None if there are no
-    advisory checks to score.
+    (``--quality-threshold``) without rendering HTML. ``suppress`` accepts the same accepted
+    advisory labels as the report, so the gate and the report agree. Returns None if there
+    are no advisory checks to score.
     """
     cases = [_result_payload(item.get("name", item["result"].file_name), item["result"]) for item in results]
-    return _aggregate(cases).get("hygiene")
+    return _aggregate(cases, suppress=set(suppress or [])).get("hygiene")
 
 
 def _hygiene_score(cat: dict) -> dict | None:
@@ -693,6 +723,12 @@ def _exec_summary(agg: dict, corpus: dict | None = None, cases: list[dict] | Non
                      f'these documents - see Coverage">{agg["docs_incomplete"]} not fully validated</span>')
     if agg.get("checks_advisory"):
         chips.append(f'<span class="hero-chip">{agg["checks_advisory"]} advisory</span>')
+    if agg.get("checks_suppressed"):
+        # Accepted known-issues: disclosed, never hidden. Tooltip names what was suppressed.
+        acc = ", ".join(f'{s["label"]} (x{s["docs"]})' for s in agg.get("suppressed", [])[:8])
+        chips.append(f'<span class="hero-chip" data-tip="Accepted advisory issues (config '
+                     f'suppressions) - excluded from hygiene &amp; health, never from the gate: '
+                     f'{_esc(acc)}">{agg["checks_suppressed"]} suppressed</span>')
     chips_html = f'<div class="hero-chips">{"".join(chips)}</div>' if chips else ""
     t = max(agg["total"], 1)
     seg = ""
@@ -1064,8 +1100,14 @@ def _action_plan_section(agg: dict) -> str:
     # cards below; a pointer here makes the remainder explicit rather than silently dropped.
     cards = ""
     for i, g in enumerate(ranked[:_ACTION_CAP], 1):
+        ranked_checks = sorted(g["checks"], key=lambda o: -o["docs"])
         evidence = ", ".join(f'{_esc(o["label"])} {round(100 * o["docs"] / g["total"])}%'
-                             for o in sorted(g["checks"], key=lambda o: -o["docs"])[:3])
+                             for o in ranked_checks[:3])
+        # A concrete measured example from the worst check (e.g. "4200 (~3000 expected)"),
+        # so Observed carries a specific number - the "why" - not just which check failed.
+        example = next((o.get("detail") for o in ranked_checks if o.get("detail")), "")
+        if example:
+            evidence += f' <span class="check-detail">e.g. {_esc(example)}</span>'
         # A concise, category-specific INSTRUCTION here - the diagnosis now lives in the
         # Observed/Likely/Possible tiers, so the fix line is just "what to do" and never
         # repeats a cause. The verbose per-check hint stays in the detailed cards below.
@@ -1687,6 +1729,16 @@ def _glossary_section(agg: dict | None = None, corpus: dict | None = None, mode:
             '</dl>'
             '<dd style="margin-top:8px">A document is PASSED only when every non-skipped step passes. Critical-step failures halt remaining steps.</dd>')
 
+    # Suppressed: only when this run actually accepted advisory issues.
+    if agg is not None and agg.get("checks_suppressed"):
+        section("Suppressed (accepted)", dl([
+            ("Suppressed", "An advisory check the project accepted as expected for this corpus "
+             "(config `suppressions` / `--suppress`) - e.g. near-duplicate boilerplate in policy "
+             "documents. Excluded from the hygiene and Platform Health scores but still counted "
+             "and shown as 'N suppressed'. Only advisory checks can be suppressed; a hard gating "
+             "failure is never suppressed, so an accepted known-issue can never mask a real one."),
+        ]))
+
     # Readiness Verdict: only when a corpus supplied one.
     if corpus is not None:
         section("Readiness Verdict",
@@ -2057,6 +2109,7 @@ def generate_batch_dashboard(
     wall_seconds: float | None = None,
     concurrency: int | None = None,
     evidence: dict | None = None,
+    suppress: list[str] | None = None,
 ) -> str:
     """Generate a self-contained, interactive HTML dashboard. Returns abs path.
 
@@ -2076,7 +2129,8 @@ def generate_batch_dashboard(
     _ICON = {s.name: s.icon for s in registered_steps()}
 
     cases = [_result_payload(item.get("name", item["result"].file_name), item["result"]) for item in results]
-    agg = _aggregate(cases, hide_skipped=hide_skipped)  # computed on FULL data, before any trim
+    # computed on FULL data, before any trim; suppressions accepted from config
+    agg = _aggregate(cases, hide_skipped=hide_skipped, suppress=set(suppress or []))
     # Wall clock vs compute time: agg["duration"] is the SUM of per-document durations, so
     # under --parallel it far exceeds the elapsed time (a 502-doc run summed 1319s but
     # finished in 329s). Carry the real wall clock and concurrency so the summary can show

{compass_kb_validation-3.1.0 → compass_kb_validation-3.2.0}/compass_kb_validation.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: compass-kb-validation
-Version: 3.1.0
+Version: 3.2.0
 Summary: Knowledge-base readiness validation for RAG & AI agents — config-driven, deterministic pipeline integrity checks.
 Author: COMPASS
 License: MIT

{compass_kb_validation-3.1.0 → compass_kb_validation-3.2.0}/discover_and_validate.py

@@ -90,6 +90,11 @@ def parse_args(argv=None) -> argparse.Namespace:
                         "--disable-check chunk_count near_duplicate_embeddings.")
     p.add_argument("--severity", nargs="*", default=[], metavar="NAME=TIER",
                    help="Override a check's severity, e.g. --severity duplicate_detection=hard.")
+    p.add_argument("--suppress", nargs="*", default=[], metavar="CHECK",
+                   help="Accept an ADVISORY check as expected for this corpus, e.g. --suppress "
+                        "'Near-duplicate embeddings'. Excluded from the hygiene / Platform Health "
+                        "scores but still counted and shown ('N suppressed'). Adds to config "
+                        "suppressions. Only advisory checks - a hard gating failure is never suppressed.")
     # --- baselines + drift (continuous gate) ---
     p.add_argument("--save-baseline", nargs="?", const="", default=None, metavar="PATH",
                    help="Snapshot this run as a baseline (default: baselines/<project>.json).")
@@ -469,6 +474,11 @@ def main(argv=None) -> int:
         print(f"  Config error: {exc}")
         return EXIT_CONFIG
 
+    # Accepted advisory issues = config suppressions + any ad-hoc --suppress (deduped, order-kept).
+    # Threaded into BOTH the hygiene gate and the report so the score and the HTML agree.
+    suppress = list(dict.fromkeys(
+        list(getattr(settings, "suppressions", []) or []) + list(args.suppress or [])))
+
     # Discovery validates the END STATE of an already-ingested corpus. Ingestion-stage
     # steps (upload, storage) already happened, so they must never HALT the run before
     # the chunk/retrieval checks that are the point. Strip them from critical_steps.
@@ -596,7 +606,7 @@ def main(argv=None) -> int:
             print(f"    evidence skipped: {exc}")
 
     if args.output_format in ("html", "all"):
-        print(f"\n  HTML report: {generate_batch_dashboard(results, base + '.html', project=args.project, environment=args.env, run_id=run_id, trends=trend_rows, corpus=corpus, mode='Discovery', profile=profile_data, wall_seconds=wall_seconds, concurrency=concurrency, evidence=evidence_by_doc)}")
+        print(f"\n  HTML report: {generate_batch_dashboard(results, base + '.html', project=args.project, environment=args.env, run_id=run_id, trends=trend_rows, corpus=corpus, mode='Discovery', profile=profile_data, wall_seconds=wall_seconds, concurrency=concurrency, evidence=evidence_by_doc, suppress=suppress)}")
     if args.output_format in ("json", "all"):
         print(f"  JSON report: {export_json(results, base + '.json')}")
     if args.output_format in ("junit", "all"):
@@ -617,7 +627,7 @@ def main(argv=None) -> int:
     # Best-effort context - a failure to score it must not crash the gate decision.
     from compass.report import hygiene_from_results
     try:
-        hygiene = hygiene_from_results(results)
+        hygiene = hygiene_from_results(results, suppress=suppress)
     except Exception:  # noqa: BLE001 - hygiene is advisory; never break the run over it
         hygiene = None
     if hygiene:

{compass_kb_validation-3.1.0 → compass_kb_validation-3.2.0}/run_validation.py

@@ -228,7 +228,7 @@ def main(argv=None) -> int:
     report_base = f"reports/{args.project}-report"
     fmt = args.output_format
     if fmt in ("html", "all"):
-        print(f"  HTML report: {generate_batch_dashboard(results, report_base + '.html', project=args.project, environment=settings.environment, run_id=run_id, trends=trend_rows, corpus=corpus, mode='Live Ingestion')}")
+        print(f"  HTML report: {generate_batch_dashboard(results, report_base + '.html', project=args.project, environment=settings.environment, run_id=run_id, trends=trend_rows, corpus=corpus, mode='Live Ingestion', suppress=list(getattr(settings, 'suppressions', []) or []))}")
     if fmt in ("json", "all"):
         print(f"  JSON report: {export_json(results, report_base + '.json')}")
     if fmt in ("junit", "all"):

{compass_kb_validation-3.1.0 → compass_kb_validation-3.2.0}/tests/test_config.py

@@ -110,6 +110,22 @@ class TestLoadSettings:
         assert s.redis.search_index == "demo_index"  # overridden
         assert s.redis.expected_embedding_dim == 1536  # inherited from base
 
+    def test_suppressions_load_from_project_yaml(self, tmp_path):
+        cfg_dir = tmp_path / "config"
+        (cfg_dir / "projects").mkdir(parents=True)
+        (cfg_dir / "base.yaml").write_text("default_env: dev\n")
+        (cfg_dir / "projects" / "demo.yaml").write_text(
+            textwrap.dedent(
+                """
+                suppressions:
+                  - "Near-duplicate embeddings"
+                  - "Duplicate text chunks"
+                """
+            )
+        )
+        s = load_settings(project="demo", env="dev", config_dir=cfg_dir)
+        assert s.suppressions == ["Near-duplicate embeddings", "Duplicate text chunks"]
+
     def test_environment_selection(self, tmp_path):
         cfg_dir = tmp_path / "config"
         cfg_dir.mkdir()

{compass_kb_validation-3.1.0 → compass_kb_validation-3.2.0}/tests/test_reporting.py

@@ -632,6 +632,49 @@ def test_engineering_wins_are_check_backed_and_specific():
                    for w in report_mod._engineering_wins({**agg, "failed": 3}))
 
 
+def test_advisory_suppression_excludes_from_scoring_but_discloses():
+    # An accepted advisory (config `suppressions`) drops out of hygiene / offenders but is
+    # still counted and listed - never silently hidden.
+    r = PipelineResult(document_id="d", file_name="d.pdf")
+    r.add_step(StepResult("Redis Chunks", StepStatus.PASSED, "ok", checks=[
+        {"label": "Near-duplicate embeddings", "passed": False, "severity": "warn"},
+        {"label": "Chunk boundaries clean", "passed": True, "severity": "warn"},
+    ]))
+    cases = [report_mod._result_payload("d.pdf", r)]
+    base = report_mod._aggregate(cases)
+    supp = report_mod._aggregate(cases, suppress={"Near-duplicate embeddings"})
+    assert any(o["label"] == "Near-duplicate embeddings" for o in base["offenders"])
+    assert not any(o["label"] == "Near-duplicate embeddings" for o in supp["offenders"])
+    assert supp["checks_suppressed"] == 1
+    assert supp["suppressed"] == [{"label": "Near-duplicate embeddings", "docs": 1}]
+    # suppressing the failing advisory lifts hygiene (only the clean advisory remains scored)
+    assert supp["hygiene"]["score"] > base["hygiene"]["score"]
+
+
+def test_hard_check_is_never_suppressed():
+    # A gating failure must count even if its label is in the suppress list - a known-issue
+    # can never mask a real failure (the integrity mandate).
+    r = PipelineResult(document_id="d", file_name="d.pdf")
+    r.add_step(StepResult("Redis Chunks", StepStatus.FAILED, "bad",
+        checks=[{"label": "Chunks exist", "passed": False, "severity": "hard"}]))
+    agg = report_mod._aggregate([report_mod._result_payload("d.pdf", r)], suppress={"Chunks exist"})
+    assert agg["checks_suppressed"] == 0
+    assert any(o["label"] == "Chunks exist" for o in agg["offenders"])
+    assert agg["checks_hard"] == 1
+
+
+def test_action_plan_surfaces_a_measured_detail():
+    # Specific RCA: the Action Plan's Observed line carries a concrete measured example from
+    # the check's own detail, not just which check failed.
+    r = PipelineResult(document_id="d", file_name="d.pdf")
+    r.add_step(StepResult("Redis Chunks", StepStatus.PASSED, "ok", checks=[
+        {"label": "Chunk count in range", "passed": False, "severity": "warn",
+         "detail": "4200 (~3000 expected)"}]))
+    agg = report_mod._aggregate([report_mod._result_payload("d.pdf", r)])
+    html = report_mod._action_plan_section(agg)
+    assert "4200 (~3000 expected)" in html          # the measured number, surfaced
+
+
 def test_glossary_is_run_driven_not_a_fixed_catalogue():
     # A run with retrieval + chunk findings: the glossary explains exactly what appeared -
     # its categories, Platform Health, the Action Plan tiers, and only the mode that ran -