compass-kb-validation 2.1.0__tar.gz → 2.3.0__tar.gz

{compass_kb_validation-2.1.0 → compass_kb_validation-2.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: compass-kb-validation
-Version: 2.1.0
+Version: 2.3.0
 Summary: Knowledge-base readiness validation for RAG & AI agents — config-driven, deterministic pipeline integrity checks.
 Author: COMPASS
 License: MIT

{compass_kb_validation-2.1.0 → compass_kb_validation-2.3.0}/compass_kb_validation.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: compass-kb-validation
-Version: 2.1.0
+Version: 2.3.0
 Summary: Knowledge-base readiness validation for RAG & AI agents — config-driven, deterministic pipeline integrity checks.
 Author: COMPASS
 License: MIT

{compass_kb_validation-2.1.0 → compass_kb_validation-2.3.0}/discover_and_validate.py

@@ -63,6 +63,16 @@ def parse_args(argv=None) -> argparse.Namespace:
                         "Replaces the default 'fail on any failure' rule.")
     p.add_argument("--require-ready", action="store_true",
                    help="Also fail unless the corpus readiness verdict is READY.")
+    p.add_argument("--quality-threshold", type=float, metavar="PCT",
+                   help="Also fail unless the KB Hygiene score is >= PCT (0-100). Hygiene is "
+                        "the share of advisory data-quality checks that passed - data-plane "
+                        "cleanliness, separate from the READY integrity gate.")
+    p.add_argument("--suggest-config", nargs="?", const="-", metavar="PATH",
+                   help="After the run, emit config refinements from what it observed "
+                        "(document count, chunk-size distribution, checks that never applied). "
+                        "Prints to stdout, or writes to PATH. Review before merging.")
+    p.add_argument("--no-tips", action="store_true",
+                   help="Suppress the end-of-run tips about relevant unused flags.")
     # --- config overrides (run scenarios without editing YAML; CLI wins over config) ---
     p.add_argument("--readiness-threshold", type=float, metavar="FRAC",
                    help="Override corpus readiness cutoff (0-1). READY when pass-rate >= this.")
@@ -163,15 +173,18 @@ def _normalize_skips(tokens: list[str]) -> list[str]:
 
 
 def gate_failed(passed: int, total: int, verdict: str, fail_under: float | None,
-                require_ready: bool) -> tuple[bool, str]:
+                require_ready: bool, quality_threshold: float | None = None,
+                hygiene_score: float | None = None) -> tuple[bool, str]:
     """Decide whether the readiness gate fails, and why.
 
     Returns (failed, reason). The gate is the product's contract with a CI
     pipeline: a non-zero exit blocks promotion of a knowledge base that is not
     retrieval-ready. Strictness is caller-controlled:
-      * default            -- fail if any document failed validation
-      * --fail-under PCT    -- fail if pass-rate < PCT (tolerates a few failures)
-      * --require-ready     -- additionally fail if the corpus verdict != READY
+      * default              -- fail if any document failed validation
+      * --fail-under PCT      -- fail if pass-rate < PCT (tolerates a few failures)
+      * --require-ready       -- additionally fail if the corpus verdict != READY
+      * --quality-threshold N -- additionally fail if KB Hygiene < N (data-plane
+                                 cleanliness, separate from the integrity checks above)
     """
     rate = (passed / total * 100.0) if total else 100.0
     if fail_under is not None:
@@ -181,6 +194,8 @@ def gate_failed(passed: int, total: int, verdict: str, fail_under: float | None,
         return True, f"{total - passed} document(s) failed validation"
     if require_ready and verdict != "READY":
         return True, f"corpus readiness verdict is {verdict} (READY required)"
+    if quality_threshold is not None and hygiene_score is not None and hygiene_score < quality_threshold:
+        return True, f"KB hygiene {hygiene_score:g}% < required {quality_threshold:g}%"
     return False, "all gate conditions met"
 
 
@@ -227,6 +242,99 @@ def _validate_one_document(pipeline: IngestionPipeline, doc: dict) -> dict:
     return {"name": name, "result": result}
 
 
+_AUTH_HINTS = ("403", "forbidden", "expired", "credential", "token", "signaturedoesnotmatch",
+               "invalidclienttokenid", "accessdenied", "unauthor")
+
+
+def _print_tips(args, hygiene: dict | None, saved_baseline: bool) -> None:
+    """End-of-run hints about the most relevant unused flags.
+
+    Discoverability, not noise: at most three, only when the flag was NOT already used and
+    the run's own outcome makes it relevant. Suppress with --no-tips. This is why a partner
+    can run for weeks without finding --save-baseline / --unreachable-as / --suggest-config.
+    """
+    tips: list[str] = []
+    if not saved_baseline:
+        tips.append("Save this run as a baseline to catch regressions next time: "
+                    "--save-baseline baselines/{}.json".format(args.project or "kb"))
+    if hygiene and hygiene.get("score", 100) < 90 and args.quality_threshold is None:
+        tips.append("Gate CI on data-plane hygiene (not just integrity): "
+                    "--quality-threshold {}".format(int(hygiene["score"])))
+    if not getattr(args, "include_retrieval", False):
+        tips.append("Add retrieval-quality checks (can a stored vector be found via KNN?): "
+                    "--include-retrieval")
+    if not args.suggest_config:
+        tips.append("Refine your config from what this run observed: --suggest-config")
+    if not tips:
+        return
+    print("\n  Tips (--no-tips to silence):")
+    for t in tips[:3]:
+        print(f"    * {t}")
+
+
+def _resolve_expected_documents(raw, project: str):
+    """Resolve the configured expected_documents, supporting ``auto``.
+
+    An int is used as-is. ``"auto"`` uses the previous run's document count for this
+    project as the baseline, so completeness alerts on drift (documents appearing or
+    disappearing) without a hand-maintained number. The FIRST run has no prior count, so
+    it returns None - the baseline is simply established this run, no false "missing".
+    """
+    if isinstance(raw, bool):   # guard: bool is an int subclass
+        return None
+    if isinstance(raw, int):
+        return raw
+    if isinstance(raw, str) and raw.strip().isdigit():
+        return int(raw.strip())
+    if isinstance(raw, str) and raw.strip().lower() == "auto":
+        try:
+            from ingestion_validation.trend_tracker import TrendTracker
+            recent = TrendTracker().summary(last_n=1, project=project)
+            if recent and recent[0].get("total_cases"):
+                return int(recent[0]["total_cases"])
+        except Exception:  # noqa: BLE001 - no history / unreadable db: establish baseline this run
+            return None
+    return None
+
+
+def _warn_on_unreachable_backends(results: list[dict]) -> None:
+    """After a run, warn loudly if any step was SKIPPED because its backend was unreachable.
+
+    An unreachable backend does not gate the run (infrastructure is not a data defect), so
+    without this the affected documents are validated on a partial pipeline and the run
+    still reports green - the exact silent degradation that let an expired STS token skip
+    315 documents unnoticed. Auth-flavoured errors get a credential-refresh hint.
+    """
+    by_step: dict[str, dict] = {}
+    for item in results:
+        for st in getattr(item.get("result"), "steps", []) or []:
+            d = getattr(st, "details", {}) or {}
+            if d.get("unreachable"):
+                rec = by_step.setdefault(st.step_name, {"count": 0, "auth": False, "error": ""})
+                rec["count"] += 1
+                err = str(d.get("last_error", "") or "")
+                if err and not rec["error"]:
+                    rec["error"] = err
+                if any(h in err.lower().replace(" ", "") for h in _AUTH_HINTS):
+                    rec["auth"] = True
+    if not by_step:
+        return
+    print("\n  " + "!" * 68)
+    for step, rec in by_step.items():
+        print(f"  WARNING: {step} was unreachable for {rec['count']} document(s) - they were "
+              f"validated WITHOUT it.")
+        if rec["auth"]:
+            print("           The endpoint rejected the request (auth). Credentials may have "
+                  "expired mid-run;")
+            print("           re-run with fresh credentials (e.g. new STS tokens) to validate "
+                  "these documents.")
+        elif rec["error"]:
+            print(f"           Last error: {rec['error'][:140]}")
+        print("           These steps did not gate the run (set pipeline.unreachable_as: "
+              "failed to change that).")
+    print("  " + "!" * 68)
+
+
 def validate_documents(settings, documents: list[dict], skip_steps: list[str],
                        parallel: bool = False, workers: int = 4) -> list[dict]:
     total = len(documents)
@@ -390,21 +498,28 @@ def main(argv=None) -> int:
         print("  Note: discovery source is not object storage; skipping the S3 Storage "
               "step (use --keep s3_storage to force it).")
 
+    import time as _t
+    _run_started = _t.perf_counter()
     results = validate_documents(settings, documents, skip_steps=skip_steps,
                                  parallel=args.parallel, workers=args.workers)
+    wall_seconds = _t.perf_counter() - _run_started
+    concurrency = args.workers if args.parallel and len(documents) > 1 else 1
 
     # Corpus-level knowledge-base readiness (completeness, dedup, coverage).
     from ingestion_validation.corpus import analyze_corpus
 
+    expected_documents = _resolve_expected_documents(
+        settings.discovery.extra.get("expected_documents"), args.project)
     corpus = analyze_corpus(
         documents, results,
-        expected_documents=settings.discovery.extra.get("expected_documents"),
+        expected_documents=expected_documents,
         expected_folders=settings.discovery.extra.get("expected_folders"),
         expected_document_names=settings.discovery.extra.get("expected_document_names"),
         readiness_threshold=settings.pipeline.readiness_threshold,
     )
     print(f"  KB readiness: {corpus['verdict']} ({corpus['passed']}/{corpus['total']} passed, "
           f"{corpus['total'] - corpus['passed']} failed)")
+    _warn_on_unreachable_backends(results)
 
     # Maintain historical trends (stdlib sqlite; on by default) and embed them in the report.
     trend_rows = _record_trends(results, args.project, args.env, run_id, "Discovery", disabled=args.no_trends)
@@ -413,14 +528,42 @@ def main(argv=None) -> int:
     failed = len(results) - passed
     base = f"reports/{args.project}-discovery-report"
     if args.output_format in ("html", "all"):
-        print(f"\n  HTML report: {generate_batch_dashboard(results, base + '.html', project=args.project, environment=args.env, run_id=run_id, trends=trend_rows, corpus=corpus, mode='Discovery', profile=profile_data)}")
+        print(f"\n  HTML report: {generate_batch_dashboard(results, base + '.html', project=args.project, environment=args.env, run_id=run_id, trends=trend_rows, corpus=corpus, mode='Discovery', profile=profile_data, wall_seconds=wall_seconds, concurrency=concurrency)}")
     if args.output_format in ("json", "all"):
         print(f"  JSON report: {export_json(results, base + '.json')}")
     if args.output_format in ("junit", "all"):
         print(f"  JUnit report: {export_junit(results, base + '.xml')}")
 
+    if args.suggest_config:
+        from ingestion_validation.profiler import suggest_config_from_run
+        snippet = suggest_config_from_run(results, documents)
+        if args.suggest_config == "-":
+            print("\n" + snippet)
+        else:
+            from pathlib import Path as _P
+            _P(args.suggest_config).write_text(snippet, encoding="utf-8")
+            print(f"  Config suggestions written: {args.suggest_config} (review before merging)")
+
+    # KB Hygiene: data-plane cleanliness (advisory checks), separate from the integrity
+    # gate. Printed for every run; gates the build only when --quality-threshold is set.
+    # Best-effort context - a failure to score it must not crash the gate decision.
+    from ingestion_validation.report import hygiene_from_results
+    try:
+        hygiene = hygiene_from_results(results)
+    except Exception:  # noqa: BLE001 - hygiene is advisory; never break the run over it
+        hygiene = None
+    if hygiene:
+        worst = hygiene.get("worst")
+        worst_note = (f" | worst: {worst['name']} {worst['rate']:g}%"
+                      if worst and worst["rate"] < 100 else "")
+        print(f"  KB hygiene: {hygiene['score']:g}% {hygiene['tier']} "
+              f"({hygiene['advisory_passed']}/{hygiene['advisory_total']} advisory checks){worst_note}")
+    hygiene_score = hygiene["score"] if hygiene else None
+
     failed_gate, reason = gate_failed(passed, len(results), corpus["verdict"],
-                                      args.fail_under, args.require_ready)
+                                      args.fail_under, args.require_ready,
+                                      quality_threshold=args.quality_threshold,
+                                      hygiene_score=hygiene_score)
 
     # Baselines + drift: snapshot a known-good run and/or diff against one.
     regression = False
@@ -438,6 +581,8 @@ def main(argv=None) -> int:
         reason = f"{len(documents)} documents < s3.min_documents ({min_docs})"
         failed_gate = True
     code = EXIT_GATE_FAILED if (failed_gate or (regression and args.fail_on_regression)) else EXIT_OK
+    if not args.no_tips:
+        _print_tips(args, hygiene, saved_baseline=args.save_baseline is not None or bool(args.against))
     print("=" * 60)
     print(f"  Results: {passed} passed, {failed} failed ({len(results)} total)")
     print(f"  Gate: {'FAILED' if failed_gate else 'PASSED'} - {reason}")

{compass_kb_validation-2.1.0 → compass_kb_validation-2.3.0}/ingestion_validation/__init__.py

@@ -5,4 +5,4 @@ end-to-end: API upload -> object storage -> knowledge graph -> vector store ->
 retrieval quality, plus cross-store consistency.
 """
 
-__version__ = "2.1.0"
+__version__ = "2.3.0"

{compass_kb_validation-2.1.0 → compass_kb_validation-2.3.0}/ingestion_validation/config_lint.py

@@ -125,6 +125,39 @@ def _lint_consistency(settings: Any) -> list[Finding]:
     return findings
 
 
+def _lint_skip_steps(settings: Any) -> list[Finding]:
+    """Flag skip_steps entries that match no registered step.
+
+    A skip_steps entry naming a step that does not exist is silently ignored at runtime -
+    so a typo ("Redis Chunk"), or a step removed in an upgrade (the v2.0 UI Visibility
+    step), quietly does nothing and the run is not scoped the way the config says. The
+    runtime stays quiet (an unknown skip is harmless there and a per-run warning is just
+    noise); lint is where config drift belongs, so it is surfaced here once, with the fix.
+    """
+    skip = list(getattr(settings, "skip_steps", []) or [])
+    if not skip:
+        return []
+    try:
+        from ingestion_validation.validators.registry import registered_steps
+        known = list(registered_steps())
+    except Exception:  # noqa: BLE001 - registry unavailable; skip this check rather than crash lint
+        return []
+
+    def _norm(s: str) -> str:
+        return "".join(str(s).lower().split()).replace("_", "").replace("-", "")
+
+    valid = {_norm(s.name) for s in known} | {_norm(s.key) for s in known}
+    findings: list[Finding] = []
+    for entry in skip:
+        if _norm(entry) not in valid:
+            names = ", ".join(sorted(s.name for s in known))
+            removed = " (the UI Visibility step was removed in v2.0)" if _norm(entry) == _norm("UI Visibility") else ""
+            findings.append(("info",
+                f"skip_steps entry '{entry}' matches no registered step{removed}; it is "
+                f"silently ignored. Remove it, or use one of: {names}."))
+    return findings
+
+
 def lint_settings(settings: Any) -> list[Finding]:
     """Run every static config check against a loaded Settings. Never connects anywhere."""
     findings: list[Finding] = []
@@ -137,6 +170,7 @@ def lint_settings(settings: Any) -> list[Finding]:
         findings += _lint_helper_queries(label, cfg)
     findings += _lint_redis(settings)
     findings += _lint_consistency(settings)
+    findings += _lint_skip_steps(settings)
     return findings
 
 

{compass_kb_validation-2.1.0 → compass_kb_validation-2.3.0}/ingestion_validation/corpus_health.py

@@ -129,8 +129,11 @@ def freshness_report(settings: Settings, *, sample: int = 500, stale_after_days:
     _INGEST_FIELDS = ("ingested_at", "indexed_at", "embedded_at", "updated_at", "created_at")
     doc_key: dict[str, str] = {}
     doc_ingest: dict[str, datetime] = {}
+    seen_docs: set[str] = set()   # every distinct document in the sample, source key or not
     for c in vs.sample_chunks(sample):
         doc = str(c.get(id_field) or c.get("document_id") or c.get("key") or "")
+        if doc:
+            seen_docs.add(doc)
         s3_key = c.get("s3_key") or ""
         if doc and s3_key and doc not in doc_key:
             doc_key[doc] = s3_key
@@ -182,6 +185,11 @@ def freshness_report(settings: Settings, *, sample: int = 500, stale_after_days:
     stale.sort(key=lambda x: -x["age_days"])
     drift.sort(key=lambda x: -x["behind_days"])
     checked = len(doc_key)
+    # Documents sampled that carry NO source key were never checked against storage - the
+    # gap between "documents in the index" and "documents freshness could verify". Left
+    # implicit, it reads as "freshness found only 176 of 502 documents"; it usually means
+    # those chunks lack an s3_key (a prefix/field mismatch) or point at a different store.
+    without_key = len(seen_docs) - checked
     if not stale and not orphans and not drift:
         verdict = "FRESH - all sampled sources within the window and in sync"
     else:
@@ -193,14 +201,21 @@ def freshness_report(settings: Settings, *, sample: int = 500, stale_after_days:
         if orphans:
             parts.append(f"{len(orphans)} orphan(s) (source object gone)")
         verdict = "STALE - " + ", ".join(parts)
-    return {
-        "documents": checked, "stale_after_days": stale_after_days,
+    result = {
+        "documents": checked, "sampled_documents": len(seen_docs),
+        "without_source_key": without_key, "stale_after_days": stale_after_days,
         "stale_count": len(stale), "orphan_count": len(orphans), "drift_count": len(drift),
         "max_age_days": max(ages) if ages else 0, "median_age_days": sorted(ages)[len(ages) // 2] if ages else 0,
         "stale": stale[: min(20, len(stale))], "orphans": orphans[: min(20, len(orphans))],
         "drift": drift[: min(20, len(drift))],
         "verdict": verdict,
     }
+    if without_key:
+        result["note"] = (
+            f"{without_key} of {len(seen_docs)} sampled document(s) carry no source key, so "
+            f"their source could not be checked - likely a missing s3_key field (prefix/field "
+            f"mismatch) or sources in a different store, not necessarily orphans.")
+    return result
 
 
 def _parse_timestamp(value: Any) -> Optional[datetime]:
@@ -222,13 +237,17 @@ def _parse_timestamp(value: Any) -> Optional[datetime]:
 def format_freshness(payload: dict) -> str:
     if payload.get("error"):
         return f"Freshness: {payload['error']}"
+    sampled = payload.get("sampled_documents")
+    checked_note = f" of {sampled} sampled" if sampled and sampled != payload["documents"] else ""
     lines = [
-        f"Freshness: {payload['documents']} document(s) checked (stale after {payload['stale_after_days']}d)",
+        f"Freshness: {payload['documents']} document(s) checked{checked_note} (stale after {payload['stale_after_days']}d)",
         f"  stale: {payload['stale_count']}   orphans (source missing): {payload['orphan_count']}   "
         f"drift (source newer than KB): {payload.get('drift_count', 0)}   "
         f"max age: {payload['max_age_days']}d   median age: {payload['median_age_days']}d",
         f"  VERDICT: {payload['verdict']}",
     ]
+    if payload.get("note"):
+        lines.append(f"  NOTE: {payload['note']}")
     if payload.get("drift"):
         lines.append("  content drift (source modified after the KB ingested it):")
         for d in payload["drift"]:

{compass_kb_validation-2.1.0 → compass_kb_validation-2.3.0}/ingestion_validation/profiler.py

@@ -537,7 +537,79 @@ def apply_profile_to_settings(settings: Any, profiles: dict) -> list[str]:
     return applied
 
 
+def suggest_config_from_run(results: list, documents: list) -> str:
+    """Suggest config refinements from what a validation RUN actually observed.
+
+    Complements ``emit_project_config`` (pre-run backend introspection) with the things
+    only a run reveals: the real document count, the observed chunk-size distribution, and
+    the checks that were NEUTRAL on every document - checks that never apply to this KB and
+    are pure noise (e.g. section_metadata when nothing carries sections). Returns a review-
+    and-merge snippet, never a file to apply blind - a wrong config key would be exactly the
+    silent-failure trap COMPASS exists to catch, so these are suggestions with the evidence.
+    """
+    import statistics
+
+    n_docs = len(documents)
+    avg_chars: list[int] = []
+    # label -> [evaluated_count, neutral_count]; a check neutral everywhere never applies.
+    support: dict[str, list[int]] = {}
+    for item in results:
+        for st in getattr(item.get("result"), "steps", []) or []:
+            details = getattr(st, "details", {}) or {}
+            cv = details.get("chunk_view")
+            if isinstance(cv, dict) and cv.get("avg_chars"):
+                avg_chars.append(int(cv["avg_chars"]))
+            status = getattr(getattr(st, "status", None), "value", "")
+            if status == "SKIPPED":
+                continue   # a step that did not run says nothing about applicability
+            for ch in getattr(st, "checks", []) or []:
+                lbl = ch.get("label", "")
+                if not lbl:
+                    continue
+                rec = support.setdefault(lbl, [0, 0])
+                if ch.get("supported", True) is False:
+                    rec[1] += 1
+                else:
+                    rec[0] += 1
+
+    never_applies = sorted(lbl for lbl, (ev, neu) in support.items() if ev == 0 and neu > 0)
+
+    lines = [
+        "# --- compass discover --suggest-config -------------------------------------",
+        "# Refinements from THIS run's observations. Review each, then merge into your",
+        "# project YAML - nothing here is applied automatically.",
+        "",
+        "# Documents discovered this run. Pin it to alert on drift (or use 'auto' to",
+        "# baseline against the previous run automatically):",
+        "discovery:",
+        "  extra:",
+        f"    expected_documents: {n_docs}",
+    ]
+    if avg_chars:
+        median_size = int(statistics.median(avg_chars))
+        lines += [
+            "",
+            f"# Observed chunk size: median {median_size} chars across {len(avg_chars)} document(s)",
+            f"# (range {min(avg_chars)}-{max(avg_chars)}). Set expected bounds if you gate on size:",
+            "# redis:",
+            f"#   chunk_size: {median_size}",
+        ]
+    if never_applies:
+        lines += [
+            "",
+            "# These checks were NOT APPLICABLE to any document this run (neutral every time) -",
+            "# the backend/data cannot answer them. Disabling removes them from the tally and",
+            "# the report noise. Verify each is genuinely irrelevant before disabling:",
+        ]
+        for lbl in never_applies:
+            lines.append(f"#   - {lbl}")
+    else:
+        lines += ["", "# Every check applied to at least one document - nothing to disable."]
+    return "\n".join(lines) + "\n"
+
+
 __all__ = [
     "profile_vectorstore", "profile_storage", "profile_graph", "profile_kb",
-    "apply_profile_to_settings", "suggest_config",
+    "apply_profile_to_settings", "suggest_config", "emit_project_config",
+    "suggest_config_from_run",
 ]