compass-kb-validation 3.7.0__tar.gz → 3.8.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (143) hide show
  1. {compass_kb_validation-3.7.0/compass_kb_validation.egg-info → compass_kb_validation-3.8.0}/PKG-INFO +21 -1
  2. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/README.md +20 -0
  3. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/__init__.py +1 -1
  4. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/embedding_drift.py +11 -6
  5. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/providers/vectorstore.py +42 -0
  6. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/validators/redis_chunks.py +12 -6
  7. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0/compass_kb_validation.egg-info}/PKG-INFO +21 -1
  8. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass_kb_validation.egg-info/SOURCES.txt +1 -0
  9. compass_kb_validation-3.8.0/tests/test_cli_commands.py +158 -0
  10. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_health.py +61 -0
  11. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_vector_fastpath.py +35 -0
  12. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/LICENSE +0 -0
  13. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/__main__.py +0 -0
  14. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/alias.py +0 -0
  15. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/ask.py +0 -0
  16. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/baseline.py +0 -0
  17. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/benchmark.py +0 -0
  18. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/chunk_coherence.py +0 -0
  19. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/citation.py +0 -0
  20. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/cli.py +0 -0
  21. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/config/__init__.py +0 -0
  22. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/config_lint.py +0 -0
  23. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/content_inspect.py +0 -0
  24. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/corpus.py +0 -0
  25. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/corpus_health.py +0 -0
  26. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/diagnostics.py +0 -0
  27. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/diff.py +0 -0
  28. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/eval_enrich.py +0 -0
  29. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/eval_interop.py +0 -0
  30. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/evidence.py +0 -0
  31. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/grounding.py +0 -0
  32. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/health.py +0 -0
  33. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/incremental.py +0 -0
  34. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/init_wizard.py +0 -0
  35. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/live_dashboard.py +0 -0
  36. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/mcp_server.py +0 -0
  37. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/models/__init__.py +0 -0
  38. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/notifications.py +0 -0
  39. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/pipeline/__init__.py +0 -0
  40. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/profiler.py +0 -0
  41. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/provenance.py +0 -0
  42. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/providers/__init__.py +0 -0
  43. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/providers/embedding.py +0 -0
  44. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/providers/graph.py +0 -0
  45. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/providers/source.py +0 -0
  46. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/providers/storage.py +0 -0
  47. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/quality_gate.py +0 -0
  48. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/remediation.py +0 -0
  49. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/report/__init__.py +0 -0
  50. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/report/_report_css.py +0 -0
  51. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/report/_report_js.py +0 -0
  52. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/report/analysis.py +0 -0
  53. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/report/audit.py +0 -0
  54. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/report/export.py +0 -0
  55. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/report/grounding_report.py +0 -0
  56. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/report/shell.py +0 -0
  57. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/retrieval_analysis.py +0 -0
  58. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/text_probes.py +0 -0
  59. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/trend_tracker.py +0 -0
  60. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/utils/__init__.py +0 -0
  61. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/validators/__init__.py +0 -0
  62. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/validators/api_upload.py +0 -0
  63. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/validators/base.py +0 -0
  64. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/validators/base_sparql.py +0 -0
  65. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/validators/cross_stage.py +0 -0
  66. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/validators/graphdb_metadata.py +0 -0
  67. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/validators/neptune_metadata.py +0 -0
  68. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/validators/registry.py +0 -0
  69. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/validators/retrieval_quality.py +0 -0
  70. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/validators/s3_storage.py +0 -0
  71. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass_kb_validation.egg-info/dependency_links.txt +0 -0
  72. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass_kb_validation.egg-info/entry_points.txt +0 -0
  73. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass_kb_validation.egg-info/requires.txt +0 -0
  74. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass_kb_validation.egg-info/top_level.txt +0 -0
  75. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/discover_and_validate.py +0 -0
  76. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/ingestion_validation/__init__.py +0 -0
  77. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/profile_kb.py +0 -0
  78. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/pyproject.toml +0 -0
  79. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/run_validation.py +0 -0
  80. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/setup.cfg +0 -0
  81. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_alias.py +0 -0
  82. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_alias_disambiguation.py +0 -0
  83. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_alt_adapters.py +0 -0
  84. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_analysis_report.py +0 -0
  85. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_api_upload.py +0 -0
  86. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_ask.py +0 -0
  87. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_baseline.py +0 -0
  88. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_benchmark.py +0 -0
  89. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_calibrate_truth.py +0 -0
  90. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_check_catalog.py +0 -0
  91. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_citation.py +0 -0
  92. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_citation_levels.py +0 -0
  93. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_cli.py +0 -0
  94. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_compat_shim.py +0 -0
  95. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_config.py +0 -0
  96. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_config_lint.py +0 -0
  97. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_content_inspect.py +0 -0
  98. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_corpus.py +0 -0
  99. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_corpus_health.py +0 -0
  100. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_coverage.py +0 -0
  101. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_cross_stage.py +0 -0
  102. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_diagnostics.py +0 -0
  103. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_diff.py +0 -0
  104. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_discover_cli.py +0 -0
  105. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_discovery_parallel.py +0 -0
  106. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_embedding_drift.py +0 -0
  107. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_eval_enrich.py +0 -0
  108. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_eval_interop.py +0 -0
  109. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_evidence.py +0 -0
  110. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_gate.py +0 -0
  111. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_graph_transport.py +0 -0
  112. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_graph_validators.py +0 -0
  113. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_grounding.py +0 -0
  114. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_grounding_report.py +0 -0
  115. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_incremental.py +0 -0
  116. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_init_wizard.py +0 -0
  117. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_manifest.py +0 -0
  118. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_mcp.py +0 -0
  119. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_migration.py +0 -0
  120. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_models.py +0 -0
  121. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_neptune_bugfixes.py +0 -0
  122. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_neptune_bugfixes2.py +0 -0
  123. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_notifications.py +0 -0
  124. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_pipeline.py +0 -0
  125. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_profiler.py +0 -0
  126. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_providers.py +0 -0
  127. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_quality_gate.py +0 -0
  128. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_redis_chunks.py +0 -0
  129. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_registry.py +0 -0
  130. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_remediation.py +0 -0
  131. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_reporting.py +0 -0
  132. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_retrieval_assertions.py +0 -0
  133. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_retrieval_fields.py +0 -0
  134. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_retrieval_quality.py +0 -0
  135. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_review_hardening.py +0 -0
  136. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_run_cli.py +0 -0
  137. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_source_verification.py +0 -0
  138. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_sources.py +0 -0
  139. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_storage_validator.py +0 -0
  140. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_trends.py +0 -0
  141. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_universal_concerns.py +0 -0
  142. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_utils.py +0 -0
  143. {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_vectorstore_filters.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: compass-kb-validation
3
- Version: 3.7.0
3
+ Version: 3.8.0
4
4
  Summary: Knowledge-base readiness validation for RAG & AI agents — config-driven, deterministic pipeline integrity checks.
5
5
  Author: COMPASS
6
6
  License: MIT
@@ -256,6 +256,26 @@ compass grounding-audit --project my-kb --eval-results cases.jsonl --verify-sour
256
256
  # a response is grounded; COMPASS CHECKS it against the KB - so it can tell you which judge is right.
257
257
  compass enrich --project my-kb --input eval.json --output enriched.json
258
258
  compass calibrate-judges --input enriched.json --metric groundedness
259
+
260
+ # Close the loop: re-decide the eval's PASS/FAIL using deterministic truth instead of judge
261
+ # noise, and report every verdict that changed - cases the eval FAILED that are actually
262
+ # grounded (rescued), and cases it PASSED that contain a fabrication (demoted). Pure
263
+ # arithmetic over the enriched file: no KB, no LLM, no credentials. CI-ready exit codes.
264
+ compass quality-gate --input enriched.json --weights weights.json --threshold 0.80
265
+
266
+ # Is retrieval the problem, or is the KB? Compare retrieval configurations over the SAME
267
+ # validated KB using the eval's own ground-truth chunks - recall / precision / MRR / NDCG /
268
+ # hit rate. No LLM: a chunk is relevant because the ground truth says so.
269
+ compass benchmark --project my-kb --input eval.json
270
+
271
+ # Grounding dropped between runs - what changed? Attributes the regression to the documents
272
+ # responsible, ranked by share ("doc X REMOVED -> 40.0 pp across 8 cases"). A drop that no
273
+ # changed document explains is reported as unexplained, never spread to make totals reconcile.
274
+ compass diff --project my-kb # or: --baseline run-a.json --current run-b.json
275
+
276
+ # Incremental discovery: re-validate only what changed (by content, not clock), plus a
277
+ # spot-check of the rest. Unchanged documents are reported as NOT re-validated, never as passed.
278
+ compass discover --project my-kb --since last-run
259
279
  ```
260
280
 
261
281
  `enrich` adds a `compass` block per case (a **citation grounding rate** over the verifiable
@@ -185,6 +185,26 @@ compass grounding-audit --project my-kb --eval-results cases.jsonl --verify-sour
185
185
  # a response is grounded; COMPASS CHECKS it against the KB - so it can tell you which judge is right.
186
186
  compass enrich --project my-kb --input eval.json --output enriched.json
187
187
  compass calibrate-judges --input enriched.json --metric groundedness
188
+
189
+ # Close the loop: re-decide the eval's PASS/FAIL using deterministic truth instead of judge
190
+ # noise, and report every verdict that changed - cases the eval FAILED that are actually
191
+ # grounded (rescued), and cases it PASSED that contain a fabrication (demoted). Pure
192
+ # arithmetic over the enriched file: no KB, no LLM, no credentials. CI-ready exit codes.
193
+ compass quality-gate --input enriched.json --weights weights.json --threshold 0.80
194
+
195
+ # Is retrieval the problem, or is the KB? Compare retrieval configurations over the SAME
196
+ # validated KB using the eval's own ground-truth chunks - recall / precision / MRR / NDCG /
197
+ # hit rate. No LLM: a chunk is relevant because the ground truth says so.
198
+ compass benchmark --project my-kb --input eval.json
199
+
200
+ # Grounding dropped between runs - what changed? Attributes the regression to the documents
201
+ # responsible, ranked by share ("doc X REMOVED -> 40.0 pp across 8 cases"). A drop that no
202
+ # changed document explains is reported as unexplained, never spread to make totals reconcile.
203
+ compass diff --project my-kb # or: --baseline run-a.json --current run-b.json
204
+
205
+ # Incremental discovery: re-validate only what changed (by content, not clock), plus a
206
+ # spot-check of the rest. Unchanged documents are reported as NOT re-validated, never as passed.
207
+ compass discover --project my-kb --since last-run
188
208
  ```
189
209
 
190
210
  `enrich` adds a `compass` block per case (a **citation grounding rate** over the verifiable
@@ -5,4 +5,4 @@ end-to-end: API upload -> object storage -> knowledge graph -> vector store ->
5
5
  retrieval quality, plus cross-store consistency.
6
6
  """
7
7
 
8
- __version__ = "3.7.0"
8
+ __version__ = "3.8.0"
@@ -20,7 +20,6 @@ deterministic - the comparison is arithmetic, no LLM judges anything.
20
20
  from __future__ import annotations
21
21
 
22
22
  import logging
23
- import math
24
23
  import struct
25
24
  from typing import Any, Optional
26
25
 
@@ -32,14 +31,20 @@ def unpack_f32(raw: bytes) -> list[float]:
32
31
 
33
32
 
34
33
  def cosine(a: list[float], b: list[float]) -> Optional[float]:
34
+ """Cosine similarity, or None when either vector is empty, mismatched or zero-length.
35
+
36
+ Returns None rather than 0.0 on a degenerate pair: here "we could not compare these" must
37
+ not read as "they are completely dissimilar", which would look exactly like model drift.
38
+ The maths delegates to the shared implementation (which takes a numpy fast path when
39
+ available); only the None-vs-0.0 contract differs, and that difference is the point.
40
+ """
35
41
  if not a or not b or len(a) != len(b):
36
42
  return None
37
- dot = sum(x * y for x, y in zip(a, b))
38
- na = math.sqrt(sum(x * x for x in a))
39
- nb = math.sqrt(sum(x * x for x in b))
40
- if na == 0 or nb == 0:
43
+ from compass.retrieval_analysis import _cosine as cosine_of
44
+
45
+ if not any(a) or not any(b):
41
46
  return None
42
- return dot / (na * nb)
47
+ return cosine_of(a, b)
43
48
 
44
49
 
45
50
  def probe_embedding_drift(
@@ -631,12 +631,54 @@ class QdrantVectorStoreAdapter(VectorStoreAdapter):
631
631
  exposes_index_schema = False
632
632
  exposes_master_metadata = False
633
633
  supports_metadata_filters = True
634
+ supports_health = True
634
635
 
635
636
  def __init__(self, config: Any, client: Any = None):
636
637
  super().__init__(config, client)
637
638
  if self.client is None:
638
639
  self.client = self._create_client(config)
639
640
 
641
+ def check_health(self, *, warn_at: float = 0.70, error_at: float = 0.85):
642
+ """Qdrant collection health from ``get_collection``.
643
+
644
+ Qdrant publishes no capacity ceiling, so there is no percentage to report and inventing
645
+ one would be a fabricated reassurance. What it does publish is the OPTIMIZER state, and
646
+ a collection that is not ``green`` is the signal that matters here: segments are not
647
+ merging, which is what resource exhaustion looks like on this backend before writes
648
+ start failing. The vector count rides along as evidence.
649
+
650
+ This is also the point of the capability being a protocol rather than a shared
651
+ implementation - the same question ("are you near a limit?") has a completely different
652
+ answer shape on Qdrant than on Redis, and neither has to pretend otherwise.
653
+ """
654
+ from compass.health import ERROR, OK, UNKNOWN, WARN, HealthReport
655
+
656
+ try:
657
+ info = self.client.get_collection(self._collection)
658
+ except Exception as exc: # noqa: BLE001 - a stats call must never break the run
659
+ return HealthReport(backend="qdrant", detail=f"get_collection failed: {exc}")
660
+ raw_status = getattr(info, "status", None)
661
+ status_name = str(getattr(raw_status, "name", raw_status) or "").lower()
662
+ points = getattr(info, "points_count", None)
663
+ state = {"green": OK, "yellow": WARN, "red": ERROR}.get(status_name, UNKNOWN)
664
+ report = HealthReport(
665
+ backend="qdrant", status=state,
666
+ vector_count=int(points) if isinstance(points, int) else None,
667
+ used=f"{points:,} vectors" if isinstance(points, int) else "",
668
+ raw={"status": status_name, "points_count": points,
669
+ "segments": getattr(info, "segments_count", None)})
670
+ if state == ERROR:
671
+ report.detail = ("collection status is 'red' - optimizers are failing, which "
672
+ "usually means the node is out of RAM or disk. Writes may be "
673
+ "rejected or silently lag.")
674
+ elif state == WARN:
675
+ report.detail = ("collection status is 'yellow' - optimization is in progress or "
676
+ "degraded; capacity pressure is the usual cause.")
677
+ elif state == UNKNOWN:
678
+ report.detail = (f"collection reported status '{status_name or 'none'}', which "
679
+ "COMPASS does not recognise - capacity was not assessed.")
680
+ return report
681
+
640
682
  @staticmethod
641
683
  def _create_client(config: Any) -> Any:
642
684
  import os
@@ -220,15 +220,21 @@ class RedisChunksValidator(BaseValidator):
220
220
 
221
221
  @staticmethod
222
222
  def _cosine(a: bytes, b: bytes) -> float:
223
+ """Cosine over two raw float32 vectors.
224
+
225
+ Unpacking is local (this is the only caller that starts from bytes), but the maths
226
+ itself is the shared implementation - three hand-rolled copies of the same formula is
227
+ three places for a subtle difference to hide. 0.0 for a degenerate pair, matching the
228
+ rest of this validator's "no signal" convention rather than propagating None.
229
+ """
230
+ from compass.retrieval_analysis import _cosine as cosine_of
231
+
223
232
  n = min(len(a), len(b)) // 4
224
233
  if n == 0:
225
234
  return 0.0
226
- va = struct.unpack(f"{n}f", a[: n * 4])
227
- vb = struct.unpack(f"{n}f", b[: n * 4])
228
- dot = sum(x * y for x, y in zip(va, vb))
229
- na = math.sqrt(sum(x * x for x in va))
230
- nb = math.sqrt(sum(y * y for y in vb))
231
- return dot / (na * nb) if na and nb else 0.0
235
+ va = list(struct.unpack(f"{n}f", a[: n * 4]))
236
+ vb = list(struct.unpack(f"{n}f", b[: n * 4]))
237
+ return cosine_of(va, vb)
232
238
 
233
239
  def _pdf_text(self, file_path: str) -> str:
234
240
  if not file_path or PdfReader is None or not Path(file_path).is_file():
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: compass-kb-validation
3
- Version: 3.7.0
3
+ Version: 3.8.0
4
4
  Summary: Knowledge-base readiness validation for RAG & AI agents — config-driven, deterministic pipeline integrity checks.
5
5
  Author: COMPASS
6
6
  License: MIT
@@ -256,6 +256,26 @@ compass grounding-audit --project my-kb --eval-results cases.jsonl --verify-sour
256
256
  # a response is grounded; COMPASS CHECKS it against the KB - so it can tell you which judge is right.
257
257
  compass enrich --project my-kb --input eval.json --output enriched.json
258
258
  compass calibrate-judges --input enriched.json --metric groundedness
259
+
260
+ # Close the loop: re-decide the eval's PASS/FAIL using deterministic truth instead of judge
261
+ # noise, and report every verdict that changed - cases the eval FAILED that are actually
262
+ # grounded (rescued), and cases it PASSED that contain a fabrication (demoted). Pure
263
+ # arithmetic over the enriched file: no KB, no LLM, no credentials. CI-ready exit codes.
264
+ compass quality-gate --input enriched.json --weights weights.json --threshold 0.80
265
+
266
+ # Is retrieval the problem, or is the KB? Compare retrieval configurations over the SAME
267
+ # validated KB using the eval's own ground-truth chunks - recall / precision / MRR / NDCG /
268
+ # hit rate. No LLM: a chunk is relevant because the ground truth says so.
269
+ compass benchmark --project my-kb --input eval.json
270
+
271
+ # Grounding dropped between runs - what changed? Attributes the regression to the documents
272
+ # responsible, ranked by share ("doc X REMOVED -> 40.0 pp across 8 cases"). A drop that no
273
+ # changed document explains is reported as unexplained, never spread to make totals reconcile.
274
+ compass diff --project my-kb # or: --baseline run-a.json --current run-b.json
275
+
276
+ # Incremental discovery: re-validate only what changed (by content, not clock), plus a
277
+ # spot-check of the rest. Unchanged documents are reported as NOT re-validated, never as passed.
278
+ compass discover --project my-kb --since last-run
259
279
  ```
260
280
 
261
281
  `enrich` adds a `compass` block per case (a **citation grounding rate** over the verifiable
@@ -86,6 +86,7 @@ tests/test_check_catalog.py
86
86
  tests/test_citation.py
87
87
  tests/test_citation_levels.py
88
88
  tests/test_cli.py
89
+ tests/test_cli_commands.py
89
90
  tests/test_compat_shim.py
90
91
  tests/test_config.py
91
92
  tests/test_config_lint.py
@@ -0,0 +1,158 @@
1
+ """CLI behaviour for the analysis commands.
2
+
3
+ cli.py carries the most user-facing surface and had the least coverage, which is where every
4
+ defect that reached a push this cycle actually lived - an argparse default that could not
5
+ distinguish "user asked for this" from "nobody said", and message strings that broke the
6
+ module. These cover the parts that are logic rather than plumbing: exit codes CI depends on,
7
+ input validation, and the flags that change a verdict.
8
+ """
9
+
10
+ import json
11
+
12
+ import pytest
13
+
14
+ from compass.cli import cmd_benchmark, cmd_diff, cmd_quality_gate, main
15
+
16
+
17
+ def _write(path, payload):
18
+ path.write_text(json.dumps(payload), encoding="utf-8")
19
+ return str(path)
20
+
21
+
22
+ # --------------------------------------------------------------------------- #
23
+ # quality-gate: the exit codes a CI pipeline branches on
24
+ # --------------------------------------------------------------------------- #
25
+ def _case(cid, truth, judge):
26
+ return {"case_id": cid,
27
+ "compass": {"groundedness": {"score": truth, "basis": "citation_numeric",
28
+ "confidence": "high"}},
29
+ "scores": {"groundedness": {"judges": {"j": judge}}}}
30
+
31
+
32
+ def test_quality_gate_exits_zero_when_everything_passes_after_correction(tmp_path, capsys):
33
+ path = _write(tmp_path / "e.json", [_case("T1", 0.95, 0.9)])
34
+ assert cmd_quality_gate(["--input", path, "--threshold", "0.8"]) == 0
35
+ assert "Quality gate" in capsys.readouterr().out
36
+
37
+
38
+ def test_quality_gate_exits_one_when_a_real_failure_remains(tmp_path):
39
+ path = _write(tmp_path / "e.json", [_case("T1", 0.1, 0.1)])
40
+ assert cmd_quality_gate(["--input", path, "--threshold", "0.8"]) == 1
41
+
42
+
43
+ def test_quality_gate_exits_two_on_unreadable_input(tmp_path):
44
+ bad = tmp_path / "broken.json"
45
+ bad.write_text("{not json", encoding="utf-8")
46
+ assert cmd_quality_gate(["--input", str(bad)]) == 2
47
+
48
+
49
+ def test_quality_gate_exits_two_when_no_cases_are_found(tmp_path):
50
+ assert cmd_quality_gate(["--input", _write(tmp_path / "e.json", {"nope": 1})]) == 2
51
+
52
+
53
+ def test_quality_gate_rejects_a_weights_file_with_no_weights(tmp_path):
54
+ cases = _write(tmp_path / "e.json", [_case("T1", 0.95, 0.9)])
55
+ weights = _write(tmp_path / "w.json", {"note": "not weights"})
56
+ assert cmd_quality_gate(["--input", cases, "--weights", weights]) == 2
57
+
58
+
59
+ def test_quality_gate_accepts_calibrate_judges_output_as_weights(tmp_path):
60
+ cases = _write(tmp_path / "e.json", [_case("T1", 0.95, 0.9)])
61
+ weights = _write(tmp_path / "w.json",
62
+ {"judges": {"j": {"recommended_weight": 0.8}}})
63
+ assert cmd_quality_gate(["--input", cases, "--weights", weights]) == 0
64
+
65
+
66
+ def test_quality_gate_json_output_is_parseable(tmp_path, capsys):
67
+ path = _write(tmp_path / "e.json", [_case("T1", 0.95, 0.9)])
68
+ cmd_quality_gate(["--input", path, "--json"])
69
+ payload = json.loads(capsys.readouterr().out)
70
+ assert "gate" in payload and payload["gate"]["threshold"] == 0.8
71
+
72
+
73
+ def test_quality_gate_writes_to_a_file_when_asked(tmp_path):
74
+ path = _write(tmp_path / "e.json", [_case("T1", 0.95, 0.9)])
75
+ out = tmp_path / "gate.txt"
76
+ cmd_quality_gate(["--input", path, "--output", str(out)])
77
+ assert "Quality gate" in out.read_text(encoding="utf-8")
78
+
79
+
80
+ # --------------------------------------------------------------------------- #
81
+ # diff
82
+ # --------------------------------------------------------------------------- #
83
+ def _snap(docs, cases=None):
84
+ return {"project": "kb", "run_id": "r", "documents": docs, "cases": cases or {},
85
+ "summary": {"documents": len(docs), "chunks": 0,
86
+ "mean_groundedness": 0.9 if cases else None}}
87
+
88
+
89
+ def test_diff_needs_two_snapshots(tmp_path, capsys):
90
+ assert cmd_diff(["--project", "nonexistent-project-xyz"]) == 2
91
+ assert "Need two run snapshots" in capsys.readouterr().err
92
+
93
+
94
+ def test_diff_rejects_a_file_that_is_not_a_snapshot(tmp_path, capsys):
95
+ a = _write(tmp_path / "a.json", {"documents": {}})
96
+ b = _write(tmp_path / "b.json", {"not": "a snapshot"})
97
+ assert cmd_diff(["--baseline", a, "--current", b]) == 2
98
+ assert "not a COMPASS run snapshot" in capsys.readouterr().err
99
+
100
+
101
+ def test_diff_reports_a_clean_comparison(tmp_path, capsys):
102
+ docs = {"a.pdf": {"file_name": "a.pdf", "etag": "1", "chunks": 5}}
103
+ a = _write(tmp_path / "a.json", _snap(docs))
104
+ b = _write(tmp_path / "b.json", _snap(docs))
105
+ assert cmd_diff(["--baseline", a, "--current", b]) == 0
106
+ assert "COMPASS diff" in capsys.readouterr().out
107
+
108
+
109
+ def test_diff_fail_on_regression_only_fires_on_a_real_drop(tmp_path):
110
+ docs = {"a.pdf": {"file_name": "a.pdf", "etag": "1", "chunks": 5}}
111
+ before = _snap(docs, {"T1": {"groundedness": 0.9, "cited_documents": ["a.pdf"]}})
112
+ after = _snap(docs, {"T1": {"groundedness": 0.2, "cited_documents": ["a.pdf"]}})
113
+ after["summary"]["mean_groundedness"] = 0.2
114
+ a, b = _write(tmp_path / "a.json", before), _write(tmp_path / "b.json", after)
115
+ assert cmd_diff(["--baseline", a, "--current", b, "--fail-on-regression"]) == 1
116
+ # Without the flag a regression is reported but does not fail the command.
117
+ assert cmd_diff(["--baseline", a, "--current", b]) == 0
118
+
119
+
120
+ # --------------------------------------------------------------------------- #
121
+ # benchmark
122
+ # --------------------------------------------------------------------------- #
123
+ def test_benchmark_rejects_an_unknown_project(tmp_path, capsys):
124
+ path = _write(tmp_path / "e.json", [{"case_id": "T1"}])
125
+ assert cmd_benchmark(["--project", "nonexistent-project-xyz", "--input", path]) == 2
126
+ assert "not found" in capsys.readouterr().err
127
+
128
+
129
+ def test_benchmark_rejects_unreadable_input(tmp_path, capsys):
130
+ assert cmd_benchmark(["--project", "local", "--input", str(tmp_path / "absent.json")]) == 2
131
+ assert "Cannot read" in capsys.readouterr().err
132
+
133
+
134
+ def test_benchmark_explains_how_to_configure_strategies(tmp_path, capsys):
135
+ # The `local` project configures no strategies; the command must say what to add rather
136
+ # than invent a default pair the project never asked for.
137
+ path = _write(tmp_path / "e.json", [{"case_id": "T1", "input": "q"}])
138
+ assert cmd_benchmark(["--project", "local", "--input", path]) == 2
139
+ err = capsys.readouterr().err
140
+ assert "No strategies configured" in err and "benchmark:" in err
141
+
142
+
143
+ # --------------------------------------------------------------------------- #
144
+ # Dispatch
145
+ # --------------------------------------------------------------------------- #
146
+ @pytest.mark.parametrize("command", ["quality-gate", "diff", "benchmark"])
147
+ def test_the_new_commands_are_reachable_from_the_dispatcher(command, capsys):
148
+ with pytest.raises(SystemExit) as exc:
149
+ main([command, "--help"])
150
+ assert exc.value.code == 0
151
+ assert command in capsys.readouterr().out
152
+
153
+
154
+ def test_the_help_listing_mentions_the_new_commands(capsys):
155
+ main([])
156
+ out = capsys.readouterr().out
157
+ for command in ("quality-gate", "diff", "benchmark"):
158
+ assert command in out
@@ -480,3 +480,64 @@ def test_an_annotation_message_cannot_break_the_workflow_command():
480
480
  line = format_github_annotations([{"name": "x,y.pdf", "result": r}])[0]
481
481
  assert "\n" not in line and line.count("::") == 2
482
482
  assert "x;y.pdf" in line # comma in the title escaped
483
+
484
+
485
+ # --------------------------------------------------------------------------- #
486
+ # The capability generalises past Redis
487
+ # --------------------------------------------------------------------------- #
488
+ class _FakeQdrant:
489
+ def __init__(self, status="green", points=1000, boom=False):
490
+ self._status, self._points, self._boom = status, points, boom
491
+
492
+ def get_collection(self, name=None):
493
+ if self._boom:
494
+ raise RuntimeError("collection gone")
495
+ return SimpleNamespace(status=self._status, points_count=self._points,
496
+ segments_count=4)
497
+
498
+
499
+ def _qdrant(client):
500
+ from compass.providers.vectorstore import QdrantVectorStoreAdapter
501
+
502
+ cfg = SimpleNamespace(search_index="c", embedding_field="v", document_id_field="id",
503
+ positional_field="", provider="qdrant", extra={})
504
+ return QdrantVectorStoreAdapter(cfg, client=client)
505
+
506
+
507
+ def test_qdrant_maps_optimizer_state_to_health():
508
+ # Qdrant publishes no capacity ceiling, so the optimizer state IS the signal - a 'red'
509
+ # collection means segments are not merging, which is resource exhaustion here.
510
+ assert _qdrant(_FakeQdrant("green")).check_health().status == OK
511
+ assert _qdrant(_FakeQdrant("yellow")).check_health().status == WARN
512
+ red = _qdrant(_FakeQdrant("red")).check_health()
513
+ assert red.status == ERROR and "out of RAM or disk" in red.detail
514
+
515
+
516
+ def test_qdrant_reports_no_percentage_it_cannot_measure():
517
+ # Inventing a usage_pct for a backend that publishes no ceiling would be a fabricated
518
+ # reassurance - the whole point of the report carrying Optional fields.
519
+ r = _qdrant(_FakeQdrant("green", points=2500)).check_health()
520
+ assert r.usage_pct is None and r.capacity is None
521
+ assert r.vector_count == 2500 and "2,500 vectors" in r.used
522
+
523
+
524
+ def test_qdrant_unknown_status_is_not_assumed_healthy():
525
+ r = _qdrant(_FakeQdrant("mauve")).check_health()
526
+ assert r.status == UNKNOWN and "does not recognise" in r.detail
527
+
528
+
529
+ def test_qdrant_health_never_raises():
530
+ assert _qdrant(_FakeQdrant(boom=True)).check_health().status == UNKNOWN
531
+
532
+
533
+ def test_two_backends_now_answer_the_same_protocol():
534
+ # The abstraction was asserted but unproven while only Redis implemented it.
535
+ from compass.providers.vectorstore import (QdrantVectorStoreAdapter,
536
+ RedisVectorStoreAdapter, VectorStoreAdapter)
537
+
538
+ assert RedisVectorStoreAdapter.supports_health is True
539
+ assert QdrantVectorStoreAdapter.supports_health is True
540
+ assert VectorStoreAdapter.supports_health is False # base stays honest
541
+ for adapter in (_redis_adapter(_FakeRedis(used=1, maxmemory=10)), _qdrant(_FakeQdrant())):
542
+ report = check_vectorstore(adapter)
543
+ assert report.status in (OK, WARN, ERROR, UNKNOWN) and report.backend
@@ -96,3 +96,38 @@ def test_pairs_above_with_zero_vectors(both_paths):
96
96
  fast, slow = both_paths(lambda: ra.pairs_above(vecs, 0.5))
97
97
  assert fast == slow
98
98
  assert [(i, j) for i, j, _ in fast] == [(1, 2)] # the zero vector matches nothing
99
+
100
+
101
+ # --------------------------------------------------------------------------- #
102
+ # One cosine implementation, three call sites
103
+ # --------------------------------------------------------------------------- #
104
+ def test_the_drift_probe_uses_the_shared_cosine_and_keeps_its_none_contract():
105
+ # Same maths as retrieval_analysis (so it gets the numpy fast path), but None rather than
106
+ # 0.0 on a degenerate pair: "could not compare" must not read as "completely dissimilar",
107
+ # which is exactly what model drift looks like.
108
+ from compass.embedding_drift import cosine
109
+
110
+ assert cosine([1.0, 0.0, 0.0], [1.0, 0.0, 0.0]) == 1.0
111
+ assert cosine([1.0, 0.0], [0.0, 1.0]) == 0.0
112
+ assert cosine([0.0, 0.0], [1.0, 0.0]) is None # zero vector
113
+ assert cosine([1.0], [1.0, 2.0]) is None # length mismatch
114
+ assert cosine([], []) is None
115
+
116
+
117
+ def test_the_chunk_validator_cosine_agrees_with_the_shared_one():
118
+ import struct
119
+
120
+ from compass.retrieval_analysis import _cosine
121
+ from compass.validators.redis_chunks import RedisChunksValidator
122
+
123
+ a, b = [1.0, 2.0, 3.0], [1.0, 2.0, 3.5]
124
+ packed = (struct.pack("3f", *a), struct.pack("3f", *b))
125
+ assert RedisChunksValidator._cosine(*packed) == pytest.approx(_cosine(a, b), abs=1e-6)
126
+
127
+
128
+ def test_the_chunk_validator_cosine_returns_zero_for_a_degenerate_pair():
129
+ # This validator's convention is 0.0 = "no signal"; propagating None would break callers
130
+ # that compare against a threshold.
131
+ from compass.validators.redis_chunks import RedisChunksValidator
132
+
133
+ assert RedisChunksValidator._cosine(b"", b"") == 0.0