compass-kb-validation 3.7.0__tar.gz → 3.8.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {compass_kb_validation-3.7.0/compass_kb_validation.egg-info → compass_kb_validation-3.8.0}/PKG-INFO +21 -1
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/README.md +20 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/__init__.py +1 -1
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/embedding_drift.py +11 -6
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/providers/vectorstore.py +42 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/validators/redis_chunks.py +12 -6
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0/compass_kb_validation.egg-info}/PKG-INFO +21 -1
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass_kb_validation.egg-info/SOURCES.txt +1 -0
- compass_kb_validation-3.8.0/tests/test_cli_commands.py +158 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_health.py +61 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_vector_fastpath.py +35 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/LICENSE +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/__main__.py +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/alias.py +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/ask.py +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/baseline.py +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/benchmark.py +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/chunk_coherence.py +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/citation.py +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/cli.py +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/config/__init__.py +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/config_lint.py +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/content_inspect.py +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/corpus.py +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/corpus_health.py +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/diagnostics.py +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/diff.py +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/eval_enrich.py +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/eval_interop.py +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/evidence.py +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/grounding.py +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/health.py +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/incremental.py +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/init_wizard.py +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/live_dashboard.py +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/mcp_server.py +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/models/__init__.py +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/notifications.py +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/pipeline/__init__.py +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/profiler.py +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/provenance.py +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/providers/__init__.py +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/providers/embedding.py +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/providers/graph.py +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/providers/source.py +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/providers/storage.py +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/quality_gate.py +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/remediation.py +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/report/__init__.py +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/report/_report_css.py +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/report/_report_js.py +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/report/analysis.py +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/report/audit.py +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/report/export.py +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/report/grounding_report.py +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/report/shell.py +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/retrieval_analysis.py +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/text_probes.py +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/trend_tracker.py +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/utils/__init__.py +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/validators/__init__.py +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/validators/api_upload.py +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/validators/base.py +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/validators/base_sparql.py +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/validators/cross_stage.py +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/validators/graphdb_metadata.py +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/validators/neptune_metadata.py +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/validators/registry.py +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/validators/retrieval_quality.py +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/validators/s3_storage.py +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass_kb_validation.egg-info/dependency_links.txt +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass_kb_validation.egg-info/entry_points.txt +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass_kb_validation.egg-info/requires.txt +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass_kb_validation.egg-info/top_level.txt +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/discover_and_validate.py +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/ingestion_validation/__init__.py +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/profile_kb.py +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/pyproject.toml +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/run_validation.py +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/setup.cfg +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_alias.py +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_alias_disambiguation.py +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_alt_adapters.py +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_analysis_report.py +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_api_upload.py +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_ask.py +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_baseline.py +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_benchmark.py +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_calibrate_truth.py +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_check_catalog.py +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_citation.py +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_citation_levels.py +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_cli.py +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_compat_shim.py +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_config.py +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_config_lint.py +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_content_inspect.py +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_corpus.py +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_corpus_health.py +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_coverage.py +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_cross_stage.py +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_diagnostics.py +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_diff.py +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_discover_cli.py +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_discovery_parallel.py +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_embedding_drift.py +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_eval_enrich.py +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_eval_interop.py +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_evidence.py +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_gate.py +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_graph_transport.py +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_graph_validators.py +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_grounding.py +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_grounding_report.py +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_incremental.py +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_init_wizard.py +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_manifest.py +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_mcp.py +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_migration.py +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_models.py +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_neptune_bugfixes.py +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_neptune_bugfixes2.py +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_notifications.py +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_pipeline.py +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_profiler.py +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_providers.py +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_quality_gate.py +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_redis_chunks.py +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_registry.py +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_remediation.py +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_reporting.py +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_retrieval_assertions.py +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_retrieval_fields.py +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_retrieval_quality.py +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_review_hardening.py +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_run_cli.py +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_source_verification.py +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_sources.py +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_storage_validator.py +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_trends.py +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_universal_concerns.py +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_utils.py +0 -0
- {compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_vectorstore_filters.py +0 -0
{compass_kb_validation-3.7.0/compass_kb_validation.egg-info → compass_kb_validation-3.8.0}/PKG-INFO
RENAMED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: compass-kb-validation
|
|
3
|
-
Version: 3.
|
|
3
|
+
Version: 3.8.0
|
|
4
4
|
Summary: Knowledge-base readiness validation for RAG & AI agents — config-driven, deterministic pipeline integrity checks.
|
|
5
5
|
Author: COMPASS
|
|
6
6
|
License: MIT
|
|
@@ -256,6 +256,26 @@ compass grounding-audit --project my-kb --eval-results cases.jsonl --verify-sour
|
|
|
256
256
|
# a response is grounded; COMPASS CHECKS it against the KB - so it can tell you which judge is right.
|
|
257
257
|
compass enrich --project my-kb --input eval.json --output enriched.json
|
|
258
258
|
compass calibrate-judges --input enriched.json --metric groundedness
|
|
259
|
+
|
|
260
|
+
# Close the loop: re-decide the eval's PASS/FAIL using deterministic truth instead of judge
|
|
261
|
+
# noise, and report every verdict that changed - cases the eval FAILED that are actually
|
|
262
|
+
# grounded (rescued), and cases it PASSED that contain a fabrication (demoted). Pure
|
|
263
|
+
# arithmetic over the enriched file: no KB, no LLM, no credentials. CI-ready exit codes.
|
|
264
|
+
compass quality-gate --input enriched.json --weights weights.json --threshold 0.80
|
|
265
|
+
|
|
266
|
+
# Is retrieval the problem, or is the KB? Compare retrieval configurations over the SAME
|
|
267
|
+
# validated KB using the eval's own ground-truth chunks - recall / precision / MRR / NDCG /
|
|
268
|
+
# hit rate. No LLM: a chunk is relevant because the ground truth says so.
|
|
269
|
+
compass benchmark --project my-kb --input eval.json
|
|
270
|
+
|
|
271
|
+
# Grounding dropped between runs - what changed? Attributes the regression to the documents
|
|
272
|
+
# responsible, ranked by share ("doc X REMOVED -> 40.0 pp across 8 cases"). A drop that no
|
|
273
|
+
# changed document explains is reported as unexplained, never spread to make totals reconcile.
|
|
274
|
+
compass diff --project my-kb # or: --baseline run-a.json --current run-b.json
|
|
275
|
+
|
|
276
|
+
# Incremental discovery: re-validate only what changed (by content, not clock), plus a
|
|
277
|
+
# spot-check of the rest. Unchanged documents are reported as NOT re-validated, never as passed.
|
|
278
|
+
compass discover --project my-kb --since last-run
|
|
259
279
|
```
|
|
260
280
|
|
|
261
281
|
`enrich` adds a `compass` block per case (a **citation grounding rate** over the verifiable
|
|
@@ -185,6 +185,26 @@ compass grounding-audit --project my-kb --eval-results cases.jsonl --verify-sour
|
|
|
185
185
|
# a response is grounded; COMPASS CHECKS it against the KB - so it can tell you which judge is right.
|
|
186
186
|
compass enrich --project my-kb --input eval.json --output enriched.json
|
|
187
187
|
compass calibrate-judges --input enriched.json --metric groundedness
|
|
188
|
+
|
|
189
|
+
# Close the loop: re-decide the eval's PASS/FAIL using deterministic truth instead of judge
|
|
190
|
+
# noise, and report every verdict that changed - cases the eval FAILED that are actually
|
|
191
|
+
# grounded (rescued), and cases it PASSED that contain a fabrication (demoted). Pure
|
|
192
|
+
# arithmetic over the enriched file: no KB, no LLM, no credentials. CI-ready exit codes.
|
|
193
|
+
compass quality-gate --input enriched.json --weights weights.json --threshold 0.80
|
|
194
|
+
|
|
195
|
+
# Is retrieval the problem, or is the KB? Compare retrieval configurations over the SAME
|
|
196
|
+
# validated KB using the eval's own ground-truth chunks - recall / precision / MRR / NDCG /
|
|
197
|
+
# hit rate. No LLM: a chunk is relevant because the ground truth says so.
|
|
198
|
+
compass benchmark --project my-kb --input eval.json
|
|
199
|
+
|
|
200
|
+
# Grounding dropped between runs - what changed? Attributes the regression to the documents
|
|
201
|
+
# responsible, ranked by share ("doc X REMOVED -> 40.0 pp across 8 cases"). A drop that no
|
|
202
|
+
# changed document explains is reported as unexplained, never spread to make totals reconcile.
|
|
203
|
+
compass diff --project my-kb # or: --baseline run-a.json --current run-b.json
|
|
204
|
+
|
|
205
|
+
# Incremental discovery: re-validate only what changed (by content, not clock), plus a
|
|
206
|
+
# spot-check of the rest. Unchanged documents are reported as NOT re-validated, never as passed.
|
|
207
|
+
compass discover --project my-kb --since last-run
|
|
188
208
|
```
|
|
189
209
|
|
|
190
210
|
`enrich` adds a `compass` block per case (a **citation grounding rate** over the verifiable
|
|
@@ -20,7 +20,6 @@ deterministic - the comparison is arithmetic, no LLM judges anything.
|
|
|
20
20
|
from __future__ import annotations
|
|
21
21
|
|
|
22
22
|
import logging
|
|
23
|
-
import math
|
|
24
23
|
import struct
|
|
25
24
|
from typing import Any, Optional
|
|
26
25
|
|
|
@@ -32,14 +31,20 @@ def unpack_f32(raw: bytes) -> list[float]:
|
|
|
32
31
|
|
|
33
32
|
|
|
34
33
|
def cosine(a: list[float], b: list[float]) -> Optional[float]:
|
|
34
|
+
"""Cosine similarity, or None when either vector is empty, mismatched or zero-length.
|
|
35
|
+
|
|
36
|
+
Returns None rather than 0.0 on a degenerate pair: here "we could not compare these" must
|
|
37
|
+
not read as "they are completely dissimilar", which would look exactly like model drift.
|
|
38
|
+
The maths delegates to the shared implementation (which takes a numpy fast path when
|
|
39
|
+
available); only the None-vs-0.0 contract differs, and that difference is the point.
|
|
40
|
+
"""
|
|
35
41
|
if not a or not b or len(a) != len(b):
|
|
36
42
|
return None
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
if na == 0 or nb == 0:
|
|
43
|
+
from compass.retrieval_analysis import _cosine as cosine_of
|
|
44
|
+
|
|
45
|
+
if not any(a) or not any(b):
|
|
41
46
|
return None
|
|
42
|
-
return
|
|
47
|
+
return cosine_of(a, b)
|
|
43
48
|
|
|
44
49
|
|
|
45
50
|
def probe_embedding_drift(
|
{compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/providers/vectorstore.py
RENAMED
|
@@ -631,12 +631,54 @@ class QdrantVectorStoreAdapter(VectorStoreAdapter):
|
|
|
631
631
|
exposes_index_schema = False
|
|
632
632
|
exposes_master_metadata = False
|
|
633
633
|
supports_metadata_filters = True
|
|
634
|
+
supports_health = True
|
|
634
635
|
|
|
635
636
|
def __init__(self, config: Any, client: Any = None):
|
|
636
637
|
super().__init__(config, client)
|
|
637
638
|
if self.client is None:
|
|
638
639
|
self.client = self._create_client(config)
|
|
639
640
|
|
|
641
|
+
def check_health(self, *, warn_at: float = 0.70, error_at: float = 0.85):
|
|
642
|
+
"""Qdrant collection health from ``get_collection``.
|
|
643
|
+
|
|
644
|
+
Qdrant publishes no capacity ceiling, so there is no percentage to report and inventing
|
|
645
|
+
one would be a fabricated reassurance. What it does publish is the OPTIMIZER state, and
|
|
646
|
+
a collection that is not ``green`` is the signal that matters here: segments are not
|
|
647
|
+
merging, which is what resource exhaustion looks like on this backend before writes
|
|
648
|
+
start failing. The vector count rides along as evidence.
|
|
649
|
+
|
|
650
|
+
This is also the point of the capability being a protocol rather than a shared
|
|
651
|
+
implementation - the same question ("are you near a limit?") has a completely different
|
|
652
|
+
answer shape on Qdrant than on Redis, and neither has to pretend otherwise.
|
|
653
|
+
"""
|
|
654
|
+
from compass.health import ERROR, OK, UNKNOWN, WARN, HealthReport
|
|
655
|
+
|
|
656
|
+
try:
|
|
657
|
+
info = self.client.get_collection(self._collection)
|
|
658
|
+
except Exception as exc: # noqa: BLE001 - a stats call must never break the run
|
|
659
|
+
return HealthReport(backend="qdrant", detail=f"get_collection failed: {exc}")
|
|
660
|
+
raw_status = getattr(info, "status", None)
|
|
661
|
+
status_name = str(getattr(raw_status, "name", raw_status) or "").lower()
|
|
662
|
+
points = getattr(info, "points_count", None)
|
|
663
|
+
state = {"green": OK, "yellow": WARN, "red": ERROR}.get(status_name, UNKNOWN)
|
|
664
|
+
report = HealthReport(
|
|
665
|
+
backend="qdrant", status=state,
|
|
666
|
+
vector_count=int(points) if isinstance(points, int) else None,
|
|
667
|
+
used=f"{points:,} vectors" if isinstance(points, int) else "",
|
|
668
|
+
raw={"status": status_name, "points_count": points,
|
|
669
|
+
"segments": getattr(info, "segments_count", None)})
|
|
670
|
+
if state == ERROR:
|
|
671
|
+
report.detail = ("collection status is 'red' - optimizers are failing, which "
|
|
672
|
+
"usually means the node is out of RAM or disk. Writes may be "
|
|
673
|
+
"rejected or silently lag.")
|
|
674
|
+
elif state == WARN:
|
|
675
|
+
report.detail = ("collection status is 'yellow' - optimization is in progress or "
|
|
676
|
+
"degraded; capacity pressure is the usual cause.")
|
|
677
|
+
elif state == UNKNOWN:
|
|
678
|
+
report.detail = (f"collection reported status '{status_name or 'none'}', which "
|
|
679
|
+
"COMPASS does not recognise - capacity was not assessed.")
|
|
680
|
+
return report
|
|
681
|
+
|
|
640
682
|
@staticmethod
|
|
641
683
|
def _create_client(config: Any) -> Any:
|
|
642
684
|
import os
|
{compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/validators/redis_chunks.py
RENAMED
|
@@ -220,15 +220,21 @@ class RedisChunksValidator(BaseValidator):
|
|
|
220
220
|
|
|
221
221
|
@staticmethod
|
|
222
222
|
def _cosine(a: bytes, b: bytes) -> float:
|
|
223
|
+
"""Cosine over two raw float32 vectors.
|
|
224
|
+
|
|
225
|
+
Unpacking is local (this is the only caller that starts from bytes), but the maths
|
|
226
|
+
itself is the shared implementation - three hand-rolled copies of the same formula is
|
|
227
|
+
three places for a subtle difference to hide. 0.0 for a degenerate pair, matching the
|
|
228
|
+
rest of this validator's "no signal" convention rather than propagating None.
|
|
229
|
+
"""
|
|
230
|
+
from compass.retrieval_analysis import _cosine as cosine_of
|
|
231
|
+
|
|
223
232
|
n = min(len(a), len(b)) // 4
|
|
224
233
|
if n == 0:
|
|
225
234
|
return 0.0
|
|
226
|
-
va = struct.unpack(f"{n}f", a[: n * 4])
|
|
227
|
-
vb = struct.unpack(f"{n}f", b[: n * 4])
|
|
228
|
-
|
|
229
|
-
na = math.sqrt(sum(x * x for x in va))
|
|
230
|
-
nb = math.sqrt(sum(y * y for y in vb))
|
|
231
|
-
return dot / (na * nb) if na and nb else 0.0
|
|
235
|
+
va = list(struct.unpack(f"{n}f", a[: n * 4]))
|
|
236
|
+
vb = list(struct.unpack(f"{n}f", b[: n * 4]))
|
|
237
|
+
return cosine_of(va, vb)
|
|
232
238
|
|
|
233
239
|
def _pdf_text(self, file_path: str) -> str:
|
|
234
240
|
if not file_path or PdfReader is None or not Path(file_path).is_file():
|
{compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0/compass_kb_validation.egg-info}/PKG-INFO
RENAMED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: compass-kb-validation
|
|
3
|
-
Version: 3.
|
|
3
|
+
Version: 3.8.0
|
|
4
4
|
Summary: Knowledge-base readiness validation for RAG & AI agents — config-driven, deterministic pipeline integrity checks.
|
|
5
5
|
Author: COMPASS
|
|
6
6
|
License: MIT
|
|
@@ -256,6 +256,26 @@ compass grounding-audit --project my-kb --eval-results cases.jsonl --verify-sour
|
|
|
256
256
|
# a response is grounded; COMPASS CHECKS it against the KB - so it can tell you which judge is right.
|
|
257
257
|
compass enrich --project my-kb --input eval.json --output enriched.json
|
|
258
258
|
compass calibrate-judges --input enriched.json --metric groundedness
|
|
259
|
+
|
|
260
|
+
# Close the loop: re-decide the eval's PASS/FAIL using deterministic truth instead of judge
|
|
261
|
+
# noise, and report every verdict that changed - cases the eval FAILED that are actually
|
|
262
|
+
# grounded (rescued), and cases it PASSED that contain a fabrication (demoted). Pure
|
|
263
|
+
# arithmetic over the enriched file: no KB, no LLM, no credentials. CI-ready exit codes.
|
|
264
|
+
compass quality-gate --input enriched.json --weights weights.json --threshold 0.80
|
|
265
|
+
|
|
266
|
+
# Is retrieval the problem, or is the KB? Compare retrieval configurations over the SAME
|
|
267
|
+
# validated KB using the eval's own ground-truth chunks - recall / precision / MRR / NDCG /
|
|
268
|
+
# hit rate. No LLM: a chunk is relevant because the ground truth says so.
|
|
269
|
+
compass benchmark --project my-kb --input eval.json
|
|
270
|
+
|
|
271
|
+
# Grounding dropped between runs - what changed? Attributes the regression to the documents
|
|
272
|
+
# responsible, ranked by share ("doc X REMOVED -> 40.0 pp across 8 cases"). A drop that no
|
|
273
|
+
# changed document explains is reported as unexplained, never spread to make totals reconcile.
|
|
274
|
+
compass diff --project my-kb # or: --baseline run-a.json --current run-b.json
|
|
275
|
+
|
|
276
|
+
# Incremental discovery: re-validate only what changed (by content, not clock), plus a
|
|
277
|
+
# spot-check of the rest. Unchanged documents are reported as NOT re-validated, never as passed.
|
|
278
|
+
compass discover --project my-kb --since last-run
|
|
259
279
|
```
|
|
260
280
|
|
|
261
281
|
`enrich` adds a `compass` block per case (a **citation grounding rate** over the verifiable
|
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
"""CLI behaviour for the analysis commands.
|
|
2
|
+
|
|
3
|
+
cli.py carries the most user-facing surface and had the least coverage, which is where every
|
|
4
|
+
defect that reached a push this cycle actually lived - an argparse default that could not
|
|
5
|
+
distinguish "user asked for this" from "nobody said", and message strings that broke the
|
|
6
|
+
module. These cover the parts that are logic rather than plumbing: exit codes CI depends on,
|
|
7
|
+
input validation, and the flags that change a verdict.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
import json
|
|
11
|
+
|
|
12
|
+
import pytest
|
|
13
|
+
|
|
14
|
+
from compass.cli import cmd_benchmark, cmd_diff, cmd_quality_gate, main
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def _write(path, payload):
|
|
18
|
+
path.write_text(json.dumps(payload), encoding="utf-8")
|
|
19
|
+
return str(path)
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
# --------------------------------------------------------------------------- #
|
|
23
|
+
# quality-gate: the exit codes a CI pipeline branches on
|
|
24
|
+
# --------------------------------------------------------------------------- #
|
|
25
|
+
def _case(cid, truth, judge):
|
|
26
|
+
return {"case_id": cid,
|
|
27
|
+
"compass": {"groundedness": {"score": truth, "basis": "citation_numeric",
|
|
28
|
+
"confidence": "high"}},
|
|
29
|
+
"scores": {"groundedness": {"judges": {"j": judge}}}}
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def test_quality_gate_exits_zero_when_everything_passes_after_correction(tmp_path, capsys):
|
|
33
|
+
path = _write(tmp_path / "e.json", [_case("T1", 0.95, 0.9)])
|
|
34
|
+
assert cmd_quality_gate(["--input", path, "--threshold", "0.8"]) == 0
|
|
35
|
+
assert "Quality gate" in capsys.readouterr().out
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def test_quality_gate_exits_one_when_a_real_failure_remains(tmp_path):
|
|
39
|
+
path = _write(tmp_path / "e.json", [_case("T1", 0.1, 0.1)])
|
|
40
|
+
assert cmd_quality_gate(["--input", path, "--threshold", "0.8"]) == 1
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def test_quality_gate_exits_two_on_unreadable_input(tmp_path):
|
|
44
|
+
bad = tmp_path / "broken.json"
|
|
45
|
+
bad.write_text("{not json", encoding="utf-8")
|
|
46
|
+
assert cmd_quality_gate(["--input", str(bad)]) == 2
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def test_quality_gate_exits_two_when_no_cases_are_found(tmp_path):
|
|
50
|
+
assert cmd_quality_gate(["--input", _write(tmp_path / "e.json", {"nope": 1})]) == 2
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def test_quality_gate_rejects_a_weights_file_with_no_weights(tmp_path):
|
|
54
|
+
cases = _write(tmp_path / "e.json", [_case("T1", 0.95, 0.9)])
|
|
55
|
+
weights = _write(tmp_path / "w.json", {"note": "not weights"})
|
|
56
|
+
assert cmd_quality_gate(["--input", cases, "--weights", weights]) == 2
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def test_quality_gate_accepts_calibrate_judges_output_as_weights(tmp_path):
|
|
60
|
+
cases = _write(tmp_path / "e.json", [_case("T1", 0.95, 0.9)])
|
|
61
|
+
weights = _write(tmp_path / "w.json",
|
|
62
|
+
{"judges": {"j": {"recommended_weight": 0.8}}})
|
|
63
|
+
assert cmd_quality_gate(["--input", cases, "--weights", weights]) == 0
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def test_quality_gate_json_output_is_parseable(tmp_path, capsys):
|
|
67
|
+
path = _write(tmp_path / "e.json", [_case("T1", 0.95, 0.9)])
|
|
68
|
+
cmd_quality_gate(["--input", path, "--json"])
|
|
69
|
+
payload = json.loads(capsys.readouterr().out)
|
|
70
|
+
assert "gate" in payload and payload["gate"]["threshold"] == 0.8
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def test_quality_gate_writes_to_a_file_when_asked(tmp_path):
|
|
74
|
+
path = _write(tmp_path / "e.json", [_case("T1", 0.95, 0.9)])
|
|
75
|
+
out = tmp_path / "gate.txt"
|
|
76
|
+
cmd_quality_gate(["--input", path, "--output", str(out)])
|
|
77
|
+
assert "Quality gate" in out.read_text(encoding="utf-8")
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
# --------------------------------------------------------------------------- #
|
|
81
|
+
# diff
|
|
82
|
+
# --------------------------------------------------------------------------- #
|
|
83
|
+
def _snap(docs, cases=None):
|
|
84
|
+
return {"project": "kb", "run_id": "r", "documents": docs, "cases": cases or {},
|
|
85
|
+
"summary": {"documents": len(docs), "chunks": 0,
|
|
86
|
+
"mean_groundedness": 0.9 if cases else None}}
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def test_diff_needs_two_snapshots(tmp_path, capsys):
|
|
90
|
+
assert cmd_diff(["--project", "nonexistent-project-xyz"]) == 2
|
|
91
|
+
assert "Need two run snapshots" in capsys.readouterr().err
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def test_diff_rejects_a_file_that_is_not_a_snapshot(tmp_path, capsys):
|
|
95
|
+
a = _write(tmp_path / "a.json", {"documents": {}})
|
|
96
|
+
b = _write(tmp_path / "b.json", {"not": "a snapshot"})
|
|
97
|
+
assert cmd_diff(["--baseline", a, "--current", b]) == 2
|
|
98
|
+
assert "not a COMPASS run snapshot" in capsys.readouterr().err
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def test_diff_reports_a_clean_comparison(tmp_path, capsys):
|
|
102
|
+
docs = {"a.pdf": {"file_name": "a.pdf", "etag": "1", "chunks": 5}}
|
|
103
|
+
a = _write(tmp_path / "a.json", _snap(docs))
|
|
104
|
+
b = _write(tmp_path / "b.json", _snap(docs))
|
|
105
|
+
assert cmd_diff(["--baseline", a, "--current", b]) == 0
|
|
106
|
+
assert "COMPASS diff" in capsys.readouterr().out
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def test_diff_fail_on_regression_only_fires_on_a_real_drop(tmp_path):
|
|
110
|
+
docs = {"a.pdf": {"file_name": "a.pdf", "etag": "1", "chunks": 5}}
|
|
111
|
+
before = _snap(docs, {"T1": {"groundedness": 0.9, "cited_documents": ["a.pdf"]}})
|
|
112
|
+
after = _snap(docs, {"T1": {"groundedness": 0.2, "cited_documents": ["a.pdf"]}})
|
|
113
|
+
after["summary"]["mean_groundedness"] = 0.2
|
|
114
|
+
a, b = _write(tmp_path / "a.json", before), _write(tmp_path / "b.json", after)
|
|
115
|
+
assert cmd_diff(["--baseline", a, "--current", b, "--fail-on-regression"]) == 1
|
|
116
|
+
# Without the flag a regression is reported but does not fail the command.
|
|
117
|
+
assert cmd_diff(["--baseline", a, "--current", b]) == 0
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
# --------------------------------------------------------------------------- #
|
|
121
|
+
# benchmark
|
|
122
|
+
# --------------------------------------------------------------------------- #
|
|
123
|
+
def test_benchmark_rejects_an_unknown_project(tmp_path, capsys):
|
|
124
|
+
path = _write(tmp_path / "e.json", [{"case_id": "T1"}])
|
|
125
|
+
assert cmd_benchmark(["--project", "nonexistent-project-xyz", "--input", path]) == 2
|
|
126
|
+
assert "not found" in capsys.readouterr().err
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def test_benchmark_rejects_unreadable_input(tmp_path, capsys):
|
|
130
|
+
assert cmd_benchmark(["--project", "local", "--input", str(tmp_path / "absent.json")]) == 2
|
|
131
|
+
assert "Cannot read" in capsys.readouterr().err
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def test_benchmark_explains_how_to_configure_strategies(tmp_path, capsys):
|
|
135
|
+
# The `local` project configures no strategies; the command must say what to add rather
|
|
136
|
+
# than invent a default pair the project never asked for.
|
|
137
|
+
path = _write(tmp_path / "e.json", [{"case_id": "T1", "input": "q"}])
|
|
138
|
+
assert cmd_benchmark(["--project", "local", "--input", path]) == 2
|
|
139
|
+
err = capsys.readouterr().err
|
|
140
|
+
assert "No strategies configured" in err and "benchmark:" in err
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
# --------------------------------------------------------------------------- #
|
|
144
|
+
# Dispatch
|
|
145
|
+
# --------------------------------------------------------------------------- #
|
|
146
|
+
@pytest.mark.parametrize("command", ["quality-gate", "diff", "benchmark"])
|
|
147
|
+
def test_the_new_commands_are_reachable_from_the_dispatcher(command, capsys):
|
|
148
|
+
with pytest.raises(SystemExit) as exc:
|
|
149
|
+
main([command, "--help"])
|
|
150
|
+
assert exc.value.code == 0
|
|
151
|
+
assert command in capsys.readouterr().out
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
def test_the_help_listing_mentions_the_new_commands(capsys):
|
|
155
|
+
main([])
|
|
156
|
+
out = capsys.readouterr().out
|
|
157
|
+
for command in ("quality-gate", "diff", "benchmark"):
|
|
158
|
+
assert command in out
|
|
@@ -480,3 +480,64 @@ def test_an_annotation_message_cannot_break_the_workflow_command():
|
|
|
480
480
|
line = format_github_annotations([{"name": "x,y.pdf", "result": r}])[0]
|
|
481
481
|
assert "\n" not in line and line.count("::") == 2
|
|
482
482
|
assert "x;y.pdf" in line # comma in the title escaped
|
|
483
|
+
|
|
484
|
+
|
|
485
|
+
# --------------------------------------------------------------------------- #
|
|
486
|
+
# The capability generalises past Redis
|
|
487
|
+
# --------------------------------------------------------------------------- #
|
|
488
|
+
class _FakeQdrant:
|
|
489
|
+
def __init__(self, status="green", points=1000, boom=False):
|
|
490
|
+
self._status, self._points, self._boom = status, points, boom
|
|
491
|
+
|
|
492
|
+
def get_collection(self, name=None):
|
|
493
|
+
if self._boom:
|
|
494
|
+
raise RuntimeError("collection gone")
|
|
495
|
+
return SimpleNamespace(status=self._status, points_count=self._points,
|
|
496
|
+
segments_count=4)
|
|
497
|
+
|
|
498
|
+
|
|
499
|
+
def _qdrant(client):
|
|
500
|
+
from compass.providers.vectorstore import QdrantVectorStoreAdapter
|
|
501
|
+
|
|
502
|
+
cfg = SimpleNamespace(search_index="c", embedding_field="v", document_id_field="id",
|
|
503
|
+
positional_field="", provider="qdrant", extra={})
|
|
504
|
+
return QdrantVectorStoreAdapter(cfg, client=client)
|
|
505
|
+
|
|
506
|
+
|
|
507
|
+
def test_qdrant_maps_optimizer_state_to_health():
|
|
508
|
+
# Qdrant publishes no capacity ceiling, so the optimizer state IS the signal - a 'red'
|
|
509
|
+
# collection means segments are not merging, which is resource exhaustion here.
|
|
510
|
+
assert _qdrant(_FakeQdrant("green")).check_health().status == OK
|
|
511
|
+
assert _qdrant(_FakeQdrant("yellow")).check_health().status == WARN
|
|
512
|
+
red = _qdrant(_FakeQdrant("red")).check_health()
|
|
513
|
+
assert red.status == ERROR and "out of RAM or disk" in red.detail
|
|
514
|
+
|
|
515
|
+
|
|
516
|
+
def test_qdrant_reports_no_percentage_it_cannot_measure():
|
|
517
|
+
# Inventing a usage_pct for a backend that publishes no ceiling would be a fabricated
|
|
518
|
+
# reassurance - the whole point of the report carrying Optional fields.
|
|
519
|
+
r = _qdrant(_FakeQdrant("green", points=2500)).check_health()
|
|
520
|
+
assert r.usage_pct is None and r.capacity is None
|
|
521
|
+
assert r.vector_count == 2500 and "2,500 vectors" in r.used
|
|
522
|
+
|
|
523
|
+
|
|
524
|
+
def test_qdrant_unknown_status_is_not_assumed_healthy():
|
|
525
|
+
r = _qdrant(_FakeQdrant("mauve")).check_health()
|
|
526
|
+
assert r.status == UNKNOWN and "does not recognise" in r.detail
|
|
527
|
+
|
|
528
|
+
|
|
529
|
+
def test_qdrant_health_never_raises():
|
|
530
|
+
assert _qdrant(_FakeQdrant(boom=True)).check_health().status == UNKNOWN
|
|
531
|
+
|
|
532
|
+
|
|
533
|
+
def test_two_backends_now_answer_the_same_protocol():
|
|
534
|
+
# The abstraction was asserted but unproven while only Redis implemented it.
|
|
535
|
+
from compass.providers.vectorstore import (QdrantVectorStoreAdapter,
|
|
536
|
+
RedisVectorStoreAdapter, VectorStoreAdapter)
|
|
537
|
+
|
|
538
|
+
assert RedisVectorStoreAdapter.supports_health is True
|
|
539
|
+
assert QdrantVectorStoreAdapter.supports_health is True
|
|
540
|
+
assert VectorStoreAdapter.supports_health is False # base stays honest
|
|
541
|
+
for adapter in (_redis_adapter(_FakeRedis(used=1, maxmemory=10)), _qdrant(_FakeQdrant())):
|
|
542
|
+
report = check_vectorstore(adapter)
|
|
543
|
+
assert report.status in (OK, WARN, ERROR, UNKNOWN) and report.backend
|
|
@@ -96,3 +96,38 @@ def test_pairs_above_with_zero_vectors(both_paths):
|
|
|
96
96
|
fast, slow = both_paths(lambda: ra.pairs_above(vecs, 0.5))
|
|
97
97
|
assert fast == slow
|
|
98
98
|
assert [(i, j) for i, j, _ in fast] == [(1, 2)] # the zero vector matches nothing
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
# --------------------------------------------------------------------------- #
|
|
102
|
+
# One cosine implementation, three call sites
|
|
103
|
+
# --------------------------------------------------------------------------- #
|
|
104
|
+
def test_the_drift_probe_uses_the_shared_cosine_and_keeps_its_none_contract():
|
|
105
|
+
# Same maths as retrieval_analysis (so it gets the numpy fast path), but None rather than
|
|
106
|
+
# 0.0 on a degenerate pair: "could not compare" must not read as "completely dissimilar",
|
|
107
|
+
# which is exactly what model drift looks like.
|
|
108
|
+
from compass.embedding_drift import cosine
|
|
109
|
+
|
|
110
|
+
assert cosine([1.0, 0.0, 0.0], [1.0, 0.0, 0.0]) == 1.0
|
|
111
|
+
assert cosine([1.0, 0.0], [0.0, 1.0]) == 0.0
|
|
112
|
+
assert cosine([0.0, 0.0], [1.0, 0.0]) is None # zero vector
|
|
113
|
+
assert cosine([1.0], [1.0, 2.0]) is None # length mismatch
|
|
114
|
+
assert cosine([], []) is None
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def test_the_chunk_validator_cosine_agrees_with_the_shared_one():
|
|
118
|
+
import struct
|
|
119
|
+
|
|
120
|
+
from compass.retrieval_analysis import _cosine
|
|
121
|
+
from compass.validators.redis_chunks import RedisChunksValidator
|
|
122
|
+
|
|
123
|
+
a, b = [1.0, 2.0, 3.0], [1.0, 2.0, 3.5]
|
|
124
|
+
packed = (struct.pack("3f", *a), struct.pack("3f", *b))
|
|
125
|
+
assert RedisChunksValidator._cosine(*packed) == pytest.approx(_cosine(a, b), abs=1e-6)
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def test_the_chunk_validator_cosine_returns_zero_for_a_degenerate_pair():
|
|
129
|
+
# This validator's convention is 0.0 = "no signal"; propagating None would break callers
|
|
130
|
+
# that compare against a threshold.
|
|
131
|
+
from compass.validators.redis_chunks import RedisChunksValidator
|
|
132
|
+
|
|
133
|
+
assert RedisChunksValidator._cosine(b"", b"") == 0.0
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/report/grounding_report.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/validators/api_upload.py
RENAMED
|
File without changes
|
|
File without changes
|
{compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/validators/base_sparql.py
RENAMED
|
File without changes
|
{compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/validators/cross_stage.py
RENAMED
|
File without changes
|
{compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/validators/graphdb_metadata.py
RENAMED
|
File without changes
|
{compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/validators/neptune_metadata.py
RENAMED
|
File without changes
|
|
File without changes
|
{compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/validators/retrieval_quality.py
RENAMED
|
File without changes
|
{compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/compass/validators/s3_storage.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/ingestion_validation/__init__.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_alias_disambiguation.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_discovery_parallel.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_retrieval_assertions.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_source_verification.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_universal_concerns.py
RENAMED
|
File without changes
|
|
File without changes
|
{compass_kb_validation-3.7.0 → compass_kb_validation-3.8.0}/tests/test_vectorstore_filters.py
RENAMED
|
File without changes
|