compass-kb-validation 3.6.0__tar.gz → 3.7.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {compass_kb_validation-3.6.0/compass_kb_validation.egg-info → compass_kb_validation-3.7.0}/PKG-INFO +1 -1
- {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/compass/__init__.py +1 -1
- {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/compass/ask.py +26 -2
- compass_kb_validation-3.7.0/compass/benchmark.py +345 -0
- {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/compass/citation.py +150 -11
- {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/compass/cli.py +384 -6
- {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/compass/config/__init__.py +35 -3
- compass_kb_validation-3.7.0/compass/diff.py +339 -0
- {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/compass/eval_enrich.py +61 -0
- {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/compass/eval_interop.py +32 -0
- {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/compass/grounding.py +27 -8
- compass_kb_validation-3.7.0/compass/health.py +280 -0
- compass_kb_validation-3.7.0/compass/incremental.py +243 -0
- {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/compass/models/__init__.py +13 -1
- {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/compass/pipeline/__init__.py +4 -0
- {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/compass/providers/graph.py +39 -0
- {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/compass/providers/vectorstore.py +73 -0
- compass_kb_validation-3.7.0/compass/quality_gate.py +275 -0
- {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/compass/report/__init__.py +4 -2
- {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/compass/report/audit.py +10 -93
- {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/compass/report/export.py +51 -1
- compass_kb_validation-3.7.0/compass/report/grounding_report.py +564 -0
- {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/compass/validators/redis_chunks.py +57 -3
- {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/compass/validators/s3_storage.py +5 -0
- {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0/compass_kb_validation.egg-info}/PKG-INFO +1 -1
- {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/compass_kb_validation.egg-info/SOURCES.txt +13 -0
- {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/discover_and_validate.py +68 -1
- {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/tests/test_alias_disambiguation.py +54 -0
- compass_kb_validation-3.7.0/tests/test_benchmark.py +221 -0
- compass_kb_validation-3.7.0/tests/test_calibrate_truth.py +41 -0
- {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/tests/test_citation_levels.py +62 -0
- compass_kb_validation-3.7.0/tests/test_diff.py +208 -0
- compass_kb_validation-3.7.0/tests/test_grounding_report.py +215 -0
- compass_kb_validation-3.7.0/tests/test_health.py +482 -0
- compass_kb_validation-3.7.0/tests/test_incremental.py +214 -0
- compass_kb_validation-3.7.0/tests/test_quality_gate.py +184 -0
- compass_kb_validation-3.7.0/tests/test_retrieval_fields.py +141 -0
- {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/tests/test_source_verification.py +3 -1
- compass_kb_validation-3.6.0/tests/test_benchmark.py +0 -43
- {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/LICENSE +0 -0
- {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/README.md +0 -0
- {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/compass/__main__.py +0 -0
- {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/compass/alias.py +0 -0
- {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/compass/baseline.py +0 -0
- {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/compass/chunk_coherence.py +0 -0
- {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/compass/config_lint.py +0 -0
- {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/compass/content_inspect.py +0 -0
- {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/compass/corpus.py +0 -0
- {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/compass/corpus_health.py +0 -0
- {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/compass/diagnostics.py +0 -0
- {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/compass/embedding_drift.py +0 -0
- {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/compass/evidence.py +0 -0
- {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/compass/init_wizard.py +0 -0
- {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/compass/live_dashboard.py +0 -0
- {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/compass/mcp_server.py +0 -0
- {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/compass/notifications.py +0 -0
- {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/compass/profiler.py +0 -0
- {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/compass/provenance.py +0 -0
- {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/compass/providers/__init__.py +0 -0
- {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/compass/providers/embedding.py +0 -0
- {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/compass/providers/source.py +0 -0
- {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/compass/providers/storage.py +0 -0
- {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/compass/remediation.py +0 -0
- {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/compass/report/_report_css.py +0 -0
- {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/compass/report/_report_js.py +0 -0
- {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/compass/report/analysis.py +0 -0
- {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/compass/report/shell.py +0 -0
- {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/compass/retrieval_analysis.py +0 -0
- {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/compass/text_probes.py +0 -0
- {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/compass/trend_tracker.py +0 -0
- {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/compass/utils/__init__.py +0 -0
- {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/compass/validators/__init__.py +0 -0
- {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/compass/validators/api_upload.py +0 -0
- {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/compass/validators/base.py +0 -0
- {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/compass/validators/base_sparql.py +0 -0
- {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/compass/validators/cross_stage.py +0 -0
- {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/compass/validators/graphdb_metadata.py +0 -0
- {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/compass/validators/neptune_metadata.py +0 -0
- {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/compass/validators/registry.py +0 -0
- {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/compass/validators/retrieval_quality.py +0 -0
- {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/compass_kb_validation.egg-info/dependency_links.txt +0 -0
- {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/compass_kb_validation.egg-info/entry_points.txt +0 -0
- {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/compass_kb_validation.egg-info/requires.txt +0 -0
- {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/compass_kb_validation.egg-info/top_level.txt +0 -0
- {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/ingestion_validation/__init__.py +0 -0
- {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/profile_kb.py +0 -0
- {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/pyproject.toml +0 -0
- {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/run_validation.py +0 -0
- {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/setup.cfg +0 -0
- {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/tests/test_alias.py +0 -0
- {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/tests/test_alt_adapters.py +0 -0
- {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/tests/test_analysis_report.py +0 -0
- {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/tests/test_api_upload.py +0 -0
- {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/tests/test_ask.py +0 -0
- {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/tests/test_baseline.py +0 -0
- {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/tests/test_check_catalog.py +0 -0
- {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/tests/test_citation.py +0 -0
- {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/tests/test_cli.py +0 -0
- {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/tests/test_compat_shim.py +0 -0
- {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/tests/test_config.py +0 -0
- {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/tests/test_config_lint.py +0 -0
- {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/tests/test_content_inspect.py +0 -0
- {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/tests/test_corpus.py +0 -0
- {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/tests/test_corpus_health.py +0 -0
- {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/tests/test_coverage.py +0 -0
- {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/tests/test_cross_stage.py +0 -0
- {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/tests/test_diagnostics.py +0 -0
- {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/tests/test_discover_cli.py +0 -0
- {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/tests/test_discovery_parallel.py +0 -0
- {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/tests/test_embedding_drift.py +0 -0
- {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/tests/test_eval_enrich.py +0 -0
- {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/tests/test_eval_interop.py +0 -0
- {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/tests/test_evidence.py +0 -0
- {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/tests/test_gate.py +0 -0
- {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/tests/test_graph_transport.py +0 -0
- {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/tests/test_graph_validators.py +0 -0
- {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/tests/test_grounding.py +0 -0
- {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/tests/test_init_wizard.py +0 -0
- {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/tests/test_manifest.py +0 -0
- {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/tests/test_mcp.py +0 -0
- {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/tests/test_migration.py +0 -0
- {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/tests/test_models.py +0 -0
- {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/tests/test_neptune_bugfixes.py +0 -0
- {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/tests/test_neptune_bugfixes2.py +0 -0
- {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/tests/test_notifications.py +0 -0
- {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/tests/test_pipeline.py +0 -0
- {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/tests/test_profiler.py +0 -0
- {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/tests/test_providers.py +0 -0
- {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/tests/test_redis_chunks.py +0 -0
- {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/tests/test_registry.py +0 -0
- {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/tests/test_remediation.py +0 -0
- {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/tests/test_reporting.py +0 -0
- {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/tests/test_retrieval_assertions.py +0 -0
- {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/tests/test_retrieval_quality.py +0 -0
- {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/tests/test_review_hardening.py +0 -0
- {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/tests/test_run_cli.py +0 -0
- {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/tests/test_sources.py +0 -0
- {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/tests/test_storage_validator.py +0 -0
- {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/tests/test_trends.py +0 -0
- {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/tests/test_universal_concerns.py +0 -0
- {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/tests/test_utils.py +0 -0
- {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/tests/test_vector_fastpath.py +0 -0
- {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/tests/test_vectorstore_filters.py +0 -0
|
@@ -40,11 +40,35 @@ def parse_filters(spec: str) -> dict:
|
|
|
40
40
|
return out
|
|
41
41
|
|
|
42
42
|
|
|
43
|
+
def _citation_fields(settings: Settings) -> list[str]:
|
|
44
|
+
"""Index fields the citation/grounding config already names.
|
|
45
|
+
|
|
46
|
+
A retrieved chunk that does not carry its document title and page cannot be matched to a
|
|
47
|
+
citation, so Level-1 grounding verification silently reports UNVERIFIABLE for every claim.
|
|
48
|
+
Rather than ask the user to restate those field names under a second config key, derive
|
|
49
|
+
them from the ``eval.gt_*`` / ``eval.citation_*`` settings that already declare them: if
|
|
50
|
+
the project configured grounding at all, retrieval returns what grounding needs.
|
|
51
|
+
Requesting a field the index lacks is harmless - RediSearch simply omits it.
|
|
52
|
+
"""
|
|
53
|
+
ev = settings.eval
|
|
54
|
+
fields = [
|
|
55
|
+
getattr(ev, "gt_chunk_doc_field", ""), # a chunk's source-document title
|
|
56
|
+
getattr(ev, "gt_chunk_page_field", ""), # a chunk's page, for page-level matching
|
|
57
|
+
getattr(ev, "gt_index_page_field", ""), # the index-side page field, if named differently
|
|
58
|
+
getattr(ev, "gt_doc_name_field", ""), # the document-name field used for L2 lookups
|
|
59
|
+
]
|
|
60
|
+
fields += list(getattr(ev, "citation_doc_fields", []) or [])
|
|
61
|
+
fields += list(getattr(ev, "citation_source_key_fields", []) or []) # Level-3 source tracing
|
|
62
|
+
return [f for f in fields if f]
|
|
63
|
+
|
|
64
|
+
|
|
43
65
|
def _return_fields(settings: Settings) -> list[str]:
|
|
44
66
|
rc = settings.redis
|
|
45
|
-
# Base identity/text
|
|
46
|
-
# harmless if the index lacks any), so
|
|
67
|
+
# Base identity/text, the fields grounding needs to match a citation, and the project's
|
|
68
|
+
# configured metadata fields (empty by default; harmless if the index lacks any), so
|
|
69
|
+
# results can feed export / comparison.
|
|
47
70
|
wanted = (["file_name", "text", rc.document_id_field, rc.positional_field, "s3_key"]
|
|
71
|
+
+ _citation_fields(settings)
|
|
48
72
|
+ list(settings.eval.metadata_fields))
|
|
49
73
|
seen: set[str] = set()
|
|
50
74
|
out: list[str] = []
|
|
@@ -0,0 +1,345 @@
|
|
|
1
|
+
"""Retrieval-strategy benchmarking: which way through this KB finds the right evidence?
|
|
2
|
+
|
|
3
|
+
`discover` proves the KB is intact. It says nothing about whether the retrieval PATH through
|
|
4
|
+
it is any good - so a team choosing between pure vector search, hybrid, or wider-then-narrower
|
|
5
|
+
has no evidence, guesses, deploys, and then blames the KB when answers are poor. Usually the
|
|
6
|
+
KB is fine and the retrieval configuration is wrong.
|
|
7
|
+
|
|
8
|
+
This runs an eval set's questions through two or more retrieval configurations against the
|
|
9
|
+
SAME validated KB and scores each one with standard information-retrieval metrics against the
|
|
10
|
+
ground-truth chunks the eval already carries. No LLM anywhere: a chunk is relevant because the
|
|
11
|
+
ground truth says so, not because a model judged it. Deterministic and repeatable - same eval
|
|
12
|
+
set and same KB give the same answer every time.
|
|
13
|
+
|
|
14
|
+
What it cannot do is invent ground truth. A case with no ground-truth chunks is EXCLUDED from
|
|
15
|
+
the metrics rather than scored zero: "we have nothing to compare against" and "retrieval found
|
|
16
|
+
nothing" are different findings, and averaging the first into the second would understate every
|
|
17
|
+
strategy equally and invisibly.
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
from __future__ import annotations
|
|
21
|
+
|
|
22
|
+
import math
|
|
23
|
+
from dataclasses import dataclass, field
|
|
24
|
+
from typing import Any, Optional
|
|
25
|
+
|
|
26
|
+
DEFAULT_TOP_K = 10
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
# --------------------------------------------------------------------------- #
|
|
30
|
+
# Information-retrieval metrics (binary relevance)
|
|
31
|
+
# --------------------------------------------------------------------------- #
|
|
32
|
+
def recall_at_k(retrieved: list[str], relevant: set, k: int) -> Optional[float]:
|
|
33
|
+
"""Share of the relevant chunks that appear in the top ``k``. None if nothing is relevant."""
|
|
34
|
+
if not relevant:
|
|
35
|
+
return None
|
|
36
|
+
found = len({r for r in retrieved[:k] if r in relevant})
|
|
37
|
+
return found / len(relevant)
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def precision_at_k(retrieved: list[str], relevant: set, k: int) -> Optional[float]:
|
|
41
|
+
"""Share of the top ``k`` that is relevant.
|
|
42
|
+
|
|
43
|
+
The denominator is k, not len(retrieved): a strategy that returns 3 results when asked for
|
|
44
|
+
10 has not earned the precision of one that returned 3 out of 3, and dividing by the
|
|
45
|
+
smaller number would flatter it for retrieving too little.
|
|
46
|
+
"""
|
|
47
|
+
if not relevant or k <= 0:
|
|
48
|
+
return None
|
|
49
|
+
found = len({r for r in retrieved[:k] if r in relevant})
|
|
50
|
+
return found / k
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def reciprocal_rank(retrieved: list[str], relevant: set) -> Optional[float]:
|
|
54
|
+
"""1 / rank of the first relevant chunk; 0.0 when none was retrieved."""
|
|
55
|
+
if not relevant:
|
|
56
|
+
return None
|
|
57
|
+
for i, chunk in enumerate(retrieved, 1):
|
|
58
|
+
if chunk in relevant:
|
|
59
|
+
return 1.0 / i
|
|
60
|
+
return 0.0
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def ndcg_at_k(retrieved: list[str], relevant: set, k: int) -> Optional[float]:
|
|
64
|
+
"""Normalised discounted cumulative gain with binary relevance.
|
|
65
|
+
|
|
66
|
+
Rewards putting relevant chunks EARLY, which recall alone cannot see: two strategies can
|
|
67
|
+
both find the evidence while one buries it at rank 9 where a top-3 context window will
|
|
68
|
+
never include it.
|
|
69
|
+
"""
|
|
70
|
+
if not relevant or k <= 0:
|
|
71
|
+
return None
|
|
72
|
+
dcg = sum(1.0 / math.log2(i + 1) for i, chunk in enumerate(retrieved[:k], 1)
|
|
73
|
+
if chunk in relevant)
|
|
74
|
+
ideal = sum(1.0 / math.log2(i + 1) for i in range(1, min(len(relevant), k) + 1))
|
|
75
|
+
return (dcg / ideal) if ideal else None
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def hit_rate(retrieved: list[str], relevant: set, k: int) -> Optional[float]:
|
|
79
|
+
"""1.0 when at least one relevant chunk is in the top ``k`` - did retrieval find anything?"""
|
|
80
|
+
if not relevant:
|
|
81
|
+
return None
|
|
82
|
+
return 1.0 if any(r in relevant for r in retrieved[:k]) else 0.0
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
METRICS = ("recall", "precision", "mrr", "ndcg", "hit_rate")
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def score_one(retrieved: list[str], relevant: set, k: int) -> dict:
|
|
89
|
+
"""Every metric for one query. Values are None when the query has no ground truth."""
|
|
90
|
+
return {
|
|
91
|
+
"recall": recall_at_k(retrieved, relevant, k),
|
|
92
|
+
"precision": precision_at_k(retrieved, relevant, k),
|
|
93
|
+
"mrr": reciprocal_rank(retrieved, relevant),
|
|
94
|
+
"ndcg": ndcg_at_k(retrieved, relevant, k),
|
|
95
|
+
"hit_rate": hit_rate(retrieved, relevant, k),
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def _mean(values: list) -> Optional[float]:
|
|
100
|
+
nums = [v for v in values if isinstance(v, (int, float)) and not isinstance(v, bool)]
|
|
101
|
+
return round(sum(nums) / len(nums), 4) if nums else None
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
# --------------------------------------------------------------------------- #
|
|
105
|
+
# Strategies
|
|
106
|
+
# --------------------------------------------------------------------------- #
|
|
107
|
+
@dataclass
|
|
108
|
+
class Strategy:
|
|
109
|
+
"""One retrieval configuration to compare. Everything comes from project YAML."""
|
|
110
|
+
|
|
111
|
+
name: str
|
|
112
|
+
type: str = "vector" # vector | hybrid
|
|
113
|
+
top_k: int = DEFAULT_TOP_K
|
|
114
|
+
filters: dict = field(default_factory=dict)
|
|
115
|
+
expand_context: int = 0 # include N neighbouring chunks around each hit
|
|
116
|
+
rerank_top_k: int = 0 # retrieve wider, then keep this many
|
|
117
|
+
extra: dict = field(default_factory=dict)
|
|
118
|
+
|
|
119
|
+
@classmethod
|
|
120
|
+
def from_config(cls, raw: dict) -> "Strategy":
|
|
121
|
+
known = {"name", "type", "top_k", "filters", "expand_context", "rerank_top_k"}
|
|
122
|
+
return cls(name=str(raw.get("name") or raw.get("type") or "strategy"),
|
|
123
|
+
type=str(raw.get("type") or "vector"),
|
|
124
|
+
top_k=int(raw.get("top_k") or DEFAULT_TOP_K),
|
|
125
|
+
filters=dict(raw.get("filters") or {}),
|
|
126
|
+
expand_context=int(raw.get("expand_context") or 0),
|
|
127
|
+
rerank_top_k=int(raw.get("rerank_top_k") or 0),
|
|
128
|
+
extra={k: v for k, v in raw.items() if k not in known})
|
|
129
|
+
|
|
130
|
+
@property
|
|
131
|
+
def effective_k(self) -> int:
|
|
132
|
+
"""The number of chunks a consumer actually sees after any narrowing step."""
|
|
133
|
+
return self.rerank_top_k or self.top_k
|
|
134
|
+
|
|
135
|
+
def unsupported(self) -> str:
|
|
136
|
+
"""Why this strategy cannot run here, or "" when it can.
|
|
137
|
+
|
|
138
|
+
A strategy naming a reranker is refused rather than silently run WITHOUT it: reporting
|
|
139
|
+
"vector-rerank: recall 0.63" for a run that never reranked would attribute the plain
|
|
140
|
+
vector score to a configuration nobody tested.
|
|
141
|
+
"""
|
|
142
|
+
if self.extra.get("reranker"):
|
|
143
|
+
return (f"'{self.extra['reranker']}' reranking is not built in - COMPASS ships no "
|
|
144
|
+
"cross-encoder provider, so this strategy cannot be measured here. Remove "
|
|
145
|
+
"the reranker to benchmark the retrieval it wraps.")
|
|
146
|
+
if self.type not in ("vector", "hybrid"):
|
|
147
|
+
return f"unknown strategy type '{self.type}' (expected 'vector' or 'hybrid')"
|
|
148
|
+
return ""
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def chunk_identity(chunk: dict, doc_field: str, page_field: str) -> str:
|
|
152
|
+
"""A stable identity for matching a retrieved chunk to a ground-truth one.
|
|
153
|
+
|
|
154
|
+
Document + page, because that is the granularity ground truth is written at and the only
|
|
155
|
+
identifier both sides reliably share - a store's internal chunk key never appears in an
|
|
156
|
+
eval file.
|
|
157
|
+
"""
|
|
158
|
+
doc = str(chunk.get(doc_field) or chunk.get("file_name") or chunk.get("document_title") or "")
|
|
159
|
+
page = str(chunk.get(page_field) or chunk.get("page_start") or "")
|
|
160
|
+
return f"{doc.strip().lower()}#{page.strip()}" if doc else ""
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def ground_truth_ids(case: dict, cfg: Any) -> set:
|
|
164
|
+
"""The chunks a case's ground truth says are relevant."""
|
|
165
|
+
from compass.eval_interop import get_path
|
|
166
|
+
|
|
167
|
+
doc_field = getattr(cfg, "gt_chunk_doc_field", "document_title")
|
|
168
|
+
page_field = getattr(cfg, "gt_chunk_page_field", "page_start")
|
|
169
|
+
found = get_path(case, getattr(cfg, "gt_chunks_path", "context.chunks"))
|
|
170
|
+
chunks: list = found if isinstance(found, list) else []
|
|
171
|
+
if not chunks and isinstance(case.get("chunks"), list):
|
|
172
|
+
chunks = case["chunks"]
|
|
173
|
+
out = {chunk_identity(c, doc_field, page_field) for c in chunks if isinstance(c, dict)}
|
|
174
|
+
return {i for i in out if i}
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
def run_strategy(settings: Any, question: str, strategy: Strategy, *, embedder: Any = None,
|
|
178
|
+
vectorstore: Any = None) -> list[str]:
|
|
179
|
+
"""Retrieve for one question under one strategy; return chunk identities in rank order."""
|
|
180
|
+
from compass.ask import ask_kb
|
|
181
|
+
|
|
182
|
+
payload = ask_kb(settings, question, top_k=max(strategy.top_k, strategy.effective_k),
|
|
183
|
+
filters=strategy.filters or None, embedder=embedder,
|
|
184
|
+
vectorstore=vectorstore)
|
|
185
|
+
doc_field = getattr(settings.eval, "gt_chunk_doc_field", "document_title")
|
|
186
|
+
page_field = getattr(settings.eval, "gt_chunk_page_field", "page_start")
|
|
187
|
+
ids: list[str] = []
|
|
188
|
+
for result in payload.get("results", []):
|
|
189
|
+
merged = {**(result.get("meta") or {}), **result}
|
|
190
|
+
identity = chunk_identity(merged, doc_field, page_field)
|
|
191
|
+
if identity and identity not in ids:
|
|
192
|
+
ids.append(identity)
|
|
193
|
+
return ids[: strategy.effective_k] if strategy.rerank_top_k else ids
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
# --------------------------------------------------------------------------- #
|
|
197
|
+
# Benchmark run
|
|
198
|
+
# --------------------------------------------------------------------------- #
|
|
199
|
+
def benchmark(settings: Any, cases: list[dict], strategies: list[Strategy], *,
|
|
200
|
+
retrieve: Any = None) -> dict:
|
|
201
|
+
"""Score every strategy over every case. ``retrieve`` is injectable for testing.
|
|
202
|
+
|
|
203
|
+
Cases with no ground truth are excluded and counted separately - scoring them zero would
|
|
204
|
+
penalise every strategy identically for a gap in the EVAL SET, and hide that the number is
|
|
205
|
+
based on fewer queries than the reader assumes.
|
|
206
|
+
"""
|
|
207
|
+
from compass.eval_interop import get_path
|
|
208
|
+
|
|
209
|
+
q_path = getattr(settings.eval, "gt_question_path", "question")
|
|
210
|
+
runner = retrieve or (lambda q, s: run_strategy(settings, q, s))
|
|
211
|
+
|
|
212
|
+
usable, skipped = [], []
|
|
213
|
+
for case in cases:
|
|
214
|
+
question = get_path(case, q_path) or case.get("input") or case.get("question")
|
|
215
|
+
relevant = ground_truth_ids(case, settings.eval)
|
|
216
|
+
if not question or not relevant:
|
|
217
|
+
skipped.append(str(case.get("case_id") or case.get("test_id") or "?"))
|
|
218
|
+
continue
|
|
219
|
+
usable.append((case, str(question), relevant))
|
|
220
|
+
|
|
221
|
+
per_strategy: list[dict] = []
|
|
222
|
+
per_query: dict[str, dict] = {}
|
|
223
|
+
for strategy in strategies:
|
|
224
|
+
blocked = strategy.unsupported()
|
|
225
|
+
if blocked:
|
|
226
|
+
per_strategy.append({"strategy": strategy.name, "skipped": True, "reason": blocked})
|
|
227
|
+
continue
|
|
228
|
+
scores: dict[str, list] = {m: [] for m in METRICS}
|
|
229
|
+
for case, question, relevant in usable:
|
|
230
|
+
cid = str(case.get("case_id") or case.get("test_id") or question[:40])
|
|
231
|
+
retrieved = runner(question, strategy)
|
|
232
|
+
result = score_one(retrieved, relevant, strategy.effective_k)
|
|
233
|
+
for metric, value in result.items():
|
|
234
|
+
scores[metric].append(value)
|
|
235
|
+
per_query.setdefault(cid, {"question": question,
|
|
236
|
+
"ground_truth": len(relevant)})[strategy.name] = {
|
|
237
|
+
"found": len([r for r in retrieved[:strategy.effective_k] if r in relevant]),
|
|
238
|
+
"recall": result["recall"], "mrr": result["mrr"]}
|
|
239
|
+
per_strategy.append({
|
|
240
|
+
"strategy": strategy.name, "skipped": False, "top_k": strategy.effective_k,
|
|
241
|
+
"queries": len(usable),
|
|
242
|
+
**{metric: _mean(values) for metric, values in scores.items()},
|
|
243
|
+
})
|
|
244
|
+
|
|
245
|
+
scored = [s for s in per_strategy if not s.get("skipped")]
|
|
246
|
+
winner = max(scored, key=lambda s: (s.get("ndcg") or 0, s.get("recall") or 0), default=None)
|
|
247
|
+
return {
|
|
248
|
+
"project": getattr(settings, "project_name", "") or "",
|
|
249
|
+
"queries": len(usable),
|
|
250
|
+
"excluded_cases": skipped,
|
|
251
|
+
"strategies": per_strategy,
|
|
252
|
+
"winner": winner["strategy"] if winner else "",
|
|
253
|
+
"per_query": per_query,
|
|
254
|
+
"method": ("binary-relevance IR metrics against the eval's own ground-truth chunks, "
|
|
255
|
+
"matched on document + page. No LLM: relevance comes from the ground truth, "
|
|
256
|
+
"not from a model's judgement."),
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
|
|
260
|
+
def worst_queries(report: dict, limit: int = 5) -> list[dict]:
|
|
261
|
+
"""Queries where the best and worst strategies disagree most - where the choice matters."""
|
|
262
|
+
rows = []
|
|
263
|
+
for cid, data in report.get("per_query", {}).items():
|
|
264
|
+
recalls: list[tuple[str, float]] = [
|
|
265
|
+
(str(name), float(v["recall"])) for name, v in data.items()
|
|
266
|
+
if isinstance(v, dict) and isinstance(v.get("recall"), (int, float))]
|
|
267
|
+
if len(recalls) < 2:
|
|
268
|
+
continue
|
|
269
|
+
best = max(recalls, key=lambda t: t[1])
|
|
270
|
+
worst = min(recalls, key=lambda t: t[1])
|
|
271
|
+
if best[1] > worst[1]:
|
|
272
|
+
rows.append({"case_id": cid, "question": data.get("question", ""),
|
|
273
|
+
"best": best[0], "best_recall": best[1],
|
|
274
|
+
"worst": worst[0], "worst_recall": worst[1],
|
|
275
|
+
"spread": round(best[1] - worst[1], 4)})
|
|
276
|
+
return sorted(rows, key=lambda r: -r["spread"])[:limit]
|
|
277
|
+
|
|
278
|
+
|
|
279
|
+
def format_report(report: dict) -> str:
|
|
280
|
+
"""Console table plus the queries where the strategy choice actually matters."""
|
|
281
|
+
L = ["=" * 72,
|
|
282
|
+
f" COMPASS benchmark - {report.get('project') or 'KB'} "
|
|
283
|
+
f"({report['queries']} queries, {len(report['strategies'])} strategies)",
|
|
284
|
+
"=" * 72, ""]
|
|
285
|
+
if not report["queries"]:
|
|
286
|
+
L.append(" No case carried both a question and ground-truth chunks, so nothing could "
|
|
287
|
+
"be measured.")
|
|
288
|
+
L.append(" Check eval.gt_question_path and eval.gt_chunks_path.")
|
|
289
|
+
L.append("=" * 72)
|
|
290
|
+
return "\n".join(L)
|
|
291
|
+
|
|
292
|
+
head = f" {'Strategy':<20}{'Recall':>9}{'Precision':>11}{'MRR':>8}{'NDCG':>8}{'Hit rate':>10}"
|
|
293
|
+
L += [head, " " + "-" * (len(head) - 2)]
|
|
294
|
+
for s in report["strategies"]:
|
|
295
|
+
if s.get("skipped"):
|
|
296
|
+
L.append(f" {s['strategy']:<20} SKIPPED - {s['reason']}")
|
|
297
|
+
continue
|
|
298
|
+
L.append(f" {s['strategy']:<20}"
|
|
299
|
+
+ "".join(f"{_fmt(s.get(m)):>{w}}" for m, w in
|
|
300
|
+
(("recall", 9), ("precision", 11), ("mrr", 8), ("ndcg", 8),
|
|
301
|
+
("hit_rate", 10))))
|
|
302
|
+
L.append("")
|
|
303
|
+
|
|
304
|
+
scored = [s for s in report["strategies"] if not s.get("skipped")]
|
|
305
|
+
if report.get("winner") and len(scored) > 1:
|
|
306
|
+
win = next(s for s in scored if s["strategy"] == report["winner"])
|
|
307
|
+
base = min(scored, key=lambda s: (s.get("ndcg") or 0))
|
|
308
|
+
L.append(f" WINNER: {win['strategy']}")
|
|
309
|
+
for metric, label in (("recall", "recall"), ("mrr", "MRR"), ("ndcg", "NDCG")):
|
|
310
|
+
a, b = win.get(metric), base.get(metric)
|
|
311
|
+
if isinstance(a, (int, float)) and isinstance(b, (int, float)) and b:
|
|
312
|
+
L.append(f" {(a - b) / b:+.0%} {label} vs {base['strategy']}")
|
|
313
|
+
L.append("")
|
|
314
|
+
|
|
315
|
+
worst = worst_queries(report)
|
|
316
|
+
if worst:
|
|
317
|
+
L.append(" WHERE THE CHOICE MATTERS (largest disagreement between strategies):")
|
|
318
|
+
for row in worst:
|
|
319
|
+
L.append(f" {row['case_id']}: \"{row['question'][:58]}\"")
|
|
320
|
+
L.append(f" {row['best']}: recall {row['best_recall']:.2f} "
|
|
321
|
+
f"{row['worst']}: recall {row['worst_recall']:.2f}")
|
|
322
|
+
L.append("")
|
|
323
|
+
if report["excluded_cases"]:
|
|
324
|
+
L.append(f" {len(report['excluded_cases'])} case(s) excluded - no question or no "
|
|
325
|
+
"ground-truth chunks to measure against, so they are not scored as misses.")
|
|
326
|
+
L.append(" Method: " + report["method"])
|
|
327
|
+
L.append("=" * 72)
|
|
328
|
+
return "\n".join(L)
|
|
329
|
+
|
|
330
|
+
|
|
331
|
+
def _fmt(value: Any) -> str:
|
|
332
|
+
return f"{value:.2f}" if isinstance(value, (int, float)) else "-"
|
|
333
|
+
|
|
334
|
+
|
|
335
|
+
def strategies_from_settings(settings: Any) -> list[Strategy]:
|
|
336
|
+
"""Read ``benchmark.strategies`` from project config."""
|
|
337
|
+
raw = getattr(getattr(settings, "benchmark", None), "strategies", None)
|
|
338
|
+
if raw is None:
|
|
339
|
+
raw = (getattr(settings, "extra", {}) or {}).get("benchmark", {}).get("strategies", [])
|
|
340
|
+
return [Strategy.from_config(s) for s in (raw or []) if isinstance(s, dict)]
|
|
341
|
+
|
|
342
|
+
|
|
343
|
+
__all__ = ["Strategy", "benchmark", "format_report", "score_one", "recall_at_k",
|
|
344
|
+
"precision_at_k", "reciprocal_rank", "ndcg_at_k", "hit_rate", "ground_truth_ids",
|
|
345
|
+
"chunk_identity", "worst_queries", "strategies_from_settings", "run_strategy"]
|
|
@@ -36,8 +36,13 @@ from typing import Any, Optional
|
|
|
36
36
|
|
|
37
37
|
# Default citation patterns. `(DOC, descriptor, p. N)` / `pp. N-M`, and a bare **file.ext**.
|
|
38
38
|
# Extra domain patterns can be added via config; these always apply.
|
|
39
|
+
# The document token is capped generously (not at 30 chars, which silently DROPPED every
|
|
40
|
+
# citation naming a real-world document title - "(Quarterly Investment Perspective, Q3, p. 4)"
|
|
41
|
+
# is 32 chars and produced zero citations, not even an unverifiable one). The character class
|
|
42
|
+
# already excludes "," and ")", and the whole pattern still requires the
|
|
43
|
+
# `, <descriptor>, p. <n>` structure, so a longer cap cannot run away through prose.
|
|
39
44
|
_PAREN_CITE = re.compile(
|
|
40
|
-
r"\(\s*([A-Z][A-Za-z0-9&/ .-]{0,
|
|
45
|
+
r"\(\s*([A-Z][A-Za-z0-9&/ .-]{0,120}?)\s*,\s*([^,)]{1,40}?)\s*,\s*pp?\.\s*(\d+)(?:\s*[-–]\s*(\d+))?\s*\)")
|
|
41
46
|
_FILE_CITE = re.compile(r"\*\*\s*([^*]+?\.(?:pdf|docx|pptx|txt|md))\s*\*\*", re.IGNORECASE)
|
|
42
47
|
|
|
43
48
|
# One tokenizer drives BOTH extraction and matching, so a claim number and a source number
|
|
@@ -335,6 +340,14 @@ class CitationVerdict:
|
|
|
335
340
|
# --- Level-3 plumbing -------------------------------------------------------------- #
|
|
336
341
|
source_key: str = "" # storage key of the cited document, resolved from its KB chunks
|
|
337
342
|
alias_resolved: str = "" # KB document name an auto-derived alias resolved the citation to
|
|
343
|
+
# A snippet of the text the claim was actually checked against, so a report can show the
|
|
344
|
+
# evidence beside the claim instead of asserting a verdict the reader cannot inspect.
|
|
345
|
+
evidence: str = ""
|
|
346
|
+
evidence_source: str = "" # eval_chunk | kb_page | source_document
|
|
347
|
+
#: How this citation's document token was resolved, step by step. Populated so a user can
|
|
348
|
+
#: see WHY an alias did or did not resolve instead of inferring it from a bare verdict -
|
|
349
|
+
#: "I added the alias and it is still UNVERIFIABLE" is otherwise undiagnosable.
|
|
350
|
+
resolution: list = field(default_factory=list)
|
|
338
351
|
|
|
339
352
|
|
|
340
353
|
_MONTHS = set("jan feb mar apr may jun jul aug sep sept oct nov dec january february march april "
|
|
@@ -388,6 +401,7 @@ def verify_level1(citations: list[Citation], eval_chunks: list[dict], cfg: Any)
|
|
|
388
401
|
verdicts.append(v)
|
|
389
402
|
continue
|
|
390
403
|
v.chunk_matched = True
|
|
404
|
+
v.evidence, v.evidence_source = _snippet(_raw_text(chunk, text_field)), "eval_chunk"
|
|
391
405
|
chunk_nums = _iter_nums(_haystack(chunk, text_field))
|
|
392
406
|
for cv in cit.values:
|
|
393
407
|
if cv.matches(chunk_nums):
|
|
@@ -421,6 +435,19 @@ def verify_level1(citations: list[Citation], eval_chunks: list[dict], cfg: Any)
|
|
|
421
435
|
return verdicts
|
|
422
436
|
|
|
423
437
|
|
|
438
|
+
def _raw_text(chunk: dict, text_field: str) -> str:
|
|
439
|
+
"""A chunk's text with its ORIGINAL casing - `_haystack` lowercases for matching, which is
|
|
440
|
+
right for comparison and wrong for anything a human reads."""
|
|
441
|
+
t = str(chunk.get(text_field, "") or "")
|
|
442
|
+
return t or " ".join(str(v) for v in chunk.values() if isinstance(v, str))
|
|
443
|
+
|
|
444
|
+
|
|
445
|
+
def _snippet(text: str, width: int = 320) -> str:
|
|
446
|
+
"""A readable slice of the text a claim was checked against (whitespace normalised)."""
|
|
447
|
+
t = re.sub(r"\s+", " ", (text or "")).strip()
|
|
448
|
+
return t if len(t) <= width else t[:width].rsplit(" ", 1)[0] + " ..."
|
|
449
|
+
|
|
450
|
+
|
|
424
451
|
def _squash(s: str) -> str:
|
|
425
452
|
"""Lowercase and drop whitespace so ``350bps`` matches a source's ``350 bps``."""
|
|
426
453
|
return re.sub(r"\s+", "", (s or "").lower())
|
|
@@ -504,6 +531,93 @@ def _page_match(chunk_page: str, cited: str, tol: int) -> bool:
|
|
|
504
531
|
return False
|
|
505
532
|
|
|
506
533
|
|
|
534
|
+
#: Share of a claim's distinctive content words that must appear in the cited source text for
|
|
535
|
+
#: the claim to count as lexically supported. Deliberately short of 1.0 - normal wording
|
|
536
|
+
#: differences ("rose"/"increased") should not sink an otherwise well-supported statement.
|
|
537
|
+
TERM_SUPPORT_THRESHOLD = 0.6
|
|
538
|
+
|
|
539
|
+
# Common words carry no evidence, so they neither help nor hurt the support ratio.
|
|
540
|
+
_COMMON = {
|
|
541
|
+
"about", "above", "after", "again", "against", "also", "another", "because", "been",
|
|
542
|
+
"before", "being", "below", "between", "both", "cannot", "could", "does", "doing", "down",
|
|
543
|
+
"during", "each", "either", "from", "further", "have", "having", "here", "into", "itself",
|
|
544
|
+
"just", "more", "most", "much", "must", "only", "other", "over", "same", "should", "since",
|
|
545
|
+
"some", "such", "than", "that", "their", "them", "then", "there", "these", "they", "this",
|
|
546
|
+
"those", "through", "under", "until", "very", "were", "what", "when", "where", "which",
|
|
547
|
+
"while", "with", "would", "your", "will", "shall", "may", "might", "the", "and", "for",
|
|
548
|
+
}
|
|
549
|
+
|
|
550
|
+
|
|
551
|
+
def _content_words(text: str) -> list[str]:
|
|
552
|
+
"""Distinctive words of a claim - long enough and common enough to carry evidence."""
|
|
553
|
+
words = re.findall(r"[A-Za-z][A-Za-z0-9'-]{3,}", text or "")
|
|
554
|
+
seen: set[str] = set()
|
|
555
|
+
out: list[str] = []
|
|
556
|
+
for w in words:
|
|
557
|
+
low = w.lower()
|
|
558
|
+
if low not in _COMMON and low not in seen:
|
|
559
|
+
seen.add(low)
|
|
560
|
+
out.append(w)
|
|
561
|
+
return out
|
|
562
|
+
|
|
563
|
+
|
|
564
|
+
def lexical_support(claim: str, source_text: str) -> tuple[float, list[str]]:
|
|
565
|
+
"""(share of the claim's distinctive words present in ``source_text``, the absent ones).
|
|
566
|
+
|
|
567
|
+
Deliberately a presence ratio, not an entailment judgement: it says how much of the
|
|
568
|
+
claim's own language the cited text actually contains. Returns (0.0, []) when the claim
|
|
569
|
+
has nothing distinctive to check.
|
|
570
|
+
"""
|
|
571
|
+
words = _content_words(claim)
|
|
572
|
+
if not words:
|
|
573
|
+
return 0.0, []
|
|
574
|
+
hay = _squash(source_text)
|
|
575
|
+
missing = [w for w in words if _squash(w) not in hay]
|
|
576
|
+
return (len(words) - len(missing)) / len(words), missing
|
|
577
|
+
|
|
578
|
+
|
|
579
|
+
def _term_verify_l2(v: "CitationVerdict", hits: list[dict], page_field: str, text_field: str,
|
|
580
|
+
page_tol: int) -> str:
|
|
581
|
+
"""Verify a non-quantitative claim against the cited page's text from the KB.
|
|
582
|
+
|
|
583
|
+
Returns the Level-2 verdict: ``TERM_VERIFIED_L2`` when the claim's distinctive language is
|
|
584
|
+
present at the cited location, else ``SOURCE_PRESENT`` (the document is there, the wording
|
|
585
|
+
is not - reported unverifiable, never a fault).
|
|
586
|
+
"""
|
|
587
|
+
# `v.negated` is only set for numeric claims, so check the claim text directly: a negated
|
|
588
|
+
# prose claim shares nearly all its words with text asserting the OPPOSITE ("did not signal
|
|
589
|
+
# a constructive outlook" vs "signalled a constructive outlook"), which would score as
|
|
590
|
+
# supported. Lexical presence cannot tell those apart, so it must not claim to.
|
|
591
|
+
if v.negated or _NEG.search(v.claim or ""):
|
|
592
|
+
return "SOURCE_PRESENT"
|
|
593
|
+
on_page = [h for h in hits if _page_match(h.get(page_field, ""), v.page, page_tol)]
|
|
594
|
+
text = " ".join(_haystack(h, text_field) for h in (on_page or hits))
|
|
595
|
+
if not text.strip():
|
|
596
|
+
return "SOURCE_PRESENT"
|
|
597
|
+
if not v.evidence:
|
|
598
|
+
# Original casing for display; `text` above is the lowercased matching buffer.
|
|
599
|
+
v.evidence = _snippet(" ".join(_raw_text(h, text_field) for h in (on_page or hits)))
|
|
600
|
+
v.evidence_source = "kb_page"
|
|
601
|
+
# Distinctive entities (quoted strings, multi-word proper nouns) are the strongest lexical
|
|
602
|
+
# evidence; fall back to the claim's content words when it names none.
|
|
603
|
+
ents = _extract_entities(v.claim)
|
|
604
|
+
if ents:
|
|
605
|
+
hay = _squash(text)
|
|
606
|
+
v.entity_missing = [e for e in ents if _squash(e) not in hay]
|
|
607
|
+
v.entities_checked = True
|
|
608
|
+
if not v.entity_missing:
|
|
609
|
+
return "TERM_VERIFIED_L2"
|
|
610
|
+
ratio, missing = lexical_support(v.claim, text)
|
|
611
|
+
if ratio >= TERM_SUPPORT_THRESHOLD:
|
|
612
|
+
v.entities_checked = True
|
|
613
|
+
v.support_pct = round(100.0 * ratio, 1)
|
|
614
|
+
return "TERM_VERIFIED_L2"
|
|
615
|
+
if missing:
|
|
616
|
+
v.entities_checked = True
|
|
617
|
+
v.entity_missing = (v.entity_missing or []) + missing[:5]
|
|
618
|
+
return "SOURCE_PRESENT"
|
|
619
|
+
|
|
620
|
+
|
|
507
621
|
def _source_key_of(hits: list[dict], key_fields: list[str]) -> str:
|
|
508
622
|
"""The storage key of the cited document, from the first KB chunk that records one."""
|
|
509
623
|
for h in hits:
|
|
@@ -535,17 +649,28 @@ def verify_level2(verdicts: list[CitationVerdict], adapter: Any, cfg: Any, l1_ra
|
|
|
535
649
|
for v in verdicts:
|
|
536
650
|
if v.l1_verdict == "GROUNDED":
|
|
537
651
|
continue
|
|
538
|
-
|
|
652
|
+
resolved = _resolve_doc(v.doc, aliases)
|
|
653
|
+
v.resolution.append(f"parser token: '{v.doc}'")
|
|
654
|
+
if resolved != v.doc:
|
|
655
|
+
v.resolution.append(f"configured alias -> '{resolved}'")
|
|
656
|
+
else:
|
|
657
|
+
v.resolution.append("configured alias: no entry matched")
|
|
658
|
+
hits = _search_doc(adapter, fields, resolved, cap)
|
|
659
|
+
v.resolution.append(
|
|
660
|
+
f"KB search on {fields}: {len(hits)} hit(s)" if hits
|
|
661
|
+
else f"KB search on {fields}: no hit for '{resolved}'")
|
|
539
662
|
if not hits and alias_map is not None:
|
|
540
663
|
# Auto-derived alias: the citation's short token may be an initialism of a real KB
|
|
541
664
|
# document name. Only a UNIQUE match resolves - an ambiguous key is left unresolved
|
|
542
665
|
# so the verdict stays honestly ambiguous instead of picking a document.
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
666
|
+
auto, why = alias_map.resolve(v.doc)
|
|
667
|
+
v.resolution.append(f"auto-alias: {why}")
|
|
668
|
+
if auto:
|
|
669
|
+
hits = _search_doc(adapter, fields, auto, cap)
|
|
670
|
+
v.resolution.append(f"KB search on '{auto}': {len(hits)} hit(s)")
|
|
546
671
|
if hits:
|
|
547
|
-
v.alias_resolved =
|
|
548
|
-
v.note = f"resolved to '{
|
|
672
|
+
v.alias_resolved = auto
|
|
673
|
+
v.note = f"resolved to '{auto}' ({why})"
|
|
549
674
|
if hits and key_fields:
|
|
550
675
|
v.source_key = _source_key_of(hits, key_fields)
|
|
551
676
|
if not hits:
|
|
@@ -568,10 +693,19 @@ def verify_level2(verdicts: list[CitationVerdict], adapter: Any, cfg: Any, l1_ra
|
|
|
568
693
|
continue
|
|
569
694
|
values = v._missing if (l1_ran and v._missing) else v.values
|
|
570
695
|
if not values:
|
|
571
|
-
|
|
696
|
+
# No quantitative claim - but the KB just handed us the cited page's actual text,
|
|
697
|
+
# so check the claim's OWN distinctive language against it instead of declaring it
|
|
698
|
+
# uncheckable. Reporting "unverifiable" while holding the source text is the thing
|
|
699
|
+
# that made a live-KB audit look like it verified nothing. A hit is a positive
|
|
700
|
+
# (weaker-tier) signal; a miss stays unverifiable and is NEVER a fault, because
|
|
701
|
+
# lexical absence can be paraphrase.
|
|
702
|
+
v.l2_verdict = _term_verify_l2(v, hits, page_field, text_field, page_tol)
|
|
572
703
|
continue
|
|
573
704
|
capped = len(hits) >= cap
|
|
574
705
|
on_page = [h for h in hits if _page_match(h.get(page_field, ""), v.page, page_tol)]
|
|
706
|
+
if not v.evidence:
|
|
707
|
+
v.evidence = _snippet(" ".join(_raw_text(h, text_field) for h in (on_page or hits)))
|
|
708
|
+
v.evidence_source = "kb_page"
|
|
575
709
|
page_nums = _iter_nums(" ".join(_haystack(h, text_field) for h in on_page))
|
|
576
710
|
doc_nums = _iter_nums(" ".join(_haystack(h, text_field) for h in hits))
|
|
577
711
|
in_page = [cv for cv in values if cv.matches(page_nums)]
|
|
@@ -709,8 +843,10 @@ def attribute(verdicts: list[CitationVerdict], l1_ran: bool) -> None:
|
|
|
709
843
|
key = _L3_ATTRIB[v.l3_verdict] # the original source is decisive - it wins
|
|
710
844
|
elif l2 == "FABRICATED_SOURCE":
|
|
711
845
|
key = "fabricated_source" # cited a document that isn't in the KB
|
|
712
|
-
elif l1 == "TERM_SUPPORTED":
|
|
713
|
-
|
|
846
|
+
elif l1 == "TERM_SUPPORTED" or l2 == "TERM_VERIFIED_L2":
|
|
847
|
+
# Distinctive language present at the cited location - weaker than a numeric match,
|
|
848
|
+
# but real evidence, and far better than calling a verifiable claim uncheckable.
|
|
849
|
+
key = "term_supported"
|
|
714
850
|
elif l2 in _L2_UNVERIFIABLE:
|
|
715
851
|
key = "unverifiable" # source unresolved / too large / no numeric claim
|
|
716
852
|
elif l1 == "UNVERIFIABLE" and not l2:
|
|
@@ -753,7 +889,10 @@ def grade(verdicts: list[CitationVerdict], l1_ran: bool) -> None:
|
|
|
753
889
|
for v in verdicts:
|
|
754
890
|
total = len(v.values)
|
|
755
891
|
if not total:
|
|
756
|
-
|
|
892
|
+
# Term support can be established at either level (the eval's chunk, or the cited
|
|
893
|
+
# page fetched from the KB), so key off the attribution rather than Level 1 alone.
|
|
894
|
+
v.tier = ("TERM_SUPPORTED" if v.attribution == "term_supported"
|
|
895
|
+
else "UNVERIFIABLE")
|
|
757
896
|
continue
|
|
758
897
|
if v.attribution == "none":
|
|
759
898
|
v.tier, v.support_pct = "SOUND", 100.0
|