compass-kb-validation 3.6.0__tar.gz → 3.7.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (143) hide show
  1. {compass_kb_validation-3.6.0/compass_kb_validation.egg-info → compass_kb_validation-3.7.0}/PKG-INFO +1 -1
  2. {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/compass/__init__.py +1 -1
  3. {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/compass/ask.py +26 -2
  4. compass_kb_validation-3.7.0/compass/benchmark.py +345 -0
  5. {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/compass/citation.py +150 -11
  6. {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/compass/cli.py +384 -6
  7. {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/compass/config/__init__.py +35 -3
  8. compass_kb_validation-3.7.0/compass/diff.py +339 -0
  9. {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/compass/eval_enrich.py +61 -0
  10. {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/compass/eval_interop.py +32 -0
  11. {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/compass/grounding.py +27 -8
  12. compass_kb_validation-3.7.0/compass/health.py +280 -0
  13. compass_kb_validation-3.7.0/compass/incremental.py +243 -0
  14. {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/compass/models/__init__.py +13 -1
  15. {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/compass/pipeline/__init__.py +4 -0
  16. {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/compass/providers/graph.py +39 -0
  17. {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/compass/providers/vectorstore.py +73 -0
  18. compass_kb_validation-3.7.0/compass/quality_gate.py +275 -0
  19. {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/compass/report/__init__.py +4 -2
  20. {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/compass/report/audit.py +10 -93
  21. {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/compass/report/export.py +51 -1
  22. compass_kb_validation-3.7.0/compass/report/grounding_report.py +564 -0
  23. {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/compass/validators/redis_chunks.py +57 -3
  24. {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/compass/validators/s3_storage.py +5 -0
  25. {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0/compass_kb_validation.egg-info}/PKG-INFO +1 -1
  26. {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/compass_kb_validation.egg-info/SOURCES.txt +13 -0
  27. {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/discover_and_validate.py +68 -1
  28. {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/tests/test_alias_disambiguation.py +54 -0
  29. compass_kb_validation-3.7.0/tests/test_benchmark.py +221 -0
  30. compass_kb_validation-3.7.0/tests/test_calibrate_truth.py +41 -0
  31. {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/tests/test_citation_levels.py +62 -0
  32. compass_kb_validation-3.7.0/tests/test_diff.py +208 -0
  33. compass_kb_validation-3.7.0/tests/test_grounding_report.py +215 -0
  34. compass_kb_validation-3.7.0/tests/test_health.py +482 -0
  35. compass_kb_validation-3.7.0/tests/test_incremental.py +214 -0
  36. compass_kb_validation-3.7.0/tests/test_quality_gate.py +184 -0
  37. compass_kb_validation-3.7.0/tests/test_retrieval_fields.py +141 -0
  38. {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/tests/test_source_verification.py +3 -1
  39. compass_kb_validation-3.6.0/tests/test_benchmark.py +0 -43
  40. {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/LICENSE +0 -0
  41. {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/README.md +0 -0
  42. {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/compass/__main__.py +0 -0
  43. {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/compass/alias.py +0 -0
  44. {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/compass/baseline.py +0 -0
  45. {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/compass/chunk_coherence.py +0 -0
  46. {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/compass/config_lint.py +0 -0
  47. {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/compass/content_inspect.py +0 -0
  48. {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/compass/corpus.py +0 -0
  49. {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/compass/corpus_health.py +0 -0
  50. {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/compass/diagnostics.py +0 -0
  51. {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/compass/embedding_drift.py +0 -0
  52. {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/compass/evidence.py +0 -0
  53. {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/compass/init_wizard.py +0 -0
  54. {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/compass/live_dashboard.py +0 -0
  55. {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/compass/mcp_server.py +0 -0
  56. {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/compass/notifications.py +0 -0
  57. {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/compass/profiler.py +0 -0
  58. {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/compass/provenance.py +0 -0
  59. {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/compass/providers/__init__.py +0 -0
  60. {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/compass/providers/embedding.py +0 -0
  61. {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/compass/providers/source.py +0 -0
  62. {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/compass/providers/storage.py +0 -0
  63. {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/compass/remediation.py +0 -0
  64. {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/compass/report/_report_css.py +0 -0
  65. {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/compass/report/_report_js.py +0 -0
  66. {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/compass/report/analysis.py +0 -0
  67. {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/compass/report/shell.py +0 -0
  68. {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/compass/retrieval_analysis.py +0 -0
  69. {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/compass/text_probes.py +0 -0
  70. {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/compass/trend_tracker.py +0 -0
  71. {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/compass/utils/__init__.py +0 -0
  72. {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/compass/validators/__init__.py +0 -0
  73. {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/compass/validators/api_upload.py +0 -0
  74. {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/compass/validators/base.py +0 -0
  75. {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/compass/validators/base_sparql.py +0 -0
  76. {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/compass/validators/cross_stage.py +0 -0
  77. {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/compass/validators/graphdb_metadata.py +0 -0
  78. {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/compass/validators/neptune_metadata.py +0 -0
  79. {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/compass/validators/registry.py +0 -0
  80. {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/compass/validators/retrieval_quality.py +0 -0
  81. {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/compass_kb_validation.egg-info/dependency_links.txt +0 -0
  82. {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/compass_kb_validation.egg-info/entry_points.txt +0 -0
  83. {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/compass_kb_validation.egg-info/requires.txt +0 -0
  84. {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/compass_kb_validation.egg-info/top_level.txt +0 -0
  85. {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/ingestion_validation/__init__.py +0 -0
  86. {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/profile_kb.py +0 -0
  87. {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/pyproject.toml +0 -0
  88. {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/run_validation.py +0 -0
  89. {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/setup.cfg +0 -0
  90. {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/tests/test_alias.py +0 -0
  91. {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/tests/test_alt_adapters.py +0 -0
  92. {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/tests/test_analysis_report.py +0 -0
  93. {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/tests/test_api_upload.py +0 -0
  94. {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/tests/test_ask.py +0 -0
  95. {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/tests/test_baseline.py +0 -0
  96. {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/tests/test_check_catalog.py +0 -0
  97. {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/tests/test_citation.py +0 -0
  98. {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/tests/test_cli.py +0 -0
  99. {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/tests/test_compat_shim.py +0 -0
  100. {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/tests/test_config.py +0 -0
  101. {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/tests/test_config_lint.py +0 -0
  102. {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/tests/test_content_inspect.py +0 -0
  103. {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/tests/test_corpus.py +0 -0
  104. {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/tests/test_corpus_health.py +0 -0
  105. {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/tests/test_coverage.py +0 -0
  106. {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/tests/test_cross_stage.py +0 -0
  107. {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/tests/test_diagnostics.py +0 -0
  108. {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/tests/test_discover_cli.py +0 -0
  109. {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/tests/test_discovery_parallel.py +0 -0
  110. {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/tests/test_embedding_drift.py +0 -0
  111. {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/tests/test_eval_enrich.py +0 -0
  112. {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/tests/test_eval_interop.py +0 -0
  113. {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/tests/test_evidence.py +0 -0
  114. {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/tests/test_gate.py +0 -0
  115. {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/tests/test_graph_transport.py +0 -0
  116. {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/tests/test_graph_validators.py +0 -0
  117. {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/tests/test_grounding.py +0 -0
  118. {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/tests/test_init_wizard.py +0 -0
  119. {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/tests/test_manifest.py +0 -0
  120. {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/tests/test_mcp.py +0 -0
  121. {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/tests/test_migration.py +0 -0
  122. {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/tests/test_models.py +0 -0
  123. {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/tests/test_neptune_bugfixes.py +0 -0
  124. {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/tests/test_neptune_bugfixes2.py +0 -0
  125. {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/tests/test_notifications.py +0 -0
  126. {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/tests/test_pipeline.py +0 -0
  127. {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/tests/test_profiler.py +0 -0
  128. {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/tests/test_providers.py +0 -0
  129. {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/tests/test_redis_chunks.py +0 -0
  130. {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/tests/test_registry.py +0 -0
  131. {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/tests/test_remediation.py +0 -0
  132. {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/tests/test_reporting.py +0 -0
  133. {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/tests/test_retrieval_assertions.py +0 -0
  134. {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/tests/test_retrieval_quality.py +0 -0
  135. {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/tests/test_review_hardening.py +0 -0
  136. {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/tests/test_run_cli.py +0 -0
  137. {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/tests/test_sources.py +0 -0
  138. {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/tests/test_storage_validator.py +0 -0
  139. {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/tests/test_trends.py +0 -0
  140. {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/tests/test_universal_concerns.py +0 -0
  141. {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/tests/test_utils.py +0 -0
  142. {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/tests/test_vector_fastpath.py +0 -0
  143. {compass_kb_validation-3.6.0 → compass_kb_validation-3.7.0}/tests/test_vectorstore_filters.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: compass-kb-validation
3
- Version: 3.6.0
3
+ Version: 3.7.0
4
4
  Summary: Knowledge-base readiness validation for RAG & AI agents — config-driven, deterministic pipeline integrity checks.
5
5
  Author: COMPASS
6
6
  License: MIT
@@ -5,4 +5,4 @@ end-to-end: API upload -> object storage -> knowledge graph -> vector store ->
5
5
  retrieval quality, plus cross-store consistency.
6
6
  """
7
7
 
8
- __version__ = "3.6.0"
8
+ __version__ = "3.7.0"
@@ -40,11 +40,35 @@ def parse_filters(spec: str) -> dict:
40
40
  return out
41
41
 
42
42
 
43
+ def _citation_fields(settings: Settings) -> list[str]:
44
+ """Index fields the citation/grounding config already names.
45
+
46
+ A retrieved chunk that does not carry its document title and page cannot be matched to a
47
+ citation, so Level-1 grounding verification silently reports UNVERIFIABLE for every claim.
48
+ Rather than ask the user to restate those field names under a second config key, derive
49
+ them from the ``eval.gt_*`` / ``eval.citation_*`` settings that already declare them: if
50
+ the project configured grounding at all, retrieval returns what grounding needs.
51
+ Requesting a field the index lacks is harmless - RediSearch simply omits it.
52
+ """
53
+ ev = settings.eval
54
+ fields = [
55
+ getattr(ev, "gt_chunk_doc_field", ""), # a chunk's source-document title
56
+ getattr(ev, "gt_chunk_page_field", ""), # a chunk's page, for page-level matching
57
+ getattr(ev, "gt_index_page_field", ""), # the index-side page field, if named differently
58
+ getattr(ev, "gt_doc_name_field", ""), # the document-name field used for L2 lookups
59
+ ]
60
+ fields += list(getattr(ev, "citation_doc_fields", []) or [])
61
+ fields += list(getattr(ev, "citation_source_key_fields", []) or []) # Level-3 source tracing
62
+ return [f for f in fields if f]
63
+
64
+
43
65
  def _return_fields(settings: Settings) -> list[str]:
44
66
  rc = settings.redis
45
- # Base identity/text + the project's configured metadata fields (empty by default;
46
- # harmless if the index lacks any), so results can feed export / comparison.
67
+ # Base identity/text, the fields grounding needs to match a citation, and the project's
68
+ # configured metadata fields (empty by default; harmless if the index lacks any), so
69
+ # results can feed export / comparison.
47
70
  wanted = (["file_name", "text", rc.document_id_field, rc.positional_field, "s3_key"]
71
+ + _citation_fields(settings)
48
72
  + list(settings.eval.metadata_fields))
49
73
  seen: set[str] = set()
50
74
  out: list[str] = []
@@ -0,0 +1,345 @@
1
+ """Retrieval-strategy benchmarking: which way through this KB finds the right evidence?
2
+
3
+ `discover` proves the KB is intact. It says nothing about whether the retrieval PATH through
4
+ it is any good - so a team choosing between pure vector search, hybrid, or wider-then-narrower
5
+ has no evidence, guesses, deploys, and then blames the KB when answers are poor. Usually the
6
+ KB is fine and the retrieval configuration is wrong.
7
+
8
+ This runs an eval set's questions through two or more retrieval configurations against the
9
+ SAME validated KB and scores each one with standard information-retrieval metrics against the
10
+ ground-truth chunks the eval already carries. No LLM anywhere: a chunk is relevant because the
11
+ ground truth says so, not because a model judged it. Deterministic and repeatable - same eval
12
+ set and same KB give the same answer every time.
13
+
14
+ What it cannot do is invent ground truth. A case with no ground-truth chunks is EXCLUDED from
15
+ the metrics rather than scored zero: "we have nothing to compare against" and "retrieval found
16
+ nothing" are different findings, and averaging the first into the second would understate every
17
+ strategy equally and invisibly.
18
+ """
19
+
20
+ from __future__ import annotations
21
+
22
+ import math
23
+ from dataclasses import dataclass, field
24
+ from typing import Any, Optional
25
+
26
+ DEFAULT_TOP_K = 10
27
+
28
+
29
+ # --------------------------------------------------------------------------- #
30
+ # Information-retrieval metrics (binary relevance)
31
+ # --------------------------------------------------------------------------- #
32
+ def recall_at_k(retrieved: list[str], relevant: set, k: int) -> Optional[float]:
33
+ """Share of the relevant chunks that appear in the top ``k``. None if nothing is relevant."""
34
+ if not relevant:
35
+ return None
36
+ found = len({r for r in retrieved[:k] if r in relevant})
37
+ return found / len(relevant)
38
+
39
+
40
+ def precision_at_k(retrieved: list[str], relevant: set, k: int) -> Optional[float]:
41
+ """Share of the top ``k`` that is relevant.
42
+
43
+ The denominator is k, not len(retrieved): a strategy that returns 3 results when asked for
44
+ 10 has not earned the precision of one that returned 3 out of 3, and dividing by the
45
+ smaller number would flatter it for retrieving too little.
46
+ """
47
+ if not relevant or k <= 0:
48
+ return None
49
+ found = len({r for r in retrieved[:k] if r in relevant})
50
+ return found / k
51
+
52
+
53
+ def reciprocal_rank(retrieved: list[str], relevant: set) -> Optional[float]:
54
+ """1 / rank of the first relevant chunk; 0.0 when none was retrieved."""
55
+ if not relevant:
56
+ return None
57
+ for i, chunk in enumerate(retrieved, 1):
58
+ if chunk in relevant:
59
+ return 1.0 / i
60
+ return 0.0
61
+
62
+
63
+ def ndcg_at_k(retrieved: list[str], relevant: set, k: int) -> Optional[float]:
64
+ """Normalised discounted cumulative gain with binary relevance.
65
+
66
+ Rewards putting relevant chunks EARLY, which recall alone cannot see: two strategies can
67
+ both find the evidence while one buries it at rank 9 where a top-3 context window will
68
+ never include it.
69
+ """
70
+ if not relevant or k <= 0:
71
+ return None
72
+ dcg = sum(1.0 / math.log2(i + 1) for i, chunk in enumerate(retrieved[:k], 1)
73
+ if chunk in relevant)
74
+ ideal = sum(1.0 / math.log2(i + 1) for i in range(1, min(len(relevant), k) + 1))
75
+ return (dcg / ideal) if ideal else None
76
+
77
+
78
+ def hit_rate(retrieved: list[str], relevant: set, k: int) -> Optional[float]:
79
+ """1.0 when at least one relevant chunk is in the top ``k`` - did retrieval find anything?"""
80
+ if not relevant:
81
+ return None
82
+ return 1.0 if any(r in relevant for r in retrieved[:k]) else 0.0
83
+
84
+
85
+ METRICS = ("recall", "precision", "mrr", "ndcg", "hit_rate")
86
+
87
+
88
+ def score_one(retrieved: list[str], relevant: set, k: int) -> dict:
89
+ """Every metric for one query. Values are None when the query has no ground truth."""
90
+ return {
91
+ "recall": recall_at_k(retrieved, relevant, k),
92
+ "precision": precision_at_k(retrieved, relevant, k),
93
+ "mrr": reciprocal_rank(retrieved, relevant),
94
+ "ndcg": ndcg_at_k(retrieved, relevant, k),
95
+ "hit_rate": hit_rate(retrieved, relevant, k),
96
+ }
97
+
98
+
99
+ def _mean(values: list) -> Optional[float]:
100
+ nums = [v for v in values if isinstance(v, (int, float)) and not isinstance(v, bool)]
101
+ return round(sum(nums) / len(nums), 4) if nums else None
102
+
103
+
104
+ # --------------------------------------------------------------------------- #
105
+ # Strategies
106
+ # --------------------------------------------------------------------------- #
107
+ @dataclass
108
+ class Strategy:
109
+ """One retrieval configuration to compare. Everything comes from project YAML."""
110
+
111
+ name: str
112
+ type: str = "vector" # vector | hybrid
113
+ top_k: int = DEFAULT_TOP_K
114
+ filters: dict = field(default_factory=dict)
115
+ expand_context: int = 0 # include N neighbouring chunks around each hit
116
+ rerank_top_k: int = 0 # retrieve wider, then keep this many
117
+ extra: dict = field(default_factory=dict)
118
+
119
+ @classmethod
120
+ def from_config(cls, raw: dict) -> "Strategy":
121
+ known = {"name", "type", "top_k", "filters", "expand_context", "rerank_top_k"}
122
+ return cls(name=str(raw.get("name") or raw.get("type") or "strategy"),
123
+ type=str(raw.get("type") or "vector"),
124
+ top_k=int(raw.get("top_k") or DEFAULT_TOP_K),
125
+ filters=dict(raw.get("filters") or {}),
126
+ expand_context=int(raw.get("expand_context") or 0),
127
+ rerank_top_k=int(raw.get("rerank_top_k") or 0),
128
+ extra={k: v for k, v in raw.items() if k not in known})
129
+
130
+ @property
131
+ def effective_k(self) -> int:
132
+ """The number of chunks a consumer actually sees after any narrowing step."""
133
+ return self.rerank_top_k or self.top_k
134
+
135
+ def unsupported(self) -> str:
136
+ """Why this strategy cannot run here, or "" when it can.
137
+
138
+ A strategy naming a reranker is refused rather than silently run WITHOUT it: reporting
139
+ "vector-rerank: recall 0.63" for a run that never reranked would attribute the plain
140
+ vector score to a configuration nobody tested.
141
+ """
142
+ if self.extra.get("reranker"):
143
+ return (f"'{self.extra['reranker']}' reranking is not built in - COMPASS ships no "
144
+ "cross-encoder provider, so this strategy cannot be measured here. Remove "
145
+ "the reranker to benchmark the retrieval it wraps.")
146
+ if self.type not in ("vector", "hybrid"):
147
+ return f"unknown strategy type '{self.type}' (expected 'vector' or 'hybrid')"
148
+ return ""
149
+
150
+
151
+ def chunk_identity(chunk: dict, doc_field: str, page_field: str) -> str:
152
+ """A stable identity for matching a retrieved chunk to a ground-truth one.
153
+
154
+ Document + page, because that is the granularity ground truth is written at and the only
155
+ identifier both sides reliably share - a store's internal chunk key never appears in an
156
+ eval file.
157
+ """
158
+ doc = str(chunk.get(doc_field) or chunk.get("file_name") or chunk.get("document_title") or "")
159
+ page = str(chunk.get(page_field) or chunk.get("page_start") or "")
160
+ return f"{doc.strip().lower()}#{page.strip()}" if doc else ""
161
+
162
+
163
+ def ground_truth_ids(case: dict, cfg: Any) -> set:
164
+ """The chunks a case's ground truth says are relevant."""
165
+ from compass.eval_interop import get_path
166
+
167
+ doc_field = getattr(cfg, "gt_chunk_doc_field", "document_title")
168
+ page_field = getattr(cfg, "gt_chunk_page_field", "page_start")
169
+ found = get_path(case, getattr(cfg, "gt_chunks_path", "context.chunks"))
170
+ chunks: list = found if isinstance(found, list) else []
171
+ if not chunks and isinstance(case.get("chunks"), list):
172
+ chunks = case["chunks"]
173
+ out = {chunk_identity(c, doc_field, page_field) for c in chunks if isinstance(c, dict)}
174
+ return {i for i in out if i}
175
+
176
+
177
+ def run_strategy(settings: Any, question: str, strategy: Strategy, *, embedder: Any = None,
178
+ vectorstore: Any = None) -> list[str]:
179
+ """Retrieve for one question under one strategy; return chunk identities in rank order."""
180
+ from compass.ask import ask_kb
181
+
182
+ payload = ask_kb(settings, question, top_k=max(strategy.top_k, strategy.effective_k),
183
+ filters=strategy.filters or None, embedder=embedder,
184
+ vectorstore=vectorstore)
185
+ doc_field = getattr(settings.eval, "gt_chunk_doc_field", "document_title")
186
+ page_field = getattr(settings.eval, "gt_chunk_page_field", "page_start")
187
+ ids: list[str] = []
188
+ for result in payload.get("results", []):
189
+ merged = {**(result.get("meta") or {}), **result}
190
+ identity = chunk_identity(merged, doc_field, page_field)
191
+ if identity and identity not in ids:
192
+ ids.append(identity)
193
+ return ids[: strategy.effective_k] if strategy.rerank_top_k else ids
194
+
195
+
196
+ # --------------------------------------------------------------------------- #
197
+ # Benchmark run
198
+ # --------------------------------------------------------------------------- #
199
+ def benchmark(settings: Any, cases: list[dict], strategies: list[Strategy], *,
200
+ retrieve: Any = None) -> dict:
201
+ """Score every strategy over every case. ``retrieve`` is injectable for testing.
202
+
203
+ Cases with no ground truth are excluded and counted separately - scoring them zero would
204
+ penalise every strategy identically for a gap in the EVAL SET, and hide that the number is
205
+ based on fewer queries than the reader assumes.
206
+ """
207
+ from compass.eval_interop import get_path
208
+
209
+ q_path = getattr(settings.eval, "gt_question_path", "question")
210
+ runner = retrieve or (lambda q, s: run_strategy(settings, q, s))
211
+
212
+ usable, skipped = [], []
213
+ for case in cases:
214
+ question = get_path(case, q_path) or case.get("input") or case.get("question")
215
+ relevant = ground_truth_ids(case, settings.eval)
216
+ if not question or not relevant:
217
+ skipped.append(str(case.get("case_id") or case.get("test_id") or "?"))
218
+ continue
219
+ usable.append((case, str(question), relevant))
220
+
221
+ per_strategy: list[dict] = []
222
+ per_query: dict[str, dict] = {}
223
+ for strategy in strategies:
224
+ blocked = strategy.unsupported()
225
+ if blocked:
226
+ per_strategy.append({"strategy": strategy.name, "skipped": True, "reason": blocked})
227
+ continue
228
+ scores: dict[str, list] = {m: [] for m in METRICS}
229
+ for case, question, relevant in usable:
230
+ cid = str(case.get("case_id") or case.get("test_id") or question[:40])
231
+ retrieved = runner(question, strategy)
232
+ result = score_one(retrieved, relevant, strategy.effective_k)
233
+ for metric, value in result.items():
234
+ scores[metric].append(value)
235
+ per_query.setdefault(cid, {"question": question,
236
+ "ground_truth": len(relevant)})[strategy.name] = {
237
+ "found": len([r for r in retrieved[:strategy.effective_k] if r in relevant]),
238
+ "recall": result["recall"], "mrr": result["mrr"]}
239
+ per_strategy.append({
240
+ "strategy": strategy.name, "skipped": False, "top_k": strategy.effective_k,
241
+ "queries": len(usable),
242
+ **{metric: _mean(values) for metric, values in scores.items()},
243
+ })
244
+
245
+ scored = [s for s in per_strategy if not s.get("skipped")]
246
+ winner = max(scored, key=lambda s: (s.get("ndcg") or 0, s.get("recall") or 0), default=None)
247
+ return {
248
+ "project": getattr(settings, "project_name", "") or "",
249
+ "queries": len(usable),
250
+ "excluded_cases": skipped,
251
+ "strategies": per_strategy,
252
+ "winner": winner["strategy"] if winner else "",
253
+ "per_query": per_query,
254
+ "method": ("binary-relevance IR metrics against the eval's own ground-truth chunks, "
255
+ "matched on document + page. No LLM: relevance comes from the ground truth, "
256
+ "not from a model's judgement."),
257
+ }
258
+
259
+
260
+ def worst_queries(report: dict, limit: int = 5) -> list[dict]:
261
+ """Queries where the best and worst strategies disagree most - where the choice matters."""
262
+ rows = []
263
+ for cid, data in report.get("per_query", {}).items():
264
+ recalls: list[tuple[str, float]] = [
265
+ (str(name), float(v["recall"])) for name, v in data.items()
266
+ if isinstance(v, dict) and isinstance(v.get("recall"), (int, float))]
267
+ if len(recalls) < 2:
268
+ continue
269
+ best = max(recalls, key=lambda t: t[1])
270
+ worst = min(recalls, key=lambda t: t[1])
271
+ if best[1] > worst[1]:
272
+ rows.append({"case_id": cid, "question": data.get("question", ""),
273
+ "best": best[0], "best_recall": best[1],
274
+ "worst": worst[0], "worst_recall": worst[1],
275
+ "spread": round(best[1] - worst[1], 4)})
276
+ return sorted(rows, key=lambda r: -r["spread"])[:limit]
277
+
278
+
279
+ def format_report(report: dict) -> str:
280
+ """Console table plus the queries where the strategy choice actually matters."""
281
+ L = ["=" * 72,
282
+ f" COMPASS benchmark - {report.get('project') or 'KB'} "
283
+ f"({report['queries']} queries, {len(report['strategies'])} strategies)",
284
+ "=" * 72, ""]
285
+ if not report["queries"]:
286
+ L.append(" No case carried both a question and ground-truth chunks, so nothing could "
287
+ "be measured.")
288
+ L.append(" Check eval.gt_question_path and eval.gt_chunks_path.")
289
+ L.append("=" * 72)
290
+ return "\n".join(L)
291
+
292
+ head = f" {'Strategy':<20}{'Recall':>9}{'Precision':>11}{'MRR':>8}{'NDCG':>8}{'Hit rate':>10}"
293
+ L += [head, " " + "-" * (len(head) - 2)]
294
+ for s in report["strategies"]:
295
+ if s.get("skipped"):
296
+ L.append(f" {s['strategy']:<20} SKIPPED - {s['reason']}")
297
+ continue
298
+ L.append(f" {s['strategy']:<20}"
299
+ + "".join(f"{_fmt(s.get(m)):>{w}}" for m, w in
300
+ (("recall", 9), ("precision", 11), ("mrr", 8), ("ndcg", 8),
301
+ ("hit_rate", 10))))
302
+ L.append("")
303
+
304
+ scored = [s for s in report["strategies"] if not s.get("skipped")]
305
+ if report.get("winner") and len(scored) > 1:
306
+ win = next(s for s in scored if s["strategy"] == report["winner"])
307
+ base = min(scored, key=lambda s: (s.get("ndcg") or 0))
308
+ L.append(f" WINNER: {win['strategy']}")
309
+ for metric, label in (("recall", "recall"), ("mrr", "MRR"), ("ndcg", "NDCG")):
310
+ a, b = win.get(metric), base.get(metric)
311
+ if isinstance(a, (int, float)) and isinstance(b, (int, float)) and b:
312
+ L.append(f" {(a - b) / b:+.0%} {label} vs {base['strategy']}")
313
+ L.append("")
314
+
315
+ worst = worst_queries(report)
316
+ if worst:
317
+ L.append(" WHERE THE CHOICE MATTERS (largest disagreement between strategies):")
318
+ for row in worst:
319
+ L.append(f" {row['case_id']}: \"{row['question'][:58]}\"")
320
+ L.append(f" {row['best']}: recall {row['best_recall']:.2f} "
321
+ f"{row['worst']}: recall {row['worst_recall']:.2f}")
322
+ L.append("")
323
+ if report["excluded_cases"]:
324
+ L.append(f" {len(report['excluded_cases'])} case(s) excluded - no question or no "
325
+ "ground-truth chunks to measure against, so they are not scored as misses.")
326
+ L.append(" Method: " + report["method"])
327
+ L.append("=" * 72)
328
+ return "\n".join(L)
329
+
330
+
331
+ def _fmt(value: Any) -> str:
332
+ return f"{value:.2f}" if isinstance(value, (int, float)) else "-"
333
+
334
+
335
+ def strategies_from_settings(settings: Any) -> list[Strategy]:
336
+ """Read ``benchmark.strategies`` from project config."""
337
+ raw = getattr(getattr(settings, "benchmark", None), "strategies", None)
338
+ if raw is None:
339
+ raw = (getattr(settings, "extra", {}) or {}).get("benchmark", {}).get("strategies", [])
340
+ return [Strategy.from_config(s) for s in (raw or []) if isinstance(s, dict)]
341
+
342
+
343
+ __all__ = ["Strategy", "benchmark", "format_report", "score_one", "recall_at_k",
344
+ "precision_at_k", "reciprocal_rank", "ndcg_at_k", "hit_rate", "ground_truth_ids",
345
+ "chunk_identity", "worst_queries", "strategies_from_settings", "run_strategy"]
@@ -36,8 +36,13 @@ from typing import Any, Optional
36
36
 
37
37
  # Default citation patterns. `(DOC, descriptor, p. N)` / `pp. N-M`, and a bare **file.ext**.
38
38
  # Extra domain patterns can be added via config; these always apply.
39
+ # The document token is capped generously (not at 30 chars, which silently DROPPED every
40
+ # citation naming a real-world document title - "(Quarterly Investment Perspective, Q3, p. 4)"
41
+ # is 32 chars and produced zero citations, not even an unverifiable one). The character class
42
+ # already excludes "," and ")", and the whole pattern still requires the
43
+ # `, <descriptor>, p. <n>` structure, so a longer cap cannot run away through prose.
39
44
  _PAREN_CITE = re.compile(
40
- r"\(\s*([A-Z][A-Za-z0-9&/ .-]{0,30}?)\s*,\s*([^,)]{1,40}?)\s*,\s*pp?\.\s*(\d+)(?:\s*[-–]\s*(\d+))?\s*\)")
45
+ r"\(\s*([A-Z][A-Za-z0-9&/ .-]{0,120}?)\s*,\s*([^,)]{1,40}?)\s*,\s*pp?\.\s*(\d+)(?:\s*[-–]\s*(\d+))?\s*\)")
41
46
  _FILE_CITE = re.compile(r"\*\*\s*([^*]+?\.(?:pdf|docx|pptx|txt|md))\s*\*\*", re.IGNORECASE)
42
47
 
43
48
  # One tokenizer drives BOTH extraction and matching, so a claim number and a source number
@@ -335,6 +340,14 @@ class CitationVerdict:
335
340
  # --- Level-3 plumbing -------------------------------------------------------------- #
336
341
  source_key: str = "" # storage key of the cited document, resolved from its KB chunks
337
342
  alias_resolved: str = "" # KB document name an auto-derived alias resolved the citation to
343
+ # A snippet of the text the claim was actually checked against, so a report can show the
344
+ # evidence beside the claim instead of asserting a verdict the reader cannot inspect.
345
+ evidence: str = ""
346
+ evidence_source: str = "" # eval_chunk | kb_page | source_document
347
+ #: How this citation's document token was resolved, step by step. Populated so a user can
348
+ #: see WHY an alias did or did not resolve instead of inferring it from a bare verdict -
349
+ #: "I added the alias and it is still UNVERIFIABLE" is otherwise undiagnosable.
350
+ resolution: list = field(default_factory=list)
338
351
 
339
352
 
340
353
  _MONTHS = set("jan feb mar apr may jun jul aug sep sept oct nov dec january february march april "
@@ -388,6 +401,7 @@ def verify_level1(citations: list[Citation], eval_chunks: list[dict], cfg: Any)
388
401
  verdicts.append(v)
389
402
  continue
390
403
  v.chunk_matched = True
404
+ v.evidence, v.evidence_source = _snippet(_raw_text(chunk, text_field)), "eval_chunk"
391
405
  chunk_nums = _iter_nums(_haystack(chunk, text_field))
392
406
  for cv in cit.values:
393
407
  if cv.matches(chunk_nums):
@@ -421,6 +435,19 @@ def verify_level1(citations: list[Citation], eval_chunks: list[dict], cfg: Any)
421
435
  return verdicts
422
436
 
423
437
 
438
+ def _raw_text(chunk: dict, text_field: str) -> str:
439
+ """A chunk's text with its ORIGINAL casing - `_haystack` lowercases for matching, which is
440
+ right for comparison and wrong for anything a human reads."""
441
+ t = str(chunk.get(text_field, "") or "")
442
+ return t or " ".join(str(v) for v in chunk.values() if isinstance(v, str))
443
+
444
+
445
+ def _snippet(text: str, width: int = 320) -> str:
446
+ """A readable slice of the text a claim was checked against (whitespace normalised)."""
447
+ t = re.sub(r"\s+", " ", (text or "")).strip()
448
+ return t if len(t) <= width else t[:width].rsplit(" ", 1)[0] + " ..."
449
+
450
+
424
451
  def _squash(s: str) -> str:
425
452
  """Lowercase and drop whitespace so ``350bps`` matches a source's ``350 bps``."""
426
453
  return re.sub(r"\s+", "", (s or "").lower())
@@ -504,6 +531,93 @@ def _page_match(chunk_page: str, cited: str, tol: int) -> bool:
504
531
  return False
505
532
 
506
533
 
534
+ #: Share of a claim's distinctive content words that must appear in the cited source text for
535
+ #: the claim to count as lexically supported. Deliberately short of 1.0 - normal wording
536
+ #: differences ("rose"/"increased") should not sink an otherwise well-supported statement.
537
+ TERM_SUPPORT_THRESHOLD = 0.6
538
+
539
+ # Common words carry no evidence, so they neither help nor hurt the support ratio.
540
+ _COMMON = {
541
+ "about", "above", "after", "again", "against", "also", "another", "because", "been",
542
+ "before", "being", "below", "between", "both", "cannot", "could", "does", "doing", "down",
543
+ "during", "each", "either", "from", "further", "have", "having", "here", "into", "itself",
544
+ "just", "more", "most", "much", "must", "only", "other", "over", "same", "should", "since",
545
+ "some", "such", "than", "that", "their", "them", "then", "there", "these", "they", "this",
546
+ "those", "through", "under", "until", "very", "were", "what", "when", "where", "which",
547
+ "while", "with", "would", "your", "will", "shall", "may", "might", "the", "and", "for",
548
+ }
549
+
550
+
551
+ def _content_words(text: str) -> list[str]:
552
+ """Distinctive words of a claim - long enough and common enough to carry evidence."""
553
+ words = re.findall(r"[A-Za-z][A-Za-z0-9'-]{3,}", text or "")
554
+ seen: set[str] = set()
555
+ out: list[str] = []
556
+ for w in words:
557
+ low = w.lower()
558
+ if low not in _COMMON and low not in seen:
559
+ seen.add(low)
560
+ out.append(w)
561
+ return out
562
+
563
+
564
+ def lexical_support(claim: str, source_text: str) -> tuple[float, list[str]]:
565
+ """(share of the claim's distinctive words present in ``source_text``, the absent ones).
566
+
567
+ Deliberately a presence ratio, not an entailment judgement: it says how much of the
568
+ claim's own language the cited text actually contains. Returns (0.0, []) when the claim
569
+ has nothing distinctive to check.
570
+ """
571
+ words = _content_words(claim)
572
+ if not words:
573
+ return 0.0, []
574
+ hay = _squash(source_text)
575
+ missing = [w for w in words if _squash(w) not in hay]
576
+ return (len(words) - len(missing)) / len(words), missing
577
+
578
+
579
+ def _term_verify_l2(v: "CitationVerdict", hits: list[dict], page_field: str, text_field: str,
580
+ page_tol: int) -> str:
581
+ """Verify a non-quantitative claim against the cited page's text from the KB.
582
+
583
+ Returns the Level-2 verdict: ``TERM_VERIFIED_L2`` when the claim's distinctive language is
584
+ present at the cited location, else ``SOURCE_PRESENT`` (the document is there, the wording
585
+ is not - reported unverifiable, never a fault).
586
+ """
587
+ # `v.negated` is only set for numeric claims, so check the claim text directly: a negated
588
+ # prose claim shares nearly all its words with text asserting the OPPOSITE ("did not signal
589
+ # a constructive outlook" vs "signalled a constructive outlook"), which would score as
590
+ # supported. Lexical presence cannot tell those apart, so it must not claim to.
591
+ if v.negated or _NEG.search(v.claim or ""):
592
+ return "SOURCE_PRESENT"
593
+ on_page = [h for h in hits if _page_match(h.get(page_field, ""), v.page, page_tol)]
594
+ text = " ".join(_haystack(h, text_field) for h in (on_page or hits))
595
+ if not text.strip():
596
+ return "SOURCE_PRESENT"
597
+ if not v.evidence:
598
+ # Original casing for display; `text` above is the lowercased matching buffer.
599
+ v.evidence = _snippet(" ".join(_raw_text(h, text_field) for h in (on_page or hits)))
600
+ v.evidence_source = "kb_page"
601
+ # Distinctive entities (quoted strings, multi-word proper nouns) are the strongest lexical
602
+ # evidence; fall back to the claim's content words when it names none.
603
+ ents = _extract_entities(v.claim)
604
+ if ents:
605
+ hay = _squash(text)
606
+ v.entity_missing = [e for e in ents if _squash(e) not in hay]
607
+ v.entities_checked = True
608
+ if not v.entity_missing:
609
+ return "TERM_VERIFIED_L2"
610
+ ratio, missing = lexical_support(v.claim, text)
611
+ if ratio >= TERM_SUPPORT_THRESHOLD:
612
+ v.entities_checked = True
613
+ v.support_pct = round(100.0 * ratio, 1)
614
+ return "TERM_VERIFIED_L2"
615
+ if missing:
616
+ v.entities_checked = True
617
+ v.entity_missing = (v.entity_missing or []) + missing[:5]
618
+ return "SOURCE_PRESENT"
619
+
620
+
507
621
  def _source_key_of(hits: list[dict], key_fields: list[str]) -> str:
508
622
  """The storage key of the cited document, from the first KB chunk that records one."""
509
623
  for h in hits:
@@ -535,17 +649,28 @@ def verify_level2(verdicts: list[CitationVerdict], adapter: Any, cfg: Any, l1_ra
535
649
  for v in verdicts:
536
650
  if v.l1_verdict == "GROUNDED":
537
651
  continue
538
- hits = _search_doc(adapter, fields, _resolve_doc(v.doc, aliases), cap)
652
+ resolved = _resolve_doc(v.doc, aliases)
653
+ v.resolution.append(f"parser token: '{v.doc}'")
654
+ if resolved != v.doc:
655
+ v.resolution.append(f"configured alias -> '{resolved}'")
656
+ else:
657
+ v.resolution.append("configured alias: no entry matched")
658
+ hits = _search_doc(adapter, fields, resolved, cap)
659
+ v.resolution.append(
660
+ f"KB search on {fields}: {len(hits)} hit(s)" if hits
661
+ else f"KB search on {fields}: no hit for '{resolved}'")
539
662
  if not hits and alias_map is not None:
540
663
  # Auto-derived alias: the citation's short token may be an initialism of a real KB
541
664
  # document name. Only a UNIQUE match resolves - an ambiguous key is left unresolved
542
665
  # so the verdict stays honestly ambiguous instead of picking a document.
543
- resolved, why = alias_map.resolve(v.doc)
544
- if resolved:
545
- hits = _search_doc(adapter, fields, resolved, cap)
666
+ auto, why = alias_map.resolve(v.doc)
667
+ v.resolution.append(f"auto-alias: {why}")
668
+ if auto:
669
+ hits = _search_doc(adapter, fields, auto, cap)
670
+ v.resolution.append(f"KB search on '{auto}': {len(hits)} hit(s)")
546
671
  if hits:
547
- v.alias_resolved = resolved
548
- v.note = f"resolved to '{resolved}' ({why})"
672
+ v.alias_resolved = auto
673
+ v.note = f"resolved to '{auto}' ({why})"
549
674
  if hits and key_fields:
550
675
  v.source_key = _source_key_of(hits, key_fields)
551
676
  if not hits:
@@ -568,10 +693,19 @@ def verify_level2(verdicts: list[CitationVerdict], adapter: Any, cfg: Any, l1_ra
568
693
  continue
569
694
  values = v._missing if (l1_ran and v._missing) else v.values
570
695
  if not values:
571
- v.l2_verdict = "SOURCE_PRESENT" # doc exists; claim not quantitatively checkable
696
+ # No quantitative claim - but the KB just handed us the cited page's actual text,
697
+ # so check the claim's OWN distinctive language against it instead of declaring it
698
+ # uncheckable. Reporting "unverifiable" while holding the source text is the thing
699
+ # that made a live-KB audit look like it verified nothing. A hit is a positive
700
+ # (weaker-tier) signal; a miss stays unverifiable and is NEVER a fault, because
701
+ # lexical absence can be paraphrase.
702
+ v.l2_verdict = _term_verify_l2(v, hits, page_field, text_field, page_tol)
572
703
  continue
573
704
  capped = len(hits) >= cap
574
705
  on_page = [h for h in hits if _page_match(h.get(page_field, ""), v.page, page_tol)]
706
+ if not v.evidence:
707
+ v.evidence = _snippet(" ".join(_raw_text(h, text_field) for h in (on_page or hits)))
708
+ v.evidence_source = "kb_page"
575
709
  page_nums = _iter_nums(" ".join(_haystack(h, text_field) for h in on_page))
576
710
  doc_nums = _iter_nums(" ".join(_haystack(h, text_field) for h in hits))
577
711
  in_page = [cv for cv in values if cv.matches(page_nums)]
@@ -709,8 +843,10 @@ def attribute(verdicts: list[CitationVerdict], l1_ran: bool) -> None:
709
843
  key = _L3_ATTRIB[v.l3_verdict] # the original source is decisive - it wins
710
844
  elif l2 == "FABRICATED_SOURCE":
711
845
  key = "fabricated_source" # cited a document that isn't in the KB
712
- elif l1 == "TERM_SUPPORTED":
713
- key = "term_supported" # distinctive entities present; weaker than numeric
846
+ elif l1 == "TERM_SUPPORTED" or l2 == "TERM_VERIFIED_L2":
847
+ # Distinctive language present at the cited location - weaker than a numeric match,
848
+ # but real evidence, and far better than calling a verifiable claim uncheckable.
849
+ key = "term_supported"
714
850
  elif l2 in _L2_UNVERIFIABLE:
715
851
  key = "unverifiable" # source unresolved / too large / no numeric claim
716
852
  elif l1 == "UNVERIFIABLE" and not l2:
@@ -753,7 +889,10 @@ def grade(verdicts: list[CitationVerdict], l1_ran: bool) -> None:
753
889
  for v in verdicts:
754
890
  total = len(v.values)
755
891
  if not total:
756
- v.tier = "TERM_SUPPORTED" if v.l1_verdict == "TERM_SUPPORTED" else "UNVERIFIABLE"
892
+ # Term support can be established at either level (the eval's chunk, or the cited
893
+ # page fetched from the KB), so key off the attribution rather than Level 1 alone.
894
+ v.tier = ("TERM_SUPPORTED" if v.attribution == "term_supported"
895
+ else "UNVERIFIABLE")
757
896
  continue
758
897
  if v.attribution == "none":
759
898
  v.tier, v.support_pct = "SOUND", 100.0