compass-kb-validation 3.5.0__tar.gz → 3.7.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (143) hide show
  1. {compass_kb_validation-3.5.0/compass_kb_validation.egg-info → compass_kb_validation-3.7.0}/PKG-INFO +5 -1
  2. {compass_kb_validation-3.5.0 → compass_kb_validation-3.7.0}/compass/__init__.py +1 -1
  3. {compass_kb_validation-3.5.0 → compass_kb_validation-3.7.0}/compass/alias.py +86 -27
  4. {compass_kb_validation-3.5.0 → compass_kb_validation-3.7.0}/compass/ask.py +26 -2
  5. compass_kb_validation-3.7.0/compass/benchmark.py +345 -0
  6. {compass_kb_validation-3.5.0 → compass_kb_validation-3.7.0}/compass/citation.py +150 -11
  7. {compass_kb_validation-3.5.0 → compass_kb_validation-3.7.0}/compass/cli.py +394 -6
  8. {compass_kb_validation-3.5.0 → compass_kb_validation-3.7.0}/compass/config/__init__.py +35 -3
  9. {compass_kb_validation-3.5.0 → compass_kb_validation-3.7.0}/compass/corpus_health.py +19 -0
  10. compass_kb_validation-3.7.0/compass/diff.py +339 -0
  11. {compass_kb_validation-3.5.0 → compass_kb_validation-3.7.0}/compass/eval_enrich.py +61 -0
  12. {compass_kb_validation-3.5.0 → compass_kb_validation-3.7.0}/compass/eval_interop.py +32 -0
  13. {compass_kb_validation-3.5.0 → compass_kb_validation-3.7.0}/compass/evidence.py +6 -10
  14. {compass_kb_validation-3.5.0 → compass_kb_validation-3.7.0}/compass/grounding.py +27 -8
  15. compass_kb_validation-3.7.0/compass/health.py +280 -0
  16. compass_kb_validation-3.7.0/compass/incremental.py +243 -0
  17. {compass_kb_validation-3.5.0 → compass_kb_validation-3.7.0}/compass/models/__init__.py +13 -1
  18. {compass_kb_validation-3.5.0 → compass_kb_validation-3.7.0}/compass/pipeline/__init__.py +4 -0
  19. {compass_kb_validation-3.5.0 → compass_kb_validation-3.7.0}/compass/providers/graph.py +39 -0
  20. {compass_kb_validation-3.5.0 → compass_kb_validation-3.7.0}/compass/providers/vectorstore.py +73 -0
  21. compass_kb_validation-3.7.0/compass/quality_gate.py +275 -0
  22. {compass_kb_validation-3.5.0 → compass_kb_validation-3.7.0}/compass/report/__init__.py +4 -2
  23. {compass_kb_validation-3.5.0 → compass_kb_validation-3.7.0}/compass/report/audit.py +10 -93
  24. {compass_kb_validation-3.5.0 → compass_kb_validation-3.7.0}/compass/report/export.py +51 -1
  25. compass_kb_validation-3.7.0/compass/report/grounding_report.py +564 -0
  26. {compass_kb_validation-3.5.0 → compass_kb_validation-3.7.0}/compass/retrieval_analysis.py +67 -1
  27. {compass_kb_validation-3.5.0 → compass_kb_validation-3.7.0}/compass/validators/redis_chunks.py +57 -3
  28. {compass_kb_validation-3.5.0 → compass_kb_validation-3.7.0}/compass/validators/s3_storage.py +5 -0
  29. {compass_kb_validation-3.5.0 → compass_kb_validation-3.7.0/compass_kb_validation.egg-info}/PKG-INFO +5 -1
  30. {compass_kb_validation-3.5.0 → compass_kb_validation-3.7.0}/compass_kb_validation.egg-info/SOURCES.txt +15 -0
  31. {compass_kb_validation-3.5.0 → compass_kb_validation-3.7.0}/compass_kb_validation.egg-info/requires.txt +5 -0
  32. {compass_kb_validation-3.5.0 → compass_kb_validation-3.7.0}/discover_and_validate.py +68 -1
  33. {compass_kb_validation-3.5.0 → compass_kb_validation-3.7.0}/pyproject.toml +5 -1
  34. compass_kb_validation-3.7.0/tests/test_alias_disambiguation.py +157 -0
  35. compass_kb_validation-3.7.0/tests/test_benchmark.py +221 -0
  36. compass_kb_validation-3.7.0/tests/test_calibrate_truth.py +41 -0
  37. {compass_kb_validation-3.5.0 → compass_kb_validation-3.7.0}/tests/test_citation_levels.py +62 -0
  38. compass_kb_validation-3.7.0/tests/test_diff.py +208 -0
  39. compass_kb_validation-3.7.0/tests/test_grounding_report.py +215 -0
  40. compass_kb_validation-3.7.0/tests/test_health.py +482 -0
  41. compass_kb_validation-3.7.0/tests/test_incremental.py +214 -0
  42. compass_kb_validation-3.7.0/tests/test_quality_gate.py +184 -0
  43. compass_kb_validation-3.7.0/tests/test_retrieval_fields.py +141 -0
  44. {compass_kb_validation-3.5.0 → compass_kb_validation-3.7.0}/tests/test_source_verification.py +3 -1
  45. compass_kb_validation-3.7.0/tests/test_vector_fastpath.py +98 -0
  46. compass_kb_validation-3.5.0/tests/test_benchmark.py +0 -43
  47. {compass_kb_validation-3.5.0 → compass_kb_validation-3.7.0}/LICENSE +0 -0
  48. {compass_kb_validation-3.5.0 → compass_kb_validation-3.7.0}/README.md +0 -0
  49. {compass_kb_validation-3.5.0 → compass_kb_validation-3.7.0}/compass/__main__.py +0 -0
  50. {compass_kb_validation-3.5.0 → compass_kb_validation-3.7.0}/compass/baseline.py +0 -0
  51. {compass_kb_validation-3.5.0 → compass_kb_validation-3.7.0}/compass/chunk_coherence.py +0 -0
  52. {compass_kb_validation-3.5.0 → compass_kb_validation-3.7.0}/compass/config_lint.py +0 -0
  53. {compass_kb_validation-3.5.0 → compass_kb_validation-3.7.0}/compass/content_inspect.py +0 -0
  54. {compass_kb_validation-3.5.0 → compass_kb_validation-3.7.0}/compass/corpus.py +0 -0
  55. {compass_kb_validation-3.5.0 → compass_kb_validation-3.7.0}/compass/diagnostics.py +0 -0
  56. {compass_kb_validation-3.5.0 → compass_kb_validation-3.7.0}/compass/embedding_drift.py +0 -0
  57. {compass_kb_validation-3.5.0 → compass_kb_validation-3.7.0}/compass/init_wizard.py +0 -0
  58. {compass_kb_validation-3.5.0 → compass_kb_validation-3.7.0}/compass/live_dashboard.py +0 -0
  59. {compass_kb_validation-3.5.0 → compass_kb_validation-3.7.0}/compass/mcp_server.py +0 -0
  60. {compass_kb_validation-3.5.0 → compass_kb_validation-3.7.0}/compass/notifications.py +0 -0
  61. {compass_kb_validation-3.5.0 → compass_kb_validation-3.7.0}/compass/profiler.py +0 -0
  62. {compass_kb_validation-3.5.0 → compass_kb_validation-3.7.0}/compass/provenance.py +0 -0
  63. {compass_kb_validation-3.5.0 → compass_kb_validation-3.7.0}/compass/providers/__init__.py +0 -0
  64. {compass_kb_validation-3.5.0 → compass_kb_validation-3.7.0}/compass/providers/embedding.py +0 -0
  65. {compass_kb_validation-3.5.0 → compass_kb_validation-3.7.0}/compass/providers/source.py +0 -0
  66. {compass_kb_validation-3.5.0 → compass_kb_validation-3.7.0}/compass/providers/storage.py +0 -0
  67. {compass_kb_validation-3.5.0 → compass_kb_validation-3.7.0}/compass/remediation.py +0 -0
  68. {compass_kb_validation-3.5.0 → compass_kb_validation-3.7.0}/compass/report/_report_css.py +0 -0
  69. {compass_kb_validation-3.5.0 → compass_kb_validation-3.7.0}/compass/report/_report_js.py +0 -0
  70. {compass_kb_validation-3.5.0 → compass_kb_validation-3.7.0}/compass/report/analysis.py +0 -0
  71. {compass_kb_validation-3.5.0 → compass_kb_validation-3.7.0}/compass/report/shell.py +0 -0
  72. {compass_kb_validation-3.5.0 → compass_kb_validation-3.7.0}/compass/text_probes.py +0 -0
  73. {compass_kb_validation-3.5.0 → compass_kb_validation-3.7.0}/compass/trend_tracker.py +0 -0
  74. {compass_kb_validation-3.5.0 → compass_kb_validation-3.7.0}/compass/utils/__init__.py +0 -0
  75. {compass_kb_validation-3.5.0 → compass_kb_validation-3.7.0}/compass/validators/__init__.py +0 -0
  76. {compass_kb_validation-3.5.0 → compass_kb_validation-3.7.0}/compass/validators/api_upload.py +0 -0
  77. {compass_kb_validation-3.5.0 → compass_kb_validation-3.7.0}/compass/validators/base.py +0 -0
  78. {compass_kb_validation-3.5.0 → compass_kb_validation-3.7.0}/compass/validators/base_sparql.py +0 -0
  79. {compass_kb_validation-3.5.0 → compass_kb_validation-3.7.0}/compass/validators/cross_stage.py +0 -0
  80. {compass_kb_validation-3.5.0 → compass_kb_validation-3.7.0}/compass/validators/graphdb_metadata.py +0 -0
  81. {compass_kb_validation-3.5.0 → compass_kb_validation-3.7.0}/compass/validators/neptune_metadata.py +0 -0
  82. {compass_kb_validation-3.5.0 → compass_kb_validation-3.7.0}/compass/validators/registry.py +0 -0
  83. {compass_kb_validation-3.5.0 → compass_kb_validation-3.7.0}/compass/validators/retrieval_quality.py +0 -0
  84. {compass_kb_validation-3.5.0 → compass_kb_validation-3.7.0}/compass_kb_validation.egg-info/dependency_links.txt +0 -0
  85. {compass_kb_validation-3.5.0 → compass_kb_validation-3.7.0}/compass_kb_validation.egg-info/entry_points.txt +0 -0
  86. {compass_kb_validation-3.5.0 → compass_kb_validation-3.7.0}/compass_kb_validation.egg-info/top_level.txt +0 -0
  87. {compass_kb_validation-3.5.0 → compass_kb_validation-3.7.0}/ingestion_validation/__init__.py +0 -0
  88. {compass_kb_validation-3.5.0 → compass_kb_validation-3.7.0}/profile_kb.py +0 -0
  89. {compass_kb_validation-3.5.0 → compass_kb_validation-3.7.0}/run_validation.py +0 -0
  90. {compass_kb_validation-3.5.0 → compass_kb_validation-3.7.0}/setup.cfg +0 -0
  91. {compass_kb_validation-3.5.0 → compass_kb_validation-3.7.0}/tests/test_alias.py +0 -0
  92. {compass_kb_validation-3.5.0 → compass_kb_validation-3.7.0}/tests/test_alt_adapters.py +0 -0
  93. {compass_kb_validation-3.5.0 → compass_kb_validation-3.7.0}/tests/test_analysis_report.py +0 -0
  94. {compass_kb_validation-3.5.0 → compass_kb_validation-3.7.0}/tests/test_api_upload.py +0 -0
  95. {compass_kb_validation-3.5.0 → compass_kb_validation-3.7.0}/tests/test_ask.py +0 -0
  96. {compass_kb_validation-3.5.0 → compass_kb_validation-3.7.0}/tests/test_baseline.py +0 -0
  97. {compass_kb_validation-3.5.0 → compass_kb_validation-3.7.0}/tests/test_check_catalog.py +0 -0
  98. {compass_kb_validation-3.5.0 → compass_kb_validation-3.7.0}/tests/test_citation.py +0 -0
  99. {compass_kb_validation-3.5.0 → compass_kb_validation-3.7.0}/tests/test_cli.py +0 -0
  100. {compass_kb_validation-3.5.0 → compass_kb_validation-3.7.0}/tests/test_compat_shim.py +0 -0
  101. {compass_kb_validation-3.5.0 → compass_kb_validation-3.7.0}/tests/test_config.py +0 -0
  102. {compass_kb_validation-3.5.0 → compass_kb_validation-3.7.0}/tests/test_config_lint.py +0 -0
  103. {compass_kb_validation-3.5.0 → compass_kb_validation-3.7.0}/tests/test_content_inspect.py +0 -0
  104. {compass_kb_validation-3.5.0 → compass_kb_validation-3.7.0}/tests/test_corpus.py +0 -0
  105. {compass_kb_validation-3.5.0 → compass_kb_validation-3.7.0}/tests/test_corpus_health.py +0 -0
  106. {compass_kb_validation-3.5.0 → compass_kb_validation-3.7.0}/tests/test_coverage.py +0 -0
  107. {compass_kb_validation-3.5.0 → compass_kb_validation-3.7.0}/tests/test_cross_stage.py +0 -0
  108. {compass_kb_validation-3.5.0 → compass_kb_validation-3.7.0}/tests/test_diagnostics.py +0 -0
  109. {compass_kb_validation-3.5.0 → compass_kb_validation-3.7.0}/tests/test_discover_cli.py +0 -0
  110. {compass_kb_validation-3.5.0 → compass_kb_validation-3.7.0}/tests/test_discovery_parallel.py +0 -0
  111. {compass_kb_validation-3.5.0 → compass_kb_validation-3.7.0}/tests/test_embedding_drift.py +0 -0
  112. {compass_kb_validation-3.5.0 → compass_kb_validation-3.7.0}/tests/test_eval_enrich.py +0 -0
  113. {compass_kb_validation-3.5.0 → compass_kb_validation-3.7.0}/tests/test_eval_interop.py +0 -0
  114. {compass_kb_validation-3.5.0 → compass_kb_validation-3.7.0}/tests/test_evidence.py +0 -0
  115. {compass_kb_validation-3.5.0 → compass_kb_validation-3.7.0}/tests/test_gate.py +0 -0
  116. {compass_kb_validation-3.5.0 → compass_kb_validation-3.7.0}/tests/test_graph_transport.py +0 -0
  117. {compass_kb_validation-3.5.0 → compass_kb_validation-3.7.0}/tests/test_graph_validators.py +0 -0
  118. {compass_kb_validation-3.5.0 → compass_kb_validation-3.7.0}/tests/test_grounding.py +0 -0
  119. {compass_kb_validation-3.5.0 → compass_kb_validation-3.7.0}/tests/test_init_wizard.py +0 -0
  120. {compass_kb_validation-3.5.0 → compass_kb_validation-3.7.0}/tests/test_manifest.py +0 -0
  121. {compass_kb_validation-3.5.0 → compass_kb_validation-3.7.0}/tests/test_mcp.py +0 -0
  122. {compass_kb_validation-3.5.0 → compass_kb_validation-3.7.0}/tests/test_migration.py +0 -0
  123. {compass_kb_validation-3.5.0 → compass_kb_validation-3.7.0}/tests/test_models.py +0 -0
  124. {compass_kb_validation-3.5.0 → compass_kb_validation-3.7.0}/tests/test_neptune_bugfixes.py +0 -0
  125. {compass_kb_validation-3.5.0 → compass_kb_validation-3.7.0}/tests/test_neptune_bugfixes2.py +0 -0
  126. {compass_kb_validation-3.5.0 → compass_kb_validation-3.7.0}/tests/test_notifications.py +0 -0
  127. {compass_kb_validation-3.5.0 → compass_kb_validation-3.7.0}/tests/test_pipeline.py +0 -0
  128. {compass_kb_validation-3.5.0 → compass_kb_validation-3.7.0}/tests/test_profiler.py +0 -0
  129. {compass_kb_validation-3.5.0 → compass_kb_validation-3.7.0}/tests/test_providers.py +0 -0
  130. {compass_kb_validation-3.5.0 → compass_kb_validation-3.7.0}/tests/test_redis_chunks.py +0 -0
  131. {compass_kb_validation-3.5.0 → compass_kb_validation-3.7.0}/tests/test_registry.py +0 -0
  132. {compass_kb_validation-3.5.0 → compass_kb_validation-3.7.0}/tests/test_remediation.py +0 -0
  133. {compass_kb_validation-3.5.0 → compass_kb_validation-3.7.0}/tests/test_reporting.py +0 -0
  134. {compass_kb_validation-3.5.0 → compass_kb_validation-3.7.0}/tests/test_retrieval_assertions.py +0 -0
  135. {compass_kb_validation-3.5.0 → compass_kb_validation-3.7.0}/tests/test_retrieval_quality.py +0 -0
  136. {compass_kb_validation-3.5.0 → compass_kb_validation-3.7.0}/tests/test_review_hardening.py +0 -0
  137. {compass_kb_validation-3.5.0 → compass_kb_validation-3.7.0}/tests/test_run_cli.py +0 -0
  138. {compass_kb_validation-3.5.0 → compass_kb_validation-3.7.0}/tests/test_sources.py +0 -0
  139. {compass_kb_validation-3.5.0 → compass_kb_validation-3.7.0}/tests/test_storage_validator.py +0 -0
  140. {compass_kb_validation-3.5.0 → compass_kb_validation-3.7.0}/tests/test_trends.py +0 -0
  141. {compass_kb_validation-3.5.0 → compass_kb_validation-3.7.0}/tests/test_universal_concerns.py +0 -0
  142. {compass_kb_validation-3.5.0 → compass_kb_validation-3.7.0}/tests/test_utils.py +0 -0
  143. {compass_kb_validation-3.5.0 → compass_kb_validation-3.7.0}/tests/test_vectorstore_filters.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: compass-kb-validation
3
- Version: 3.5.0
3
+ Version: 3.7.0
4
4
  Summary: Knowledge-base readiness validation for RAG & AI agents — config-driven, deterministic pipeline integrity checks.
5
5
  Author: COMPASS
6
6
  License: MIT
@@ -29,6 +29,8 @@ Provides-Extra: pdf
29
29
  Requires-Dist: PyPDF2>=3.0.0; extra == "pdf"
30
30
  Provides-Extra: content
31
31
  Requires-Dist: beautifulsoup4>=4.9.0; extra == "content"
32
+ Provides-Extra: perf
33
+ Requires-Dist: numpy>=1.24; extra == "perf"
32
34
  Provides-Extra: qdrant
33
35
  Requires-Dist: qdrant-client>=1.7.0; extra == "qdrant"
34
36
  Provides-Extra: pgvector
@@ -53,6 +55,7 @@ Requires-Dist: psycopg[binary]>=3.1; extra == "all"
53
55
  Requires-Dist: pinecone>=5.0.0; extra == "all"
54
56
  Requires-Dist: weaviate-client>=4.5.0; extra == "all"
55
57
  Requires-Dist: chromadb>=0.5.0; extra == "all"
58
+ Requires-Dist: numpy>=1.24; extra == "all"
56
59
  Requires-Dist: azure-storage-blob>=12.0.0; extra == "all"
57
60
  Requires-Dist: azure-identity>=1.15.0; extra == "all"
58
61
  Requires-Dist: google-cloud-storage>=2.10.0; extra == "all"
@@ -63,6 +66,7 @@ Requires-Dist: pyflakes>=3.0.0; extra == "dev"
63
66
  Requires-Dist: pyright==1.1.409; extra == "dev"
64
67
  Requires-Dist: bandit>=1.7.5; extra == "dev"
65
68
  Requires-Dist: pip-audit>=2.6.0; extra == "dev"
69
+ Requires-Dist: numpy>=1.24; extra == "dev"
66
70
  Dynamic: license-file
67
71
 
68
72
  # COMPASS — the quality gate for your RAG knowledge base
@@ -5,4 +5,4 @@ end-to-end: API upload -> object storage -> knowledge graph -> vector store ->
5
5
  retrieval quality, plus cross-store consistency.
6
6
  """
7
7
 
8
- __version__ = "3.5.0"
8
+ __version__ = "3.7.0"
@@ -34,20 +34,60 @@ _MONTHS = {"jan", "feb", "mar", "apr", "may", "jun", "jul", "aug", "sep", "sept"
34
34
  _EXT = re.compile(r"\.(?:pdf|docx?|pptx?|xlsx?|txt|md|html?)$", re.IGNORECASE)
35
35
 
36
36
 
37
+ #: Canonical month keys, so "Jan" / "JAN" / "January" all compare equal.
38
+ _MONTH_CANON = {m: m[:3] for m in _MONTHS}
39
+ _QUARTER = re.compile(r"^(?:q[1-4]|h[12]|fy)$", re.IGNORECASE)
40
+ _YEAR = re.compile(r"^(?:19|20)\d\d$")
41
+
42
+
37
43
  def _tokens(name: str) -> list[str]:
38
- """Alphabetic, non-stopword, non-date tokens of a document name, lowercased."""
44
+ """Alphabetic, non-stopword, non-period tokens of a document name, lowercased.
45
+
46
+ Period descriptors (months, quarters, years) are excluded here because they are shared by
47
+ every issue of a periodic series and so carry no *identifying* signal - but they are NOT
48
+ discarded: :func:`_period_of` keeps them for disambiguation between issues.
49
+ """
39
50
  stem = _EXT.sub("", name or "")
40
51
  raw = re.split(r"[\s,_./\\()\[\]-]+", stem)
41
52
  return [t.lower() for t in raw
42
53
  if t and any(c.isalpha() for c in t)
43
54
  and t.lower() not in _STOP and t.lower() not in _MONTHS
44
- and not re.fullmatch(r"(?:19|20)\d\d", t)]
55
+ and not _QUARTER.match(t) and not _YEAR.match(t)]
45
56
 
46
57
 
47
- def _initialism(name: str) -> str:
48
- """First letters of a name's distinctive words ("Weekly Investment Update" -> "wiu")."""
49
- toks = _tokens(name)
50
- return "".join(t[0] for t in toks) if len(toks) >= 2 else ""
58
+ def _period_of(name: str) -> set[str]:
59
+ """The period descriptors a name carries, canonicalised ({"jan", "2026"}, {"q2"}, ...).
60
+
61
+ This is what separates the January issue of a monthly series from the July one. Stripping
62
+ it (as matching does) is right for finding the SERIES; using it is what picks the ISSUE.
63
+ """
64
+ out: set[str] = set()
65
+ for t in re.split(r"[\s,_./\\()\[\]-]+", _EXT.sub("", name or "")):
66
+ low = t.lower()
67
+ if low in _MONTH_CANON:
68
+ out.add(_MONTH_CANON[low])
69
+ elif _QUARTER.match(low) or _YEAR.match(low):
70
+ out.add(low)
71
+ return out
72
+
73
+
74
+ def _periods_conflict(a: set[str], b: set[str]) -> bool:
75
+ """True when both names name a period and they disagree - so they are different issues.
76
+
77
+ Compared per KIND (month vs quarter vs year): a citation saying "Jan 2026" and a document
78
+ named "January" agree on the month and simply do not state a year, which is not a conflict.
79
+ Only a kind both sides state, and disagree on, is one.
80
+ """
81
+ def _kind(s: set[str], pred) -> set[str]:
82
+ return {x for x in s if pred(x)}
83
+
84
+ for pred in (lambda x: x in _MONTH_CANON.values(),
85
+ lambda x: bool(_QUARTER.match(x)),
86
+ lambda x: bool(_YEAR.match(x))):
87
+ ka, kb = _kind(a, pred), _kind(b, pred)
88
+ if ka and kb and not (ka & kb):
89
+ return True
90
+ return False
51
91
 
52
92
 
53
93
  def _norm(s: str) -> str:
@@ -66,23 +106,31 @@ def _keys_for(name: str) -> set[str]:
66
106
  return keys
67
107
  keys.add(_norm(stem))
68
108
  toks = _tokens(name)
69
- init = _initialism(name)
70
- if len(init) >= 2:
71
- keys.add(init)
72
- # Prefix initialisms too: a real name usually carries a trailing qualifier the citation
73
- # drops ("Weekly Investment Update Q1" is cited as "WIU"). Length >= 3 keeps these
74
- # specific enough to be worth matching; anything shorter collides constantly and, being
75
- # ambiguous, would never resolve anyway.
76
- for i in range(3, len(toks)):
77
- keys.add("".join(t[0] for t in toks[:i]))
78
- # The distinctive words alone, without the trailing qualifier - "weeklyinvestmentupdate".
79
- for i in range(2, len(toks)):
80
- keys.add("".join(toks[:i]))
109
+ # Contiguous PREFIX and SUFFIX runs of the distinctive tokens, as both an initialism and
110
+ # the joined words. Suffix runs matter as much as prefix ones: a publisher or vendor
111
+ # prefix ("Acme Advisors - Q2 Quarterly Investment Perspective") otherwise hides the
112
+ # acronym the citation actually uses - the whole-name initialism is "btqip", never "qip".
113
+ # Internal runs are deliberately NOT generated: they multiply keys quadratically and buy
114
+ # little, since a citation nearly always names the head or the tail of a title.
115
+ for run in _runs(toks):
116
+ keys.add("".join(run)) # "quarterlyinvestmentperspective"
117
+ if len(run) >= 2:
118
+ keys.add("".join(t[0] for t in run)) # "qip"
81
119
  # A leading acronym already written in the name ("WIU - Weekly Investment Update").
82
120
  lead = re.match(r"^\s*([A-Z]{2,6})\b", stem)
83
121
  if lead:
84
122
  keys.add(lead.group(1).lower())
85
- return {k for k in keys if len(k) >= 2}
123
+ # A single generic word is never a key: "update" must not select one arbitrary document.
124
+ return {k for k in keys if len(k) >= 2} - {t for t in toks}
125
+
126
+
127
+ def _runs(toks: list[str]) -> list[list[str]]:
128
+ """Contiguous prefix and suffix runs of length >= 2 (the shapes a citation abbreviates)."""
129
+ out: list[list[str]] = []
130
+ for i in range(2, len(toks) + 1):
131
+ out.append(toks[:i]) # prefix run
132
+ out.append(toks[-i:]) # suffix run
133
+ return out
86
134
 
87
135
 
88
136
  @dataclass
@@ -112,12 +160,23 @@ class AliasMap:
112
160
  if token.lower() in low:
113
161
  return low[token.lower()], "configured alias"
114
162
  # 2. The citation token verbatim (or its distinctive head) against the derived keys.
163
+ cited_period = _period_of(token)
115
164
  for candidate in self._candidates(token):
116
165
  owners = self.by_key.get(candidate)
117
166
  if not owners:
118
167
  continue
168
+ # The period the citation names selects the ISSUE within the series. Without this,
169
+ # a "(MIS, Jan 2026)" citation could resolve to the July issue, and the January
170
+ # figure - correctly stated - would then be reported a fabrication. A verdict must
171
+ # never be manufactured by resolving to the wrong document.
172
+ if cited_period:
173
+ matching = [o for o in owners if not _periods_conflict(cited_period, _period_of(o))]
174
+ if not matching:
175
+ return None, (f"'{candidate}' matches {len(owners)} KB document(s), but none "
176
+ f"is the {'/'.join(sorted(cited_period))} issue - not resolved")
177
+ owners = matching
119
178
  if len(owners) == 1:
120
- return owners[0], _why(candidate, owners[0])
179
+ return owners[0], _why(candidate, owners[0], cited_period)
121
180
  return None, (f"'{candidate}' matches {len(owners)} KB documents "
122
181
  f"({', '.join(owners[:3])}) - ambiguous, not resolved")
123
182
  return None, "no KB document name derives this citation token"
@@ -144,15 +203,15 @@ class AliasMap:
144
203
  return out
145
204
 
146
205
 
147
- def _why(key: str, name: str) -> str:
206
+ def _why(key: str, name: str, cited_period: Optional[set] = None) -> str:
148
207
  """Explain which property of the KB document name the citation token matched."""
208
+ period = (f", selected by period {'/'.join(sorted(cited_period))}"
209
+ if cited_period and _period_of(name) else "")
149
210
  if key == _norm(name) or key == _norm(_EXT.sub("", name)):
150
- return "citation token matches the KB document name"
151
- toks = _tokens(name)
152
- if key == "".join(t[0] for t in toks) or key in {"".join(t[0] for t in toks[:i])
153
- for i in range(2, len(toks) + 1)}:
154
- return f"'{key.upper()}' is the initialism of KB document '{name}'"
155
- return f"citation token maps to KB document '{name}'"
211
+ return f"citation token matches the KB document name{period}"
212
+ if key in {"".join(t[0] for t in run) for run in _runs(_tokens(name))}:
213
+ return f"'{key.upper()}' is the initialism of KB document '{name}'{period}"
214
+ return f"citation token maps to KB document '{name}'{period}"
156
215
 
157
216
 
158
217
  def build_alias_map(adapter: Any, fields: list[str], configured: Optional[dict] = None,
@@ -40,11 +40,35 @@ def parse_filters(spec: str) -> dict:
40
40
  return out
41
41
 
42
42
 
43
+ def _citation_fields(settings: Settings) -> list[str]:
44
+ """Index fields the citation/grounding config already names.
45
+
46
+ A retrieved chunk that does not carry its document title and page cannot be matched to a
47
+ citation, so Level-1 grounding verification silently reports UNVERIFIABLE for every claim.
48
+ Rather than ask the user to restate those field names under a second config key, derive
49
+ them from the ``eval.gt_*`` / ``eval.citation_*`` settings that already declare them: if
50
+ the project configured grounding at all, retrieval returns what grounding needs.
51
+ Requesting a field the index lacks is harmless - RediSearch simply omits it.
52
+ """
53
+ ev = settings.eval
54
+ fields = [
55
+ getattr(ev, "gt_chunk_doc_field", ""), # a chunk's source-document title
56
+ getattr(ev, "gt_chunk_page_field", ""), # a chunk's page, for page-level matching
57
+ getattr(ev, "gt_index_page_field", ""), # the index-side page field, if named differently
58
+ getattr(ev, "gt_doc_name_field", ""), # the document-name field used for L2 lookups
59
+ ]
60
+ fields += list(getattr(ev, "citation_doc_fields", []) or [])
61
+ fields += list(getattr(ev, "citation_source_key_fields", []) or []) # Level-3 source tracing
62
+ return [f for f in fields if f]
63
+
64
+
43
65
  def _return_fields(settings: Settings) -> list[str]:
44
66
  rc = settings.redis
45
- # Base identity/text + the project's configured metadata fields (empty by default;
46
- # harmless if the index lacks any), so results can feed export / comparison.
67
+ # Base identity/text, the fields grounding needs to match a citation, and the project's
68
+ # configured metadata fields (empty by default; harmless if the index lacks any), so
69
+ # results can feed export / comparison.
47
70
  wanted = (["file_name", "text", rc.document_id_field, rc.positional_field, "s3_key"]
71
+ + _citation_fields(settings)
48
72
  + list(settings.eval.metadata_fields))
49
73
  seen: set[str] = set()
50
74
  out: list[str] = []
@@ -0,0 +1,345 @@
1
+ """Retrieval-strategy benchmarking: which way through this KB finds the right evidence?
2
+
3
+ `discover` proves the KB is intact. It says nothing about whether the retrieval PATH through
4
+ it is any good - so a team choosing between pure vector search, hybrid, or wider-then-narrower
5
+ has no evidence, guesses, deploys, and then blames the KB when answers are poor. Usually the
6
+ KB is fine and the retrieval configuration is wrong.
7
+
8
+ This runs an eval set's questions through two or more retrieval configurations against the
9
+ SAME validated KB and scores each one with standard information-retrieval metrics against the
10
+ ground-truth chunks the eval already carries. No LLM anywhere: a chunk is relevant because the
11
+ ground truth says so, not because a model judged it. Deterministic and repeatable - same eval
12
+ set and same KB give the same answer every time.
13
+
14
+ What it cannot do is invent ground truth. A case with no ground-truth chunks is EXCLUDED from
15
+ the metrics rather than scored zero: "we have nothing to compare against" and "retrieval found
16
+ nothing" are different findings, and averaging the first into the second would understate every
17
+ strategy equally and invisibly.
18
+ """
19
+
20
+ from __future__ import annotations
21
+
22
+ import math
23
+ from dataclasses import dataclass, field
24
+ from typing import Any, Optional
25
+
26
+ DEFAULT_TOP_K = 10
27
+
28
+
29
+ # --------------------------------------------------------------------------- #
30
+ # Information-retrieval metrics (binary relevance)
31
+ # --------------------------------------------------------------------------- #
32
+ def recall_at_k(retrieved: list[str], relevant: set, k: int) -> Optional[float]:
33
+ """Share of the relevant chunks that appear in the top ``k``. None if nothing is relevant."""
34
+ if not relevant:
35
+ return None
36
+ found = len({r for r in retrieved[:k] if r in relevant})
37
+ return found / len(relevant)
38
+
39
+
40
+ def precision_at_k(retrieved: list[str], relevant: set, k: int) -> Optional[float]:
41
+ """Share of the top ``k`` that is relevant.
42
+
43
+ The denominator is k, not len(retrieved): a strategy that returns 3 results when asked for
44
+ 10 has not earned the precision of one that returned 3 out of 3, and dividing by the
45
+ smaller number would flatter it for retrieving too little.
46
+ """
47
+ if not relevant or k <= 0:
48
+ return None
49
+ found = len({r for r in retrieved[:k] if r in relevant})
50
+ return found / k
51
+
52
+
53
+ def reciprocal_rank(retrieved: list[str], relevant: set) -> Optional[float]:
54
+ """1 / rank of the first relevant chunk; 0.0 when none was retrieved."""
55
+ if not relevant:
56
+ return None
57
+ for i, chunk in enumerate(retrieved, 1):
58
+ if chunk in relevant:
59
+ return 1.0 / i
60
+ return 0.0
61
+
62
+
63
+ def ndcg_at_k(retrieved: list[str], relevant: set, k: int) -> Optional[float]:
64
+ """Normalised discounted cumulative gain with binary relevance.
65
+
66
+ Rewards putting relevant chunks EARLY, which recall alone cannot see: two strategies can
67
+ both find the evidence while one buries it at rank 9 where a top-3 context window will
68
+ never include it.
69
+ """
70
+ if not relevant or k <= 0:
71
+ return None
72
+ dcg = sum(1.0 / math.log2(i + 1) for i, chunk in enumerate(retrieved[:k], 1)
73
+ if chunk in relevant)
74
+ ideal = sum(1.0 / math.log2(i + 1) for i in range(1, min(len(relevant), k) + 1))
75
+ return (dcg / ideal) if ideal else None
76
+
77
+
78
+ def hit_rate(retrieved: list[str], relevant: set, k: int) -> Optional[float]:
79
+ """1.0 when at least one relevant chunk is in the top ``k`` - did retrieval find anything?"""
80
+ if not relevant:
81
+ return None
82
+ return 1.0 if any(r in relevant for r in retrieved[:k]) else 0.0
83
+
84
+
85
+ METRICS = ("recall", "precision", "mrr", "ndcg", "hit_rate")
86
+
87
+
88
+ def score_one(retrieved: list[str], relevant: set, k: int) -> dict:
89
+ """Every metric for one query. Values are None when the query has no ground truth."""
90
+ return {
91
+ "recall": recall_at_k(retrieved, relevant, k),
92
+ "precision": precision_at_k(retrieved, relevant, k),
93
+ "mrr": reciprocal_rank(retrieved, relevant),
94
+ "ndcg": ndcg_at_k(retrieved, relevant, k),
95
+ "hit_rate": hit_rate(retrieved, relevant, k),
96
+ }
97
+
98
+
99
+ def _mean(values: list) -> Optional[float]:
100
+ nums = [v for v in values if isinstance(v, (int, float)) and not isinstance(v, bool)]
101
+ return round(sum(nums) / len(nums), 4) if nums else None
102
+
103
+
104
+ # --------------------------------------------------------------------------- #
105
+ # Strategies
106
+ # --------------------------------------------------------------------------- #
107
+ @dataclass
108
+ class Strategy:
109
+ """One retrieval configuration to compare. Everything comes from project YAML."""
110
+
111
+ name: str
112
+ type: str = "vector" # vector | hybrid
113
+ top_k: int = DEFAULT_TOP_K
114
+ filters: dict = field(default_factory=dict)
115
+ expand_context: int = 0 # include N neighbouring chunks around each hit
116
+ rerank_top_k: int = 0 # retrieve wider, then keep this many
117
+ extra: dict = field(default_factory=dict)
118
+
119
+ @classmethod
120
+ def from_config(cls, raw: dict) -> "Strategy":
121
+ known = {"name", "type", "top_k", "filters", "expand_context", "rerank_top_k"}
122
+ return cls(name=str(raw.get("name") or raw.get("type") or "strategy"),
123
+ type=str(raw.get("type") or "vector"),
124
+ top_k=int(raw.get("top_k") or DEFAULT_TOP_K),
125
+ filters=dict(raw.get("filters") or {}),
126
+ expand_context=int(raw.get("expand_context") or 0),
127
+ rerank_top_k=int(raw.get("rerank_top_k") or 0),
128
+ extra={k: v for k, v in raw.items() if k not in known})
129
+
130
+ @property
131
+ def effective_k(self) -> int:
132
+ """The number of chunks a consumer actually sees after any narrowing step."""
133
+ return self.rerank_top_k or self.top_k
134
+
135
+ def unsupported(self) -> str:
136
+ """Why this strategy cannot run here, or "" when it can.
137
+
138
+ A strategy naming a reranker is refused rather than silently run WITHOUT it: reporting
139
+ "vector-rerank: recall 0.63" for a run that never reranked would attribute the plain
140
+ vector score to a configuration nobody tested.
141
+ """
142
+ if self.extra.get("reranker"):
143
+ return (f"'{self.extra['reranker']}' reranking is not built in - COMPASS ships no "
144
+ "cross-encoder provider, so this strategy cannot be measured here. Remove "
145
+ "the reranker to benchmark the retrieval it wraps.")
146
+ if self.type not in ("vector", "hybrid"):
147
+ return f"unknown strategy type '{self.type}' (expected 'vector' or 'hybrid')"
148
+ return ""
149
+
150
+
151
+ def chunk_identity(chunk: dict, doc_field: str, page_field: str) -> str:
152
+ """A stable identity for matching a retrieved chunk to a ground-truth one.
153
+
154
+ Document + page, because that is the granularity ground truth is written at and the only
155
+ identifier both sides reliably share - a store's internal chunk key never appears in an
156
+ eval file.
157
+ """
158
+ doc = str(chunk.get(doc_field) or chunk.get("file_name") or chunk.get("document_title") or "")
159
+ page = str(chunk.get(page_field) or chunk.get("page_start") or "")
160
+ return f"{doc.strip().lower()}#{page.strip()}" if doc else ""
161
+
162
+
163
+ def ground_truth_ids(case: dict, cfg: Any) -> set:
164
+ """The chunks a case's ground truth says are relevant."""
165
+ from compass.eval_interop import get_path
166
+
167
+ doc_field = getattr(cfg, "gt_chunk_doc_field", "document_title")
168
+ page_field = getattr(cfg, "gt_chunk_page_field", "page_start")
169
+ found = get_path(case, getattr(cfg, "gt_chunks_path", "context.chunks"))
170
+ chunks: list = found if isinstance(found, list) else []
171
+ if not chunks and isinstance(case.get("chunks"), list):
172
+ chunks = case["chunks"]
173
+ out = {chunk_identity(c, doc_field, page_field) for c in chunks if isinstance(c, dict)}
174
+ return {i for i in out if i}
175
+
176
+
177
+ def run_strategy(settings: Any, question: str, strategy: Strategy, *, embedder: Any = None,
178
+ vectorstore: Any = None) -> list[str]:
179
+ """Retrieve for one question under one strategy; return chunk identities in rank order."""
180
+ from compass.ask import ask_kb
181
+
182
+ payload = ask_kb(settings, question, top_k=max(strategy.top_k, strategy.effective_k),
183
+ filters=strategy.filters or None, embedder=embedder,
184
+ vectorstore=vectorstore)
185
+ doc_field = getattr(settings.eval, "gt_chunk_doc_field", "document_title")
186
+ page_field = getattr(settings.eval, "gt_chunk_page_field", "page_start")
187
+ ids: list[str] = []
188
+ for result in payload.get("results", []):
189
+ merged = {**(result.get("meta") or {}), **result}
190
+ identity = chunk_identity(merged, doc_field, page_field)
191
+ if identity and identity not in ids:
192
+ ids.append(identity)
193
+ return ids[: strategy.effective_k] if strategy.rerank_top_k else ids
194
+
195
+
196
+ # --------------------------------------------------------------------------- #
197
+ # Benchmark run
198
+ # --------------------------------------------------------------------------- #
199
+ def benchmark(settings: Any, cases: list[dict], strategies: list[Strategy], *,
200
+ retrieve: Any = None) -> dict:
201
+ """Score every strategy over every case. ``retrieve`` is injectable for testing.
202
+
203
+ Cases with no ground truth are excluded and counted separately - scoring them zero would
204
+ penalise every strategy identically for a gap in the EVAL SET, and hide that the number is
205
+ based on fewer queries than the reader assumes.
206
+ """
207
+ from compass.eval_interop import get_path
208
+
209
+ q_path = getattr(settings.eval, "gt_question_path", "question")
210
+ runner = retrieve or (lambda q, s: run_strategy(settings, q, s))
211
+
212
+ usable, skipped = [], []
213
+ for case in cases:
214
+ question = get_path(case, q_path) or case.get("input") or case.get("question")
215
+ relevant = ground_truth_ids(case, settings.eval)
216
+ if not question or not relevant:
217
+ skipped.append(str(case.get("case_id") or case.get("test_id") or "?"))
218
+ continue
219
+ usable.append((case, str(question), relevant))
220
+
221
+ per_strategy: list[dict] = []
222
+ per_query: dict[str, dict] = {}
223
+ for strategy in strategies:
224
+ blocked = strategy.unsupported()
225
+ if blocked:
226
+ per_strategy.append({"strategy": strategy.name, "skipped": True, "reason": blocked})
227
+ continue
228
+ scores: dict[str, list] = {m: [] for m in METRICS}
229
+ for case, question, relevant in usable:
230
+ cid = str(case.get("case_id") or case.get("test_id") or question[:40])
231
+ retrieved = runner(question, strategy)
232
+ result = score_one(retrieved, relevant, strategy.effective_k)
233
+ for metric, value in result.items():
234
+ scores[metric].append(value)
235
+ per_query.setdefault(cid, {"question": question,
236
+ "ground_truth": len(relevant)})[strategy.name] = {
237
+ "found": len([r for r in retrieved[:strategy.effective_k] if r in relevant]),
238
+ "recall": result["recall"], "mrr": result["mrr"]}
239
+ per_strategy.append({
240
+ "strategy": strategy.name, "skipped": False, "top_k": strategy.effective_k,
241
+ "queries": len(usable),
242
+ **{metric: _mean(values) for metric, values in scores.items()},
243
+ })
244
+
245
+ scored = [s for s in per_strategy if not s.get("skipped")]
246
+ winner = max(scored, key=lambda s: (s.get("ndcg") or 0, s.get("recall") or 0), default=None)
247
+ return {
248
+ "project": getattr(settings, "project_name", "") or "",
249
+ "queries": len(usable),
250
+ "excluded_cases": skipped,
251
+ "strategies": per_strategy,
252
+ "winner": winner["strategy"] if winner else "",
253
+ "per_query": per_query,
254
+ "method": ("binary-relevance IR metrics against the eval's own ground-truth chunks, "
255
+ "matched on document + page. No LLM: relevance comes from the ground truth, "
256
+ "not from a model's judgement."),
257
+ }
258
+
259
+
260
+ def worst_queries(report: dict, limit: int = 5) -> list[dict]:
261
+ """Queries where the best and worst strategies disagree most - where the choice matters."""
262
+ rows = []
263
+ for cid, data in report.get("per_query", {}).items():
264
+ recalls: list[tuple[str, float]] = [
265
+ (str(name), float(v["recall"])) for name, v in data.items()
266
+ if isinstance(v, dict) and isinstance(v.get("recall"), (int, float))]
267
+ if len(recalls) < 2:
268
+ continue
269
+ best = max(recalls, key=lambda t: t[1])
270
+ worst = min(recalls, key=lambda t: t[1])
271
+ if best[1] > worst[1]:
272
+ rows.append({"case_id": cid, "question": data.get("question", ""),
273
+ "best": best[0], "best_recall": best[1],
274
+ "worst": worst[0], "worst_recall": worst[1],
275
+ "spread": round(best[1] - worst[1], 4)})
276
+ return sorted(rows, key=lambda r: -r["spread"])[:limit]
277
+
278
+
279
+ def format_report(report: dict) -> str:
280
+ """Console table plus the queries where the strategy choice actually matters."""
281
+ L = ["=" * 72,
282
+ f" COMPASS benchmark - {report.get('project') or 'KB'} "
283
+ f"({report['queries']} queries, {len(report['strategies'])} strategies)",
284
+ "=" * 72, ""]
285
+ if not report["queries"]:
286
+ L.append(" No case carried both a question and ground-truth chunks, so nothing could "
287
+ "be measured.")
288
+ L.append(" Check eval.gt_question_path and eval.gt_chunks_path.")
289
+ L.append("=" * 72)
290
+ return "\n".join(L)
291
+
292
+ head = f" {'Strategy':<20}{'Recall':>9}{'Precision':>11}{'MRR':>8}{'NDCG':>8}{'Hit rate':>10}"
293
+ L += [head, " " + "-" * (len(head) - 2)]
294
+ for s in report["strategies"]:
295
+ if s.get("skipped"):
296
+ L.append(f" {s['strategy']:<20} SKIPPED - {s['reason']}")
297
+ continue
298
+ L.append(f" {s['strategy']:<20}"
299
+ + "".join(f"{_fmt(s.get(m)):>{w}}" for m, w in
300
+ (("recall", 9), ("precision", 11), ("mrr", 8), ("ndcg", 8),
301
+ ("hit_rate", 10))))
302
+ L.append("")
303
+
304
+ scored = [s for s in report["strategies"] if not s.get("skipped")]
305
+ if report.get("winner") and len(scored) > 1:
306
+ win = next(s for s in scored if s["strategy"] == report["winner"])
307
+ base = min(scored, key=lambda s: (s.get("ndcg") or 0))
308
+ L.append(f" WINNER: {win['strategy']}")
309
+ for metric, label in (("recall", "recall"), ("mrr", "MRR"), ("ndcg", "NDCG")):
310
+ a, b = win.get(metric), base.get(metric)
311
+ if isinstance(a, (int, float)) and isinstance(b, (int, float)) and b:
312
+ L.append(f" {(a - b) / b:+.0%} {label} vs {base['strategy']}")
313
+ L.append("")
314
+
315
+ worst = worst_queries(report)
316
+ if worst:
317
+ L.append(" WHERE THE CHOICE MATTERS (largest disagreement between strategies):")
318
+ for row in worst:
319
+ L.append(f" {row['case_id']}: \"{row['question'][:58]}\"")
320
+ L.append(f" {row['best']}: recall {row['best_recall']:.2f} "
321
+ f"{row['worst']}: recall {row['worst_recall']:.2f}")
322
+ L.append("")
323
+ if report["excluded_cases"]:
324
+ L.append(f" {len(report['excluded_cases'])} case(s) excluded - no question or no "
325
+ "ground-truth chunks to measure against, so they are not scored as misses.")
326
+ L.append(" Method: " + report["method"])
327
+ L.append("=" * 72)
328
+ return "\n".join(L)
329
+
330
+
331
+ def _fmt(value: Any) -> str:
332
+ return f"{value:.2f}" if isinstance(value, (int, float)) else "-"
333
+
334
+
335
+ def strategies_from_settings(settings: Any) -> list[Strategy]:
336
+ """Read ``benchmark.strategies`` from project config."""
337
+ raw = getattr(getattr(settings, "benchmark", None), "strategies", None)
338
+ if raw is None:
339
+ raw = (getattr(settings, "extra", {}) or {}).get("benchmark", {}).get("strategies", [])
340
+ return [Strategy.from_config(s) for s in (raw or []) if isinstance(s, dict)]
341
+
342
+
343
+ __all__ = ["Strategy", "benchmark", "format_report", "score_one", "recall_at_k",
344
+ "precision_at_k", "reciprocal_rank", "ndcg_at_k", "hit_rate", "ground_truth_ids",
345
+ "chunk_identity", "worst_queries", "strategies_from_settings", "run_strategy"]