dandelion-memory 1.0.0rc15__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (232) hide show
  1. benchmarks/__init__.py +3 -0
  2. benchmarks/adapters/__init__.py +29 -0
  3. benchmarks/adapters/ablations.py +406 -0
  4. benchmarks/adapters/base.py +72 -0
  5. benchmarks/adapters/controlled/__init__.py +13 -0
  6. benchmarks/adapters/controlled/core_archival.py +115 -0
  7. benchmarks/adapters/controlled/extract_resolve.py +130 -0
  8. benchmarks/adapters/controlled/hybrid_rag.py +120 -0
  9. benchmarks/adapters/controlled/no_late_binding.py +113 -0
  10. benchmarks/adapters/dandelion.py +94 -0
  11. benchmarks/adapters/native/__init__.py +31 -0
  12. benchmarks/adapters/native/base.py +45 -0
  13. benchmarks/adapters/native/dandelion_native.py +164 -0
  14. benchmarks/adapters/native/dandelion_native_b0.py +165 -0
  15. benchmarks/adapters/native/dandelion_native_b1.py +170 -0
  16. benchmarks/adapters/native/dandelion_native_b2.py +169 -0
  17. benchmarks/adapters/native/dandelion_native_b3.py +171 -0
  18. benchmarks/adapters/native/dandelion_native_c4.py +261 -0
  19. benchmarks/adapters/native/dandelion_native_c4_1.py +281 -0
  20. benchmarks/adapters/native/dandelion_native_c4_2.py +435 -0
  21. benchmarks/adapters/native/event_evidence_b2.py +616 -0
  22. benchmarks/adapters/native/evidence_b2.py +609 -0
  23. benchmarks/adapters/native/letta_native.py +253 -0
  24. benchmarks/adapters/native/mem0_native.py +219 -0
  25. benchmarks/adapters/native/store_vector_b0.py +792 -0
  26. benchmarks/adapters/native/store_vector_b2.py +808 -0
  27. benchmarks/agent_adapter_stdio_acceptance.py +86 -0
  28. benchmarks/bootstrap_cross_stack.py +218 -0
  29. benchmarks/canonical_profile_instrumentation.py +167 -0
  30. benchmarks/canonical_public_baseline.py +288 -0
  31. benchmarks/cli.py +525 -0
  32. benchmarks/collector_public30_offline.py +1653 -0
  33. benchmarks/datasets/__init__.py +45 -0
  34. benchmarks/datasets/conversational_generator.py +238 -0
  35. benchmarks/datasets/generator.py +283 -0
  36. benchmarks/datasets/holdout_topology_generator.py +290 -0
  37. benchmarks/datasets/holdout_topology_generator_v2.py +294 -0
  38. benchmarks/datasets/multi_world.py +1643 -0
  39. benchmarks/datasets/scale_generator.py +289 -0
  40. benchmarks/datasets/schema.py +57 -0
  41. benchmarks/deep_collision_stress.py +199 -0
  42. benchmarks/evaluators/__init__.py +12 -0
  43. benchmarks/evaluators/manifest.py +142 -0
  44. benchmarks/evaluators/metrics.py +277 -0
  45. benchmarks/evaluators/runner.py +258 -0
  46. benchmarks/f1_step0/build_dev_measurement_manifest_v1.py +127 -0
  47. benchmarks/f1_step0/build_dev_measurement_manifest_v2.py +127 -0
  48. benchmarks/f1_step0/build_dev_measurement_manifest_v3.py +146 -0
  49. benchmarks/f1_step0/build_ledger.py +143 -0
  50. benchmarks/f1_step0/build_ledger_v2.py +296 -0
  51. benchmarks/f1_step0/compare_dev52_readiness_v1.py +106 -0
  52. benchmarks/f1_step0/locator_public_smoke.py +172 -0
  53. benchmarks/f1_step0/prepare_dev52_owner_v1.py +133 -0
  54. benchmarks/f1_step0/prepare_dev52_owner_v2.py +133 -0
  55. benchmarks/f1_step0/public_identity_probe.py +191 -0
  56. benchmarks/f1_step0/public_response_mapper.py +101 -0
  57. benchmarks/f1_step0/real_source_public_preflight.py +244 -0
  58. benchmarks/f1_step0/run_exact_fusion_dev_experiment_v1.py +83 -0
  59. benchmarks/f1_step0/scorer.py +110 -0
  60. benchmarks/f1_step0/source_aware_dev_measurement_v1.py +627 -0
  61. benchmarks/f1_step0/source_aware_dev_measurement_v2.py +659 -0
  62. benchmarks/f1_step0/source_aware_dev_measurement_v3.py +776 -0
  63. benchmarks/f1_step0/test_public_response_mapper.py +110 -0
  64. benchmarks/f1_step0/test_real_source_public_preflight.py +25 -0
  65. benchmarks/f1_step0/test_scorer.py +158 -0
  66. benchmarks/f1_step0/test_source_aware_dev_measurement.py +189 -0
  67. benchmarks/f1_step0/test_source_aware_dev_measurement_v2.py +263 -0
  68. benchmarks/f1_step0/test_source_aware_dev_measurement_v3.py +334 -0
  69. benchmarks/f1_step0/typed_readback_public_smoke.py +181 -0
  70. benchmarks/forensic_c4_1_dev.py +258 -0
  71. benchmarks/forensic_c4_dev.py +255 -0
  72. benchmarks/forensic_deep25_c4_1.py +478 -0
  73. benchmarks/legacy_migration_acceptance.py +29 -0
  74. benchmarks/longmemeval_actor.py +497 -0
  75. benchmarks/longmemeval_agent_judge.py +593 -0
  76. benchmarks/longmemeval_budget.py +139 -0
  77. benchmarks/longmemeval_full.py +410 -0
  78. benchmarks/longmemeval_judge.py +327 -0
  79. benchmarks/longmemeval_pilot.py +316 -0
  80. benchmarks/longmemeval_s.py +318 -0
  81. benchmarks/memory_bench.py +248 -0
  82. benchmarks/memory_formation/__init__.py +1 -0
  83. benchmarks/memory_formation/audit.py +801 -0
  84. benchmarks/memory_formation/real_audit.py +688 -0
  85. benchmarks/memory_session_acceptance.py +279 -0
  86. benchmarks/native_codex_host.py +513 -0
  87. benchmarks/owner_dense_acceptance.py +126 -0
  88. benchmarks/pilot_seeker_vault.py +266 -0
  89. benchmarks/prepare_public30_ripgrep_range_owner.py +115 -0
  90. benchmarks/profile_canonical.py +373 -0
  91. benchmarks/profile_canonical_lanes.py +405 -0
  92. benchmarks/profile_search_chunks.py +351 -0
  93. benchmarks/public_ranges_faq_proof.py +121 -0
  94. benchmarks/public_repos/build_history_scope_v3.py +330 -0
  95. benchmarks/public_repos/build_oracle.py +108 -0
  96. benchmarks/public_repos/build_oracle_v2.py +169 -0
  97. benchmarks/public_repos/build_oracle_v3.py +176 -0
  98. benchmarks/public_repos/map_public_v2.py +296 -0
  99. benchmarks/public_repos/project_prompts_v2.py +31 -0
  100. benchmarks/public_repos/project_prompts_v3.py +29 -0
  101. benchmarks/public_repos/ready_admin_v2.py +214 -0
  102. benchmarks/public_repos/score_public_v2.py +258 -0
  103. benchmarks/public_repos/test_map_public_v2.py +171 -0
  104. benchmarks/public_repos/test_public_repo_v2.py +44 -0
  105. benchmarks/public_repos/test_public_repo_v3.py +46 -0
  106. benchmarks/public_repos/test_ready_admin_v2.py +117 -0
  107. benchmarks/public_repos/test_score_public_v2.py +163 -0
  108. benchmarks/public_repos/test_wire_v2.py +292 -0
  109. benchmarks/public_repos/wire_v2.py +603 -0
  110. benchmarks/real_blocker_fix_acceptance.py +155 -0
  111. benchmarks/real_scan_rescan_acceptance.py +41 -0
  112. benchmarks/recall_v2_20k_smoke.py +112 -0
  113. benchmarks/recall_v2_e1_retrieve.py +70 -0
  114. benchmarks/recall_v2_e1_score.py +271 -0
  115. benchmarks/recall_v2_e2_prepare.py +69 -0
  116. benchmarks/recall_v2_e2_public_score.py +91 -0
  117. benchmarks/recall_v2_e2_score.py +108 -0
  118. benchmarks/recall_v2_navigation_mcp_replay.py +308 -0
  119. benchmarks/recall_v2_reader_capture.py +118 -0
  120. benchmarks/recall_v2_reader_replay.py +136 -0
  121. benchmarks/run_blind_h10_holdout.py +284 -0
  122. benchmarks/run_blind_h9_holdout.py +275 -0
  123. benchmarks/run_c4_2_nightly_scale.py +446 -0
  124. benchmarks/run_context_scale_simulation.py +189 -0
  125. benchmarks/run_conversational_bench.py +170 -0
  126. benchmarks/run_h11_evaluation.py +316 -0
  127. benchmarks/run_h11_v2_evaluation.py +431 -0
  128. benchmarks/run_native_fixed12.py +469 -0
  129. benchmarks/run_native_preflight.py +280 -0
  130. benchmarks/run_scale_stress.py +225 -0
  131. benchmarks/solanazation_brownfield_01/build_oracle.py +970 -0
  132. benchmarks/solanazation_brownfield_01/check_level_a.py +78 -0
  133. benchmarks/solanazation_brownfield_01/debug_scope.py +13 -0
  134. benchmarks/solanazation_brownfield_01/evaluate_bootstrap_02.py +570 -0
  135. benchmarks/solanazation_brownfield_01/evaluate_first_pass.py +306 -0
  136. benchmarks/solanazation_brownfield_01/generate_dryrun_report.py +222 -0
  137. benchmarks/solanazation_brownfield_01/generate_manifests.py +146 -0
  138. benchmarks/solanazation_brownfield_01/inspect_misses.py +29 -0
  139. benchmarks/solanazation_brownfield_01/run_miss_forensic.py +206 -0
  140. benchmarks/solanazation_brownfield_01/test_facts.py +15 -0
  141. benchmarks/statistics/__init__.py +15 -0
  142. benchmarks/statistics/significance.py +326 -0
  143. benchmarks/test_native_mcp_preflight.py +229 -0
  144. benchmarks/test_prepare_public30_range_owner.py +35 -0
  145. benchmarks/test_public30_source_reference.py +698 -0
  146. benchmarks/test_recall_v2_e1_reader_input.py +89 -0
  147. benchmarks/test_recall_v2_navigation_mcp_replay.py +155 -0
  148. benchmarks/test_recall_v2_reader_replay.py +41 -0
  149. dandelion/__init__.py +3 -0
  150. dandelion/__main__.py +11 -0
  151. dandelion/agent_adapter.py +2369 -0
  152. dandelion/bootstrap/__init__.py +872 -0
  153. dandelion/bootstrap/alias_policy.py +50 -0
  154. dandelion/bootstrap/authority.py +225 -0
  155. dandelion/bootstrap/canonical_history.py +412 -0
  156. dandelion/bootstrap/classifier.py +403 -0
  157. dandelion/bootstrap/component_graph.py +280 -0
  158. dandelion/bootstrap/discovery.py +279 -0
  159. dandelion/bootstrap/domain_induction.py +180 -0
  160. dandelion/bootstrap/domain_topology.py +673 -0
  161. dandelion/bootstrap/evidence_envelope.py +132 -0
  162. dandelion/bootstrap/extractors/base.py +171 -0
  163. dandelion/bootstrap/extractors/exact.py +1234 -0
  164. dandelion/bootstrap/extractors/generic.py +100 -0
  165. dandelion/bootstrap/extractors/git_temporal.py +111 -0
  166. dandelion/bootstrap/extractors/registry.py +345 -0
  167. dandelion/bootstrap/extractors/relations.py +89 -0
  168. dandelion/bootstrap/extractors/semantic.py +424 -0
  169. dandelion/bootstrap/extractors/status.py +218 -0
  170. dandelion/bootstrap/git_sweeper.py +135 -0
  171. dandelion/bootstrap/manifest.py +246 -0
  172. dandelion/bootstrap/quarantine.py +264 -0
  173. dandelion/bootstrap/report.py +259 -0
  174. dandelion/bootstrap/root_resolver.py +221 -0
  175. dandelion/bootstrap/tech_profiler.py +125 -0
  176. dandelion/bootstrap/topology.py +75 -0
  177. dandelion/bootstrap/topology_index.py +110 -0
  178. dandelion/bootstrap/transaction.py +315 -0
  179. dandelion/canonical_bootstrap.py +257 -0
  180. dandelion/canonical_memory.py +4130 -0
  181. dandelion/captured_input.py +295 -0
  182. dandelion/cli.py +767 -0
  183. dandelion/config.py +40 -0
  184. dandelion/derived_index.py +745 -0
  185. dandelion/domains.py +706 -0
  186. dandelion/embedder.py +229 -0
  187. dandelion/event_evidence.py +571 -0
  188. dandelion/evidence.py +812 -0
  189. dandelion/evidence_response.py +453 -0
  190. dandelion/evidence_workspace.py +2360 -0
  191. dandelion/exact_context.py +129 -0
  192. dandelion/hydrator.py +124 -0
  193. dandelion/inspector.py +442 -0
  194. dandelion/inspector_assets/dandelion-mark.svg +33 -0
  195. dandelion/inspector_assets/dandelion_graph.js +361 -0
  196. dandelion/inspector_assets/inspector.css +223 -0
  197. dandelion/inspector_assets/inspector.html +135 -0
  198. dandelion/inspector_assets/inspector.js +1059 -0
  199. dandelion/legacy_import.py +431 -0
  200. dandelion/live_source_grants.py +133 -0
  201. dandelion/local_owner.py +114 -0
  202. dandelion/memory_client.py +252 -0
  203. dandelion/memory_contract.py +107 -0
  204. dandelion/memory_owner.py +560 -0
  205. dandelion/reader_providers.py +118 -0
  206. dandelion/recall_v2.py +767 -0
  207. dandelion/reconciler.py +219 -0
  208. dandelion/request_budget.py +125 -0
  209. dandelion/retained_basis.py +223 -0
  210. dandelion/retrieval.py +485 -0
  211. dandelion/router.py +405 -0
  212. dandelion/search_terms.py +53 -0
  213. dandelion/security.py +124 -0
  214. dandelion/server.py +810 -0
  215. dandelion/source_content_fts.py +219 -0
  216. dandelion/source_delivery.py +89 -0
  217. dandelion/source_history.py +241 -0
  218. dandelion/source_identity.py +88 -0
  219. dandelion/source_navigator.py +5 -0
  220. dandelion/source_reader.py +444 -0
  221. dandelion/source_references.py +379 -0
  222. dandelion/source_sections.py +265 -0
  223. dandelion/store_kv.py +684 -0
  224. dandelion/store_vector.py +1289 -0
  225. dandelion/studio.py +325 -0
  226. dandelion/wal.py +227 -0
  227. dandelion_memory-1.0.0rc15.dist-info/METADATA +48 -0
  228. dandelion_memory-1.0.0rc15.dist-info/RECORD +232 -0
  229. dandelion_memory-1.0.0rc15.dist-info/WHEEL +5 -0
  230. dandelion_memory-1.0.0rc15.dist-info/entry_points.txt +5 -0
  231. dandelion_memory-1.0.0rc15.dist-info/licenses/LICENSE +21 -0
  232. dandelion_memory-1.0.0rc15.dist-info/top_level.txt +2 -0
@@ -0,0 +1,435 @@
1
+ """Dandelion Native Benchmark Adapter: Candidate C4.2 (Typed Multi-Source Local Retrieval).
2
+
3
+ Architectural Evolution (C4.1 -> C4.2):
4
+ 1. Typed Multi-Source Local Retrieval:
5
+ - Source A (Dense Semantic): vector search in Qdrant within resolved partition(s).
6
+ - Source B (Local Lexical): token overlap within resolved partition(s) via partition-scoped inverted index.
7
+ - Source C (Exact Fact-Key / Attribute Alignment): fact-key segment index within resolved partition(s).
8
+ 2. Candidate Union & Deduplication:
9
+ - Evaluates Dense ∪ Local Lexical ∪ Exact Ref before RRF fusion.
10
+ - Eliminates intra-partition single-source candidate starvation.
11
+ 3. Candidate Source Attribution Telemetry:
12
+ - Classifies candidates into: dense_only, lexical_only, exact_only, dense+lexical, lexical+exact, dense+exact, all_three.
13
+ 4. Deterministic Evidenced Ranking Tuple:
14
+ - (exact_attribute_alignment, rrf_score, lexical_hits, vector_score, stable_event_id).
15
+ 5. Strict Invariants:
16
+ - Zero runtime LLM calls.
17
+ - Candidate pool strictly bounded (Gate 4: <= 500 unique chunks).
18
+ - Zero new architectural planes (no KG, no external router, no extra memory store).
19
+ """
20
+
21
+ from __future__ import annotations
22
+
23
+ import re
24
+ import time
25
+ from collections import Counter, defaultdict
26
+ from typing import Any, Optional
27
+
28
+ import dandelion.store_kv
29
+ from dandelion.config import COLLECTION_NAME
30
+ from dandelion.domains import (
31
+ PartitionDescriptor,
32
+ PartitionPath,
33
+ UNSCOPED_LEGACY,
34
+ extract_component_root,
35
+ )
36
+ from dandelion.embedder import Embedder
37
+ from dandelion.evidence import (
38
+ ScopeResolutionStatus,
39
+ ScopeResolver,
40
+ SupportLevel,
41
+ extract_memory_evidence,
42
+ extract_query_evidence,
43
+ resolve_evidence,
44
+ )
45
+ from dandelion.hydrator import hydrate_template, pin_snapshots
46
+ from dandelion.router import EpistemicProgressiveWidener, PartitionRouterIndex, ScopeLevel
47
+ from dandelion.store_kv import KVStore
48
+ from dandelion.store_vector import (
49
+ QUERY_STOP_WORDS,
50
+ TECHNICAL_CODE_ID,
51
+ VectorStore,
52
+ )
53
+ from qdrant_client import models
54
+
55
+ from benchmarks.adapters.native.dandelion_native_c4_1 import DandelionNativeAdapterC41
56
+
57
+
58
+ class DandelionNativeAdapterC42(DandelionNativeAdapterC41):
59
+ """Benchmark adapter evaluating Candidate C4.2 (Typed Multi-Source Local Retrieval)."""
60
+
61
+ def __init__(self) -> None:
62
+ super().__init__()
63
+ self.chunk_data: dict[str, dict[str, Any]] = {}
64
+ self.partition_words: dict[str, dict[str, set[str]]] = defaultdict(lambda: defaultdict(set))
65
+ self.partition_fact_keys: dict[str, dict[str, set[str]]] = defaultdict(lambda: defaultdict(set))
66
+ self.last_query_attribution: Counter[str] = Counter()
67
+ self.total_attribution_stats: Counter[str] = Counter()
68
+ self.winning_attribution_stats: Counter[str] = Counter()
69
+
70
+ @property
71
+ def system_info(self) -> dict[str, Any]:
72
+ return {
73
+ "name": "dandelion_native_c4_2",
74
+ "version": "0.4.2",
75
+ "architecture": "typed_multi_source_local_retrieval_over_partition_trees",
76
+ "candidate": "C4.2",
77
+ "embedding_model": "sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2",
78
+ "llm_used": "none (deterministic symbolic hydration)",
79
+ "vector_store": "qdrant_in_memory_partitioned",
80
+ "hydration_policy": "read_time_late_binding",
81
+ "security_policy": "deny_by_default_secret_masking",
82
+ "routing_policy": "structural_path_authority_with_descriptor_discovery",
83
+ "candidate_sources": ["dense", "local_lexical", "exact_fact_key_segments"],
84
+ }
85
+
86
+ @property
87
+ def telemetry(self) -> dict[str, Any]:
88
+ avg_tok = (self.total_tokens / max(1, self.query_count)) if self.query_count else 0
89
+ return {
90
+ "query_count": self.query_count,
91
+ "llm_calls_total": 0,
92
+ "llm_calls_per_query": 0.0,
93
+ "api_calls_total": 0,
94
+ "api_calls_per_query": 0.0,
95
+ "cost_total_usd": 0.0,
96
+ "cost_per_1k_queries_usd": 0.0,
97
+ "mean_memory_context_tokens": round(avg_tok, 1),
98
+ "last_widening_telemetry": self.widener.last_telemetry,
99
+ "candidate_source_attribution_total": dict(self.total_attribution_stats),
100
+ "winning_candidate_source_attribution": dict(self.winning_attribution_stats),
101
+ }
102
+
103
+ def reset(self) -> None:
104
+ """Reset internal stores, partition router, and multi-source indexes."""
105
+ super().reset()
106
+ self.chunk_data.clear()
107
+ self.partition_words.clear()
108
+ self.partition_fact_keys.clear()
109
+ self.last_query_attribution.clear()
110
+ self.total_attribution_stats.clear()
111
+ self.winning_attribution_stats.clear()
112
+
113
+ def ingest_memory(
114
+ self,
115
+ event_id: str,
116
+ template: str,
117
+ domain: str,
118
+ timestamp: float,
119
+ supersedes: Optional[str] = None,
120
+ source: str = "tool_observed",
121
+ ) -> str:
122
+ """Ingest episodic memory and update partition-scoped lexical and fact-key indexes."""
123
+ point_id = super().ingest_memory(
124
+ event_id=event_id,
125
+ template=template,
126
+ domain=domain,
127
+ timestamp=timestamp,
128
+ supersedes=supersedes,
129
+ source=source,
130
+ )
131
+ ppath = PartitionPath.from_domain(domain)
132
+ pid = ppath.canonical_id
133
+
134
+ # 1. Lexical index: non-stop words
135
+ tmpl_clean = template.lower()
136
+ words = {
137
+ w.strip(".:,-/?!\"'(){}[]")
138
+ for w in re.split(r"[\s\-._/]+", tmpl_clean)
139
+ if len(w) > 2 and w not in QUERY_STOP_WORDS
140
+ }
141
+ for w in words:
142
+ self.partition_words[pid][w].add(point_id)
143
+
144
+ # 2. Fact-key segment index: referenced facts
145
+ ref_keys = re.findall(r"\{\{ref:([\w.]+)(?:@rev:\d+)?\}\}", template)
146
+ for r in ref_keys:
147
+ for seg in re.split(r"[._\-]+", r):
148
+ seg_clean = seg.lower().strip()
149
+ if len(seg_clean) > 2 and seg_clean not in QUERY_STOP_WORDS:
150
+ self.partition_fact_keys[pid][seg_clean].add(point_id)
151
+
152
+ self.chunk_data[point_id] = {
153
+ "id": point_id,
154
+ "event_id": event_id,
155
+ "template": template,
156
+ "domain": domain,
157
+ "partition_id": pid,
158
+ "created_at": timestamp,
159
+ "source": source,
160
+ "referenced_facts": ref_keys,
161
+ "words": words,
162
+ }
163
+ return point_id
164
+
165
+ def recall(
166
+ self,
167
+ query: str,
168
+ domain: Optional[str] = None,
169
+ limit: int = 3,
170
+ ) -> list[dict[str, Any]]:
171
+ """Typed Multi-Source hierarchical scope-routed retrieval across partitions."""
172
+ self.query_count += 1
173
+ self.last_query_attribution.clear()
174
+
175
+ # 1. Structural Scope Resolution Gate
176
+ scope = ScopeResolver.resolve(
177
+ query=query,
178
+ retrieval_scope=domain,
179
+ registry=self.kv.get_domain_registry(),
180
+ )
181
+ if scope.status == ScopeResolutionStatus.EXPLICIT_UNRESOLVED:
182
+ return []
183
+
184
+ # Enforce asserted scope conflict
185
+ if scope.status == ScopeResolutionStatus.RESOLVED and domain:
186
+ retrieval_root = extract_component_root(domain)
187
+ domain_parts = set(p.lower() for p in domain.replace("/", ".").split(".") if p)
188
+ is_match = (
189
+ scope.asserted_scope == retrieval_root
190
+ or (scope.asserted_scope and scope.asserted_scope.lower() in domain_parts)
191
+ or domain.lower().startswith((scope.asserted_scope or "").lower())
192
+ or (scope.asserted_scope or "").lower().startswith(retrieval_root.lower())
193
+ )
194
+ if not is_match:
195
+ return []
196
+
197
+ # Effective explicit scope
198
+ effective_scope = domain or (
199
+ scope.raw_asserted_text
200
+ if (scope.raw_asserted_text and ("." in scope.raw_asserted_text or "/" in scope.raw_asserted_text))
201
+ else scope.asserted_scope
202
+ )
203
+
204
+ domain_tokens = set(re.findall(r"[\w]+", (effective_scope or domain or "").lower()))
205
+ raw_query_words = {
206
+ w.lower().strip(".:,-/?!\"'(){}[]")
207
+ for w in re.split(r"[\s\-._/]+", query)
208
+ if len(w) > 2 and w.lower() not in QUERY_STOP_WORDS
209
+ }
210
+ q_words = {w for w in raw_query_words if w not in domain_tokens}
211
+ query_idents = {
212
+ ident.strip(".:,-/")
213
+ for ident in TECHNICAL_CODE_ID.findall(query)
214
+ if ident.lower() not in QUERY_STOP_WORDS
215
+ }
216
+ match_tokens = q_words | query_idents
217
+
218
+ embedder = Embedder.get()
219
+ q_vec = embedder.embed_query(query)
220
+ q_ev = extract_query_evidence(
221
+ query=query,
222
+ domain=effective_scope or domain,
223
+ known_domains=self.vec._known_domains,
224
+ registry=self.kv.get_domain_registry(),
225
+ )
226
+
227
+ # 2. Local retrieval function: Typed Multi-Source Local Retrieval inside proposed partitions
228
+ def _multi_source_local_retrieval(pids: list[str]) -> list[dict[str, Any]]:
229
+ # Source A: Dense Semantic Candidates (limit=20)
230
+ dense_filter = models.Filter(
231
+ must=[
232
+ models.FieldCondition(key="is_active", match=models.MatchValue(value=True)),
233
+ models.FieldCondition(
234
+ key="partition_id",
235
+ match=models.MatchAny(any=pids) if len(pids) > 1 else models.MatchValue(value=pids[0]),
236
+ ),
237
+ ]
238
+ )
239
+ q_res = self.vec._client.query_points(
240
+ collection_name=COLLECTION_NAME,
241
+ query=q_vec,
242
+ query_filter=dense_filter,
243
+ limit=max(limit * 5, 20),
244
+ with_payload=True,
245
+ )
246
+ dense_ranks: dict[str, int] = {}
247
+ dense_scores: dict[str, float] = {}
248
+ for r_idx, pt in enumerate(q_res.points):
249
+ pid_str = str(pt.id)
250
+ dense_ranks[pid_str] = r_idx + 1
251
+ dense_scores[pid_str] = float(pt.score or 0.0)
252
+
253
+ # Source B: Local Lexical Candidates (limit=20)
254
+ lexical_counts: Counter[str] = Counter()
255
+ for pid in pids:
256
+ w_map = self.partition_words.get(pid, {})
257
+ for qw in q_words:
258
+ if qw in w_map:
259
+ for cid in w_map[qw]:
260
+ lexical_counts[cid] += 1
261
+ lexical_ranks: dict[str, int] = {}
262
+ for l_idx, (cid, cnt) in enumerate(lexical_counts.most_common(max(limit * 5, 20))):
263
+ if cnt > 0:
264
+ lexical_ranks[cid] = l_idx + 1
265
+
266
+ # Source C: Exact Fact-Key Segment Alignment Candidates (limit=20)
267
+ exact_counts: Counter[str] = Counter()
268
+ for pid in pids:
269
+ fk_map = self.partition_fact_keys.get(pid, {})
270
+ for mt in match_tokens:
271
+ if mt in fk_map:
272
+ for cid in fk_map[mt]:
273
+ exact_counts[cid] += 1
274
+ exact_ranks: dict[str, int] = {}
275
+ for e_idx, (cid, cnt) in enumerate(exact_counts.most_common(max(limit * 5, 20))):
276
+ if cnt > 0:
277
+ exact_ranks[cid] = e_idx + 1
278
+
279
+ # Candidate Union & Deduplication (Bounded <= 60 chunks per partition set)
280
+ all_candidate_ids = set(dense_ranks.keys()) | set(lexical_ranks.keys()) | set(exact_ranks.keys())
281
+
282
+ candidates: list[dict[str, Any]] = []
283
+ K = 60
284
+
285
+ for cid in all_candidate_ids:
286
+ in_d = cid in dense_ranks
287
+ in_l = cid in lexical_ranks
288
+ in_e = cid in exact_ranks
289
+
290
+ # Attribution classification
291
+ if in_d and in_l and in_e:
292
+ attribution = "all_three"
293
+ elif in_d and in_l:
294
+ attribution = "dense+lexical"
295
+ elif in_l and in_e:
296
+ attribution = "lexical+exact"
297
+ elif in_d and in_e:
298
+ attribution = "dense+exact"
299
+ elif in_d:
300
+ attribution = "dense_only"
301
+ elif in_l:
302
+ attribution = "lexical_only"
303
+ elif in_e:
304
+ attribution = "exact_only"
305
+ else:
306
+ attribution = "unknown"
307
+
308
+ self.last_query_attribution[attribution] += 1
309
+ self.total_attribution_stats[attribution] += 1
310
+
311
+ item = self.chunk_data.get(cid)
312
+ if not item:
313
+ continue
314
+
315
+ tmpl = item["template"]
316
+ dom = item["domain"]
317
+
318
+ # RRF channel contributions
319
+ d_r = dense_ranks.get(cid, 9999)
320
+ l_r = lexical_ranks.get(cid, 9999)
321
+ e_r = exact_ranks.get(cid, 9999)
322
+
323
+ rrf_dense = (1.0 / (K + d_r)) if in_d else 0.0
324
+ rrf_lex = (1.0 / (K + l_r)) if in_l else 0.0
325
+ rrf_exact = (1.0 / (K + e_r)) if in_e else 0.0
326
+
327
+ v_score = dense_scores.get(cid, 0.0)
328
+
329
+ # Evidence Gate
330
+ m_ev = extract_memory_evidence(
331
+ domain=dom,
332
+ template=tmpl,
333
+ referenced_facts=item.get("referenced_facts", []),
334
+ event_id=item.get("event_id", cid),
335
+ )
336
+ supp_level, ev_score, reason = resolve_evidence(
337
+ query_ev=q_ev,
338
+ mem_ev=m_ev,
339
+ vector_score=v_score,
340
+ is_fact_boosted=in_e,
341
+ )
342
+
343
+ if supp_level in (SupportLevel.UNSUPPORTED, SupportLevel.WEAK):
344
+ continue
345
+
346
+ rrf_evidence = (2.0 / (K + 1)) if supp_level == SupportLevel.STRONG else (1.0 / (K + 1))
347
+
348
+ # Exact attribute alignment count
349
+ chunk_refs = item.get("referenced_facts", [])
350
+ fact_key_segments: set[str] = set()
351
+ for r in chunk_refs:
352
+ for seg in re.split(r"[._\-]+", r):
353
+ s_clean = seg.lower().strip()
354
+ if len(s_clean) > 2 and s_clean not in QUERY_STOP_WORDS:
355
+ fact_key_segments.add(s_clean)
356
+ attr_match_count = len(match_tokens & fact_key_segments)
357
+
358
+ score = (rrf_dense + rrf_lex + rrf_exact + rrf_evidence) * 30.0
359
+
360
+ cand_record = dict(item)
361
+ cand_record.update({
362
+ "score": round(score, 4),
363
+ "vector_score": v_score,
364
+ "attr_match_count": attr_match_count,
365
+ "dense_rank": d_r if in_d else None,
366
+ "lexical_rank": l_r if in_l else None,
367
+ "exact_rank": e_r if in_e else None,
368
+ "lexical_hits": lexical_counts.get(cid, 0),
369
+ "attribution": attribution,
370
+ "support_level": supp_level.value,
371
+ "evidence_score": round(ev_score, 4),
372
+ "evidence_reason": reason,
373
+ })
374
+ candidates.append(cand_record)
375
+
376
+ # Deterministic Ranking Tuple:
377
+ # 1. Exact attribute alignment (structural discriminant)
378
+ # 2. RRF fused score (dense + lexical + exact fusion)
379
+ # 3. Raw lexical hits
380
+ # 4. Dense cosine similarity
381
+ # 5. Stable event_id tie-breaker
382
+ candidates.sort(
383
+ key=lambda x: (
384
+ x.get("attr_match_count", 0),
385
+ x["score"],
386
+ x.get("lexical_hits", 0),
387
+ x["vector_score"],
388
+ x.get("event_id", x["id"]),
389
+ ),
390
+ reverse=True,
391
+ )
392
+ return candidates
393
+
394
+ # 3. Epistemic progressive widening cascade (L0 -> L1 -> L2 -> L3 -> L4 bounded)
395
+ admitted, _ = self.widener.widen_and_retrieve(
396
+ query=query,
397
+ retrieval_fn=_multi_source_local_retrieval,
398
+ explicit_scope=effective_scope,
399
+ max_results=limit,
400
+ candidate_budget=500,
401
+ )
402
+
403
+ # Track winning candidate attribution
404
+ if admitted:
405
+ win_attr = admitted[0].get("attribution", "unknown")
406
+ self.winning_attribution_stats[win_attr] += 1
407
+
408
+ # 4. Late-binding fact hydration on admitted candidates
409
+ hydrated: list[dict[str, Any]] = []
410
+ for r in admitted:
411
+ text = hydrate_template(
412
+ text=r["template"],
413
+ kv_resolver=self.kv.resolve_ref,
414
+ chunk_domain=r.get("domain", "general"),
415
+ allow_secrets=False,
416
+ )
417
+ hydrated.append({
418
+ "id": r["id"],
419
+ "event_id": r.get("event_id", r["id"]),
420
+ "text": text,
421
+ "domain": r.get("domain", "general"),
422
+ "partition_id": r.get("partition_id", UNSCOPED_LEGACY.canonical_id),
423
+ "score": r.get("score", 0.0),
424
+ "evidence_score": r.get("evidence_score", 0.0),
425
+ "vector_score": r.get("vector_score", 0.0),
426
+ "attr_match_count": r.get("attr_match_count", 0),
427
+ "attribution": r.get("attribution", "unknown"),
428
+ "has_fact": bool(r.get("exact_rank") is not None),
429
+ "has_attr_match": bool(r.get("attr_match_count", 0) > 0),
430
+ "is_historical": "{{ref:" in r["template"] and "@rev:" in r["template"],
431
+ })
432
+
433
+ tok_sum = sum(len(h["text"].split()) for h in hydrated)
434
+ self.total_tokens += tok_sum
435
+ return hydrated