dandelion-memory 1.0.0rc15__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- benchmarks/__init__.py +3 -0
- benchmarks/adapters/__init__.py +29 -0
- benchmarks/adapters/ablations.py +406 -0
- benchmarks/adapters/base.py +72 -0
- benchmarks/adapters/controlled/__init__.py +13 -0
- benchmarks/adapters/controlled/core_archival.py +115 -0
- benchmarks/adapters/controlled/extract_resolve.py +130 -0
- benchmarks/adapters/controlled/hybrid_rag.py +120 -0
- benchmarks/adapters/controlled/no_late_binding.py +113 -0
- benchmarks/adapters/dandelion.py +94 -0
- benchmarks/adapters/native/__init__.py +31 -0
- benchmarks/adapters/native/base.py +45 -0
- benchmarks/adapters/native/dandelion_native.py +164 -0
- benchmarks/adapters/native/dandelion_native_b0.py +165 -0
- benchmarks/adapters/native/dandelion_native_b1.py +170 -0
- benchmarks/adapters/native/dandelion_native_b2.py +169 -0
- benchmarks/adapters/native/dandelion_native_b3.py +171 -0
- benchmarks/adapters/native/dandelion_native_c4.py +261 -0
- benchmarks/adapters/native/dandelion_native_c4_1.py +281 -0
- benchmarks/adapters/native/dandelion_native_c4_2.py +435 -0
- benchmarks/adapters/native/event_evidence_b2.py +616 -0
- benchmarks/adapters/native/evidence_b2.py +609 -0
- benchmarks/adapters/native/letta_native.py +253 -0
- benchmarks/adapters/native/mem0_native.py +219 -0
- benchmarks/adapters/native/store_vector_b0.py +792 -0
- benchmarks/adapters/native/store_vector_b2.py +808 -0
- benchmarks/agent_adapter_stdio_acceptance.py +86 -0
- benchmarks/bootstrap_cross_stack.py +218 -0
- benchmarks/canonical_profile_instrumentation.py +167 -0
- benchmarks/canonical_public_baseline.py +288 -0
- benchmarks/cli.py +525 -0
- benchmarks/collector_public30_offline.py +1653 -0
- benchmarks/datasets/__init__.py +45 -0
- benchmarks/datasets/conversational_generator.py +238 -0
- benchmarks/datasets/generator.py +283 -0
- benchmarks/datasets/holdout_topology_generator.py +290 -0
- benchmarks/datasets/holdout_topology_generator_v2.py +294 -0
- benchmarks/datasets/multi_world.py +1643 -0
- benchmarks/datasets/scale_generator.py +289 -0
- benchmarks/datasets/schema.py +57 -0
- benchmarks/deep_collision_stress.py +199 -0
- benchmarks/evaluators/__init__.py +12 -0
- benchmarks/evaluators/manifest.py +142 -0
- benchmarks/evaluators/metrics.py +277 -0
- benchmarks/evaluators/runner.py +258 -0
- benchmarks/f1_step0/build_dev_measurement_manifest_v1.py +127 -0
- benchmarks/f1_step0/build_dev_measurement_manifest_v2.py +127 -0
- benchmarks/f1_step0/build_dev_measurement_manifest_v3.py +146 -0
- benchmarks/f1_step0/build_ledger.py +143 -0
- benchmarks/f1_step0/build_ledger_v2.py +296 -0
- benchmarks/f1_step0/compare_dev52_readiness_v1.py +106 -0
- benchmarks/f1_step0/locator_public_smoke.py +172 -0
- benchmarks/f1_step0/prepare_dev52_owner_v1.py +133 -0
- benchmarks/f1_step0/prepare_dev52_owner_v2.py +133 -0
- benchmarks/f1_step0/public_identity_probe.py +191 -0
- benchmarks/f1_step0/public_response_mapper.py +101 -0
- benchmarks/f1_step0/real_source_public_preflight.py +244 -0
- benchmarks/f1_step0/run_exact_fusion_dev_experiment_v1.py +83 -0
- benchmarks/f1_step0/scorer.py +110 -0
- benchmarks/f1_step0/source_aware_dev_measurement_v1.py +627 -0
- benchmarks/f1_step0/source_aware_dev_measurement_v2.py +659 -0
- benchmarks/f1_step0/source_aware_dev_measurement_v3.py +776 -0
- benchmarks/f1_step0/test_public_response_mapper.py +110 -0
- benchmarks/f1_step0/test_real_source_public_preflight.py +25 -0
- benchmarks/f1_step0/test_scorer.py +158 -0
- benchmarks/f1_step0/test_source_aware_dev_measurement.py +189 -0
- benchmarks/f1_step0/test_source_aware_dev_measurement_v2.py +263 -0
- benchmarks/f1_step0/test_source_aware_dev_measurement_v3.py +334 -0
- benchmarks/f1_step0/typed_readback_public_smoke.py +181 -0
- benchmarks/forensic_c4_1_dev.py +258 -0
- benchmarks/forensic_c4_dev.py +255 -0
- benchmarks/forensic_deep25_c4_1.py +478 -0
- benchmarks/legacy_migration_acceptance.py +29 -0
- benchmarks/longmemeval_actor.py +497 -0
- benchmarks/longmemeval_agent_judge.py +593 -0
- benchmarks/longmemeval_budget.py +139 -0
- benchmarks/longmemeval_full.py +410 -0
- benchmarks/longmemeval_judge.py +327 -0
- benchmarks/longmemeval_pilot.py +316 -0
- benchmarks/longmemeval_s.py +318 -0
- benchmarks/memory_bench.py +248 -0
- benchmarks/memory_formation/__init__.py +1 -0
- benchmarks/memory_formation/audit.py +801 -0
- benchmarks/memory_formation/real_audit.py +688 -0
- benchmarks/memory_session_acceptance.py +279 -0
- benchmarks/native_codex_host.py +513 -0
- benchmarks/owner_dense_acceptance.py +126 -0
- benchmarks/pilot_seeker_vault.py +266 -0
- benchmarks/prepare_public30_ripgrep_range_owner.py +115 -0
- benchmarks/profile_canonical.py +373 -0
- benchmarks/profile_canonical_lanes.py +405 -0
- benchmarks/profile_search_chunks.py +351 -0
- benchmarks/public_ranges_faq_proof.py +121 -0
- benchmarks/public_repos/build_history_scope_v3.py +330 -0
- benchmarks/public_repos/build_oracle.py +108 -0
- benchmarks/public_repos/build_oracle_v2.py +169 -0
- benchmarks/public_repos/build_oracle_v3.py +176 -0
- benchmarks/public_repos/map_public_v2.py +296 -0
- benchmarks/public_repos/project_prompts_v2.py +31 -0
- benchmarks/public_repos/project_prompts_v3.py +29 -0
- benchmarks/public_repos/ready_admin_v2.py +214 -0
- benchmarks/public_repos/score_public_v2.py +258 -0
- benchmarks/public_repos/test_map_public_v2.py +171 -0
- benchmarks/public_repos/test_public_repo_v2.py +44 -0
- benchmarks/public_repos/test_public_repo_v3.py +46 -0
- benchmarks/public_repos/test_ready_admin_v2.py +117 -0
- benchmarks/public_repos/test_score_public_v2.py +163 -0
- benchmarks/public_repos/test_wire_v2.py +292 -0
- benchmarks/public_repos/wire_v2.py +603 -0
- benchmarks/real_blocker_fix_acceptance.py +155 -0
- benchmarks/real_scan_rescan_acceptance.py +41 -0
- benchmarks/recall_v2_20k_smoke.py +112 -0
- benchmarks/recall_v2_e1_retrieve.py +70 -0
- benchmarks/recall_v2_e1_score.py +271 -0
- benchmarks/recall_v2_e2_prepare.py +69 -0
- benchmarks/recall_v2_e2_public_score.py +91 -0
- benchmarks/recall_v2_e2_score.py +108 -0
- benchmarks/recall_v2_navigation_mcp_replay.py +308 -0
- benchmarks/recall_v2_reader_capture.py +118 -0
- benchmarks/recall_v2_reader_replay.py +136 -0
- benchmarks/run_blind_h10_holdout.py +284 -0
- benchmarks/run_blind_h9_holdout.py +275 -0
- benchmarks/run_c4_2_nightly_scale.py +446 -0
- benchmarks/run_context_scale_simulation.py +189 -0
- benchmarks/run_conversational_bench.py +170 -0
- benchmarks/run_h11_evaluation.py +316 -0
- benchmarks/run_h11_v2_evaluation.py +431 -0
- benchmarks/run_native_fixed12.py +469 -0
- benchmarks/run_native_preflight.py +280 -0
- benchmarks/run_scale_stress.py +225 -0
- benchmarks/solanazation_brownfield_01/build_oracle.py +970 -0
- benchmarks/solanazation_brownfield_01/check_level_a.py +78 -0
- benchmarks/solanazation_brownfield_01/debug_scope.py +13 -0
- benchmarks/solanazation_brownfield_01/evaluate_bootstrap_02.py +570 -0
- benchmarks/solanazation_brownfield_01/evaluate_first_pass.py +306 -0
- benchmarks/solanazation_brownfield_01/generate_dryrun_report.py +222 -0
- benchmarks/solanazation_brownfield_01/generate_manifests.py +146 -0
- benchmarks/solanazation_brownfield_01/inspect_misses.py +29 -0
- benchmarks/solanazation_brownfield_01/run_miss_forensic.py +206 -0
- benchmarks/solanazation_brownfield_01/test_facts.py +15 -0
- benchmarks/statistics/__init__.py +15 -0
- benchmarks/statistics/significance.py +326 -0
- benchmarks/test_native_mcp_preflight.py +229 -0
- benchmarks/test_prepare_public30_range_owner.py +35 -0
- benchmarks/test_public30_source_reference.py +698 -0
- benchmarks/test_recall_v2_e1_reader_input.py +89 -0
- benchmarks/test_recall_v2_navigation_mcp_replay.py +155 -0
- benchmarks/test_recall_v2_reader_replay.py +41 -0
- dandelion/__init__.py +3 -0
- dandelion/__main__.py +11 -0
- dandelion/agent_adapter.py +2369 -0
- dandelion/bootstrap/__init__.py +872 -0
- dandelion/bootstrap/alias_policy.py +50 -0
- dandelion/bootstrap/authority.py +225 -0
- dandelion/bootstrap/canonical_history.py +412 -0
- dandelion/bootstrap/classifier.py +403 -0
- dandelion/bootstrap/component_graph.py +280 -0
- dandelion/bootstrap/discovery.py +279 -0
- dandelion/bootstrap/domain_induction.py +180 -0
- dandelion/bootstrap/domain_topology.py +673 -0
- dandelion/bootstrap/evidence_envelope.py +132 -0
- dandelion/bootstrap/extractors/base.py +171 -0
- dandelion/bootstrap/extractors/exact.py +1234 -0
- dandelion/bootstrap/extractors/generic.py +100 -0
- dandelion/bootstrap/extractors/git_temporal.py +111 -0
- dandelion/bootstrap/extractors/registry.py +345 -0
- dandelion/bootstrap/extractors/relations.py +89 -0
- dandelion/bootstrap/extractors/semantic.py +424 -0
- dandelion/bootstrap/extractors/status.py +218 -0
- dandelion/bootstrap/git_sweeper.py +135 -0
- dandelion/bootstrap/manifest.py +246 -0
- dandelion/bootstrap/quarantine.py +264 -0
- dandelion/bootstrap/report.py +259 -0
- dandelion/bootstrap/root_resolver.py +221 -0
- dandelion/bootstrap/tech_profiler.py +125 -0
- dandelion/bootstrap/topology.py +75 -0
- dandelion/bootstrap/topology_index.py +110 -0
- dandelion/bootstrap/transaction.py +315 -0
- dandelion/canonical_bootstrap.py +257 -0
- dandelion/canonical_memory.py +4130 -0
- dandelion/captured_input.py +295 -0
- dandelion/cli.py +767 -0
- dandelion/config.py +40 -0
- dandelion/derived_index.py +745 -0
- dandelion/domains.py +706 -0
- dandelion/embedder.py +229 -0
- dandelion/event_evidence.py +571 -0
- dandelion/evidence.py +812 -0
- dandelion/evidence_response.py +453 -0
- dandelion/evidence_workspace.py +2360 -0
- dandelion/exact_context.py +129 -0
- dandelion/hydrator.py +124 -0
- dandelion/inspector.py +442 -0
- dandelion/inspector_assets/dandelion-mark.svg +33 -0
- dandelion/inspector_assets/dandelion_graph.js +361 -0
- dandelion/inspector_assets/inspector.css +223 -0
- dandelion/inspector_assets/inspector.html +135 -0
- dandelion/inspector_assets/inspector.js +1059 -0
- dandelion/legacy_import.py +431 -0
- dandelion/live_source_grants.py +133 -0
- dandelion/local_owner.py +114 -0
- dandelion/memory_client.py +252 -0
- dandelion/memory_contract.py +107 -0
- dandelion/memory_owner.py +560 -0
- dandelion/reader_providers.py +118 -0
- dandelion/recall_v2.py +767 -0
- dandelion/reconciler.py +219 -0
- dandelion/request_budget.py +125 -0
- dandelion/retained_basis.py +223 -0
- dandelion/retrieval.py +485 -0
- dandelion/router.py +405 -0
- dandelion/search_terms.py +53 -0
- dandelion/security.py +124 -0
- dandelion/server.py +810 -0
- dandelion/source_content_fts.py +219 -0
- dandelion/source_delivery.py +89 -0
- dandelion/source_history.py +241 -0
- dandelion/source_identity.py +88 -0
- dandelion/source_navigator.py +5 -0
- dandelion/source_reader.py +444 -0
- dandelion/source_references.py +379 -0
- dandelion/source_sections.py +265 -0
- dandelion/store_kv.py +684 -0
- dandelion/store_vector.py +1289 -0
- dandelion/studio.py +325 -0
- dandelion/wal.py +227 -0
- dandelion_memory-1.0.0rc15.dist-info/METADATA +48 -0
- dandelion_memory-1.0.0rc15.dist-info/RECORD +232 -0
- dandelion_memory-1.0.0rc15.dist-info/WHEEL +5 -0
- dandelion_memory-1.0.0rc15.dist-info/entry_points.txt +5 -0
- dandelion_memory-1.0.0rc15.dist-info/licenses/LICENSE +21 -0
- dandelion_memory-1.0.0rc15.dist-info/top_level.txt +2 -0
|
@@ -0,0 +1,435 @@
|
|
|
1
|
+
"""Dandelion Native Benchmark Adapter: Candidate C4.2 (Typed Multi-Source Local Retrieval).
|
|
2
|
+
|
|
3
|
+
Architectural Evolution (C4.1 -> C4.2):
|
|
4
|
+
1. Typed Multi-Source Local Retrieval:
|
|
5
|
+
- Source A (Dense Semantic): vector search in Qdrant within resolved partition(s).
|
|
6
|
+
- Source B (Local Lexical): token overlap within resolved partition(s) via partition-scoped inverted index.
|
|
7
|
+
- Source C (Exact Fact-Key / Attribute Alignment): fact-key segment index within resolved partition(s).
|
|
8
|
+
2. Candidate Union & Deduplication:
|
|
9
|
+
- Evaluates Dense ∪ Local Lexical ∪ Exact Ref before RRF fusion.
|
|
10
|
+
- Eliminates intra-partition single-source candidate starvation.
|
|
11
|
+
3. Candidate Source Attribution Telemetry:
|
|
12
|
+
- Classifies candidates into: dense_only, lexical_only, exact_only, dense+lexical, lexical+exact, dense+exact, all_three.
|
|
13
|
+
4. Deterministic Evidenced Ranking Tuple:
|
|
14
|
+
- (exact_attribute_alignment, rrf_score, lexical_hits, vector_score, stable_event_id).
|
|
15
|
+
5. Strict Invariants:
|
|
16
|
+
- Zero runtime LLM calls.
|
|
17
|
+
- Candidate pool strictly bounded (Gate 4: <= 500 unique chunks).
|
|
18
|
+
- Zero new architectural planes (no KG, no external router, no extra memory store).
|
|
19
|
+
"""
|
|
20
|
+
|
|
21
|
+
from __future__ import annotations
|
|
22
|
+
|
|
23
|
+
import re
|
|
24
|
+
import time
|
|
25
|
+
from collections import Counter, defaultdict
|
|
26
|
+
from typing import Any, Optional
|
|
27
|
+
|
|
28
|
+
import dandelion.store_kv
|
|
29
|
+
from dandelion.config import COLLECTION_NAME
|
|
30
|
+
from dandelion.domains import (
|
|
31
|
+
PartitionDescriptor,
|
|
32
|
+
PartitionPath,
|
|
33
|
+
UNSCOPED_LEGACY,
|
|
34
|
+
extract_component_root,
|
|
35
|
+
)
|
|
36
|
+
from dandelion.embedder import Embedder
|
|
37
|
+
from dandelion.evidence import (
|
|
38
|
+
ScopeResolutionStatus,
|
|
39
|
+
ScopeResolver,
|
|
40
|
+
SupportLevel,
|
|
41
|
+
extract_memory_evidence,
|
|
42
|
+
extract_query_evidence,
|
|
43
|
+
resolve_evidence,
|
|
44
|
+
)
|
|
45
|
+
from dandelion.hydrator import hydrate_template, pin_snapshots
|
|
46
|
+
from dandelion.router import EpistemicProgressiveWidener, PartitionRouterIndex, ScopeLevel
|
|
47
|
+
from dandelion.store_kv import KVStore
|
|
48
|
+
from dandelion.store_vector import (
|
|
49
|
+
QUERY_STOP_WORDS,
|
|
50
|
+
TECHNICAL_CODE_ID,
|
|
51
|
+
VectorStore,
|
|
52
|
+
)
|
|
53
|
+
from qdrant_client import models
|
|
54
|
+
|
|
55
|
+
from benchmarks.adapters.native.dandelion_native_c4_1 import DandelionNativeAdapterC41
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
class DandelionNativeAdapterC42(DandelionNativeAdapterC41):
|
|
59
|
+
"""Benchmark adapter evaluating Candidate C4.2 (Typed Multi-Source Local Retrieval)."""
|
|
60
|
+
|
|
61
|
+
def __init__(self) -> None:
|
|
62
|
+
super().__init__()
|
|
63
|
+
self.chunk_data: dict[str, dict[str, Any]] = {}
|
|
64
|
+
self.partition_words: dict[str, dict[str, set[str]]] = defaultdict(lambda: defaultdict(set))
|
|
65
|
+
self.partition_fact_keys: dict[str, dict[str, set[str]]] = defaultdict(lambda: defaultdict(set))
|
|
66
|
+
self.last_query_attribution: Counter[str] = Counter()
|
|
67
|
+
self.total_attribution_stats: Counter[str] = Counter()
|
|
68
|
+
self.winning_attribution_stats: Counter[str] = Counter()
|
|
69
|
+
|
|
70
|
+
@property
|
|
71
|
+
def system_info(self) -> dict[str, Any]:
|
|
72
|
+
return {
|
|
73
|
+
"name": "dandelion_native_c4_2",
|
|
74
|
+
"version": "0.4.2",
|
|
75
|
+
"architecture": "typed_multi_source_local_retrieval_over_partition_trees",
|
|
76
|
+
"candidate": "C4.2",
|
|
77
|
+
"embedding_model": "sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2",
|
|
78
|
+
"llm_used": "none (deterministic symbolic hydration)",
|
|
79
|
+
"vector_store": "qdrant_in_memory_partitioned",
|
|
80
|
+
"hydration_policy": "read_time_late_binding",
|
|
81
|
+
"security_policy": "deny_by_default_secret_masking",
|
|
82
|
+
"routing_policy": "structural_path_authority_with_descriptor_discovery",
|
|
83
|
+
"candidate_sources": ["dense", "local_lexical", "exact_fact_key_segments"],
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
@property
|
|
87
|
+
def telemetry(self) -> dict[str, Any]:
|
|
88
|
+
avg_tok = (self.total_tokens / max(1, self.query_count)) if self.query_count else 0
|
|
89
|
+
return {
|
|
90
|
+
"query_count": self.query_count,
|
|
91
|
+
"llm_calls_total": 0,
|
|
92
|
+
"llm_calls_per_query": 0.0,
|
|
93
|
+
"api_calls_total": 0,
|
|
94
|
+
"api_calls_per_query": 0.0,
|
|
95
|
+
"cost_total_usd": 0.0,
|
|
96
|
+
"cost_per_1k_queries_usd": 0.0,
|
|
97
|
+
"mean_memory_context_tokens": round(avg_tok, 1),
|
|
98
|
+
"last_widening_telemetry": self.widener.last_telemetry,
|
|
99
|
+
"candidate_source_attribution_total": dict(self.total_attribution_stats),
|
|
100
|
+
"winning_candidate_source_attribution": dict(self.winning_attribution_stats),
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
def reset(self) -> None:
|
|
104
|
+
"""Reset internal stores, partition router, and multi-source indexes."""
|
|
105
|
+
super().reset()
|
|
106
|
+
self.chunk_data.clear()
|
|
107
|
+
self.partition_words.clear()
|
|
108
|
+
self.partition_fact_keys.clear()
|
|
109
|
+
self.last_query_attribution.clear()
|
|
110
|
+
self.total_attribution_stats.clear()
|
|
111
|
+
self.winning_attribution_stats.clear()
|
|
112
|
+
|
|
113
|
+
def ingest_memory(
|
|
114
|
+
self,
|
|
115
|
+
event_id: str,
|
|
116
|
+
template: str,
|
|
117
|
+
domain: str,
|
|
118
|
+
timestamp: float,
|
|
119
|
+
supersedes: Optional[str] = None,
|
|
120
|
+
source: str = "tool_observed",
|
|
121
|
+
) -> str:
|
|
122
|
+
"""Ingest episodic memory and update partition-scoped lexical and fact-key indexes."""
|
|
123
|
+
point_id = super().ingest_memory(
|
|
124
|
+
event_id=event_id,
|
|
125
|
+
template=template,
|
|
126
|
+
domain=domain,
|
|
127
|
+
timestamp=timestamp,
|
|
128
|
+
supersedes=supersedes,
|
|
129
|
+
source=source,
|
|
130
|
+
)
|
|
131
|
+
ppath = PartitionPath.from_domain(domain)
|
|
132
|
+
pid = ppath.canonical_id
|
|
133
|
+
|
|
134
|
+
# 1. Lexical index: non-stop words
|
|
135
|
+
tmpl_clean = template.lower()
|
|
136
|
+
words = {
|
|
137
|
+
w.strip(".:,-/?!\"'(){}[]")
|
|
138
|
+
for w in re.split(r"[\s\-._/]+", tmpl_clean)
|
|
139
|
+
if len(w) > 2 and w not in QUERY_STOP_WORDS
|
|
140
|
+
}
|
|
141
|
+
for w in words:
|
|
142
|
+
self.partition_words[pid][w].add(point_id)
|
|
143
|
+
|
|
144
|
+
# 2. Fact-key segment index: referenced facts
|
|
145
|
+
ref_keys = re.findall(r"\{\{ref:([\w.]+)(?:@rev:\d+)?\}\}", template)
|
|
146
|
+
for r in ref_keys:
|
|
147
|
+
for seg in re.split(r"[._\-]+", r):
|
|
148
|
+
seg_clean = seg.lower().strip()
|
|
149
|
+
if len(seg_clean) > 2 and seg_clean not in QUERY_STOP_WORDS:
|
|
150
|
+
self.partition_fact_keys[pid][seg_clean].add(point_id)
|
|
151
|
+
|
|
152
|
+
self.chunk_data[point_id] = {
|
|
153
|
+
"id": point_id,
|
|
154
|
+
"event_id": event_id,
|
|
155
|
+
"template": template,
|
|
156
|
+
"domain": domain,
|
|
157
|
+
"partition_id": pid,
|
|
158
|
+
"created_at": timestamp,
|
|
159
|
+
"source": source,
|
|
160
|
+
"referenced_facts": ref_keys,
|
|
161
|
+
"words": words,
|
|
162
|
+
}
|
|
163
|
+
return point_id
|
|
164
|
+
|
|
165
|
+
def recall(
|
|
166
|
+
self,
|
|
167
|
+
query: str,
|
|
168
|
+
domain: Optional[str] = None,
|
|
169
|
+
limit: int = 3,
|
|
170
|
+
) -> list[dict[str, Any]]:
|
|
171
|
+
"""Typed Multi-Source hierarchical scope-routed retrieval across partitions."""
|
|
172
|
+
self.query_count += 1
|
|
173
|
+
self.last_query_attribution.clear()
|
|
174
|
+
|
|
175
|
+
# 1. Structural Scope Resolution Gate
|
|
176
|
+
scope = ScopeResolver.resolve(
|
|
177
|
+
query=query,
|
|
178
|
+
retrieval_scope=domain,
|
|
179
|
+
registry=self.kv.get_domain_registry(),
|
|
180
|
+
)
|
|
181
|
+
if scope.status == ScopeResolutionStatus.EXPLICIT_UNRESOLVED:
|
|
182
|
+
return []
|
|
183
|
+
|
|
184
|
+
# Enforce asserted scope conflict
|
|
185
|
+
if scope.status == ScopeResolutionStatus.RESOLVED and domain:
|
|
186
|
+
retrieval_root = extract_component_root(domain)
|
|
187
|
+
domain_parts = set(p.lower() for p in domain.replace("/", ".").split(".") if p)
|
|
188
|
+
is_match = (
|
|
189
|
+
scope.asserted_scope == retrieval_root
|
|
190
|
+
or (scope.asserted_scope and scope.asserted_scope.lower() in domain_parts)
|
|
191
|
+
or domain.lower().startswith((scope.asserted_scope or "").lower())
|
|
192
|
+
or (scope.asserted_scope or "").lower().startswith(retrieval_root.lower())
|
|
193
|
+
)
|
|
194
|
+
if not is_match:
|
|
195
|
+
return []
|
|
196
|
+
|
|
197
|
+
# Effective explicit scope
|
|
198
|
+
effective_scope = domain or (
|
|
199
|
+
scope.raw_asserted_text
|
|
200
|
+
if (scope.raw_asserted_text and ("." in scope.raw_asserted_text or "/" in scope.raw_asserted_text))
|
|
201
|
+
else scope.asserted_scope
|
|
202
|
+
)
|
|
203
|
+
|
|
204
|
+
domain_tokens = set(re.findall(r"[\w]+", (effective_scope or domain or "").lower()))
|
|
205
|
+
raw_query_words = {
|
|
206
|
+
w.lower().strip(".:,-/?!\"'(){}[]")
|
|
207
|
+
for w in re.split(r"[\s\-._/]+", query)
|
|
208
|
+
if len(w) > 2 and w.lower() not in QUERY_STOP_WORDS
|
|
209
|
+
}
|
|
210
|
+
q_words = {w for w in raw_query_words if w not in domain_tokens}
|
|
211
|
+
query_idents = {
|
|
212
|
+
ident.strip(".:,-/")
|
|
213
|
+
for ident in TECHNICAL_CODE_ID.findall(query)
|
|
214
|
+
if ident.lower() not in QUERY_STOP_WORDS
|
|
215
|
+
}
|
|
216
|
+
match_tokens = q_words | query_idents
|
|
217
|
+
|
|
218
|
+
embedder = Embedder.get()
|
|
219
|
+
q_vec = embedder.embed_query(query)
|
|
220
|
+
q_ev = extract_query_evidence(
|
|
221
|
+
query=query,
|
|
222
|
+
domain=effective_scope or domain,
|
|
223
|
+
known_domains=self.vec._known_domains,
|
|
224
|
+
registry=self.kv.get_domain_registry(),
|
|
225
|
+
)
|
|
226
|
+
|
|
227
|
+
# 2. Local retrieval function: Typed Multi-Source Local Retrieval inside proposed partitions
|
|
228
|
+
def _multi_source_local_retrieval(pids: list[str]) -> list[dict[str, Any]]:
|
|
229
|
+
# Source A: Dense Semantic Candidates (limit=20)
|
|
230
|
+
dense_filter = models.Filter(
|
|
231
|
+
must=[
|
|
232
|
+
models.FieldCondition(key="is_active", match=models.MatchValue(value=True)),
|
|
233
|
+
models.FieldCondition(
|
|
234
|
+
key="partition_id",
|
|
235
|
+
match=models.MatchAny(any=pids) if len(pids) > 1 else models.MatchValue(value=pids[0]),
|
|
236
|
+
),
|
|
237
|
+
]
|
|
238
|
+
)
|
|
239
|
+
q_res = self.vec._client.query_points(
|
|
240
|
+
collection_name=COLLECTION_NAME,
|
|
241
|
+
query=q_vec,
|
|
242
|
+
query_filter=dense_filter,
|
|
243
|
+
limit=max(limit * 5, 20),
|
|
244
|
+
with_payload=True,
|
|
245
|
+
)
|
|
246
|
+
dense_ranks: dict[str, int] = {}
|
|
247
|
+
dense_scores: dict[str, float] = {}
|
|
248
|
+
for r_idx, pt in enumerate(q_res.points):
|
|
249
|
+
pid_str = str(pt.id)
|
|
250
|
+
dense_ranks[pid_str] = r_idx + 1
|
|
251
|
+
dense_scores[pid_str] = float(pt.score or 0.0)
|
|
252
|
+
|
|
253
|
+
# Source B: Local Lexical Candidates (limit=20)
|
|
254
|
+
lexical_counts: Counter[str] = Counter()
|
|
255
|
+
for pid in pids:
|
|
256
|
+
w_map = self.partition_words.get(pid, {})
|
|
257
|
+
for qw in q_words:
|
|
258
|
+
if qw in w_map:
|
|
259
|
+
for cid in w_map[qw]:
|
|
260
|
+
lexical_counts[cid] += 1
|
|
261
|
+
lexical_ranks: dict[str, int] = {}
|
|
262
|
+
for l_idx, (cid, cnt) in enumerate(lexical_counts.most_common(max(limit * 5, 20))):
|
|
263
|
+
if cnt > 0:
|
|
264
|
+
lexical_ranks[cid] = l_idx + 1
|
|
265
|
+
|
|
266
|
+
# Source C: Exact Fact-Key Segment Alignment Candidates (limit=20)
|
|
267
|
+
exact_counts: Counter[str] = Counter()
|
|
268
|
+
for pid in pids:
|
|
269
|
+
fk_map = self.partition_fact_keys.get(pid, {})
|
|
270
|
+
for mt in match_tokens:
|
|
271
|
+
if mt in fk_map:
|
|
272
|
+
for cid in fk_map[mt]:
|
|
273
|
+
exact_counts[cid] += 1
|
|
274
|
+
exact_ranks: dict[str, int] = {}
|
|
275
|
+
for e_idx, (cid, cnt) in enumerate(exact_counts.most_common(max(limit * 5, 20))):
|
|
276
|
+
if cnt > 0:
|
|
277
|
+
exact_ranks[cid] = e_idx + 1
|
|
278
|
+
|
|
279
|
+
# Candidate Union & Deduplication (Bounded <= 60 chunks per partition set)
|
|
280
|
+
all_candidate_ids = set(dense_ranks.keys()) | set(lexical_ranks.keys()) | set(exact_ranks.keys())
|
|
281
|
+
|
|
282
|
+
candidates: list[dict[str, Any]] = []
|
|
283
|
+
K = 60
|
|
284
|
+
|
|
285
|
+
for cid in all_candidate_ids:
|
|
286
|
+
in_d = cid in dense_ranks
|
|
287
|
+
in_l = cid in lexical_ranks
|
|
288
|
+
in_e = cid in exact_ranks
|
|
289
|
+
|
|
290
|
+
# Attribution classification
|
|
291
|
+
if in_d and in_l and in_e:
|
|
292
|
+
attribution = "all_three"
|
|
293
|
+
elif in_d and in_l:
|
|
294
|
+
attribution = "dense+lexical"
|
|
295
|
+
elif in_l and in_e:
|
|
296
|
+
attribution = "lexical+exact"
|
|
297
|
+
elif in_d and in_e:
|
|
298
|
+
attribution = "dense+exact"
|
|
299
|
+
elif in_d:
|
|
300
|
+
attribution = "dense_only"
|
|
301
|
+
elif in_l:
|
|
302
|
+
attribution = "lexical_only"
|
|
303
|
+
elif in_e:
|
|
304
|
+
attribution = "exact_only"
|
|
305
|
+
else:
|
|
306
|
+
attribution = "unknown"
|
|
307
|
+
|
|
308
|
+
self.last_query_attribution[attribution] += 1
|
|
309
|
+
self.total_attribution_stats[attribution] += 1
|
|
310
|
+
|
|
311
|
+
item = self.chunk_data.get(cid)
|
|
312
|
+
if not item:
|
|
313
|
+
continue
|
|
314
|
+
|
|
315
|
+
tmpl = item["template"]
|
|
316
|
+
dom = item["domain"]
|
|
317
|
+
|
|
318
|
+
# RRF channel contributions
|
|
319
|
+
d_r = dense_ranks.get(cid, 9999)
|
|
320
|
+
l_r = lexical_ranks.get(cid, 9999)
|
|
321
|
+
e_r = exact_ranks.get(cid, 9999)
|
|
322
|
+
|
|
323
|
+
rrf_dense = (1.0 / (K + d_r)) if in_d else 0.0
|
|
324
|
+
rrf_lex = (1.0 / (K + l_r)) if in_l else 0.0
|
|
325
|
+
rrf_exact = (1.0 / (K + e_r)) if in_e else 0.0
|
|
326
|
+
|
|
327
|
+
v_score = dense_scores.get(cid, 0.0)
|
|
328
|
+
|
|
329
|
+
# Evidence Gate
|
|
330
|
+
m_ev = extract_memory_evidence(
|
|
331
|
+
domain=dom,
|
|
332
|
+
template=tmpl,
|
|
333
|
+
referenced_facts=item.get("referenced_facts", []),
|
|
334
|
+
event_id=item.get("event_id", cid),
|
|
335
|
+
)
|
|
336
|
+
supp_level, ev_score, reason = resolve_evidence(
|
|
337
|
+
query_ev=q_ev,
|
|
338
|
+
mem_ev=m_ev,
|
|
339
|
+
vector_score=v_score,
|
|
340
|
+
is_fact_boosted=in_e,
|
|
341
|
+
)
|
|
342
|
+
|
|
343
|
+
if supp_level in (SupportLevel.UNSUPPORTED, SupportLevel.WEAK):
|
|
344
|
+
continue
|
|
345
|
+
|
|
346
|
+
rrf_evidence = (2.0 / (K + 1)) if supp_level == SupportLevel.STRONG else (1.0 / (K + 1))
|
|
347
|
+
|
|
348
|
+
# Exact attribute alignment count
|
|
349
|
+
chunk_refs = item.get("referenced_facts", [])
|
|
350
|
+
fact_key_segments: set[str] = set()
|
|
351
|
+
for r in chunk_refs:
|
|
352
|
+
for seg in re.split(r"[._\-]+", r):
|
|
353
|
+
s_clean = seg.lower().strip()
|
|
354
|
+
if len(s_clean) > 2 and s_clean not in QUERY_STOP_WORDS:
|
|
355
|
+
fact_key_segments.add(s_clean)
|
|
356
|
+
attr_match_count = len(match_tokens & fact_key_segments)
|
|
357
|
+
|
|
358
|
+
score = (rrf_dense + rrf_lex + rrf_exact + rrf_evidence) * 30.0
|
|
359
|
+
|
|
360
|
+
cand_record = dict(item)
|
|
361
|
+
cand_record.update({
|
|
362
|
+
"score": round(score, 4),
|
|
363
|
+
"vector_score": v_score,
|
|
364
|
+
"attr_match_count": attr_match_count,
|
|
365
|
+
"dense_rank": d_r if in_d else None,
|
|
366
|
+
"lexical_rank": l_r if in_l else None,
|
|
367
|
+
"exact_rank": e_r if in_e else None,
|
|
368
|
+
"lexical_hits": lexical_counts.get(cid, 0),
|
|
369
|
+
"attribution": attribution,
|
|
370
|
+
"support_level": supp_level.value,
|
|
371
|
+
"evidence_score": round(ev_score, 4),
|
|
372
|
+
"evidence_reason": reason,
|
|
373
|
+
})
|
|
374
|
+
candidates.append(cand_record)
|
|
375
|
+
|
|
376
|
+
# Deterministic Ranking Tuple:
|
|
377
|
+
# 1. Exact attribute alignment (structural discriminant)
|
|
378
|
+
# 2. RRF fused score (dense + lexical + exact fusion)
|
|
379
|
+
# 3. Raw lexical hits
|
|
380
|
+
# 4. Dense cosine similarity
|
|
381
|
+
# 5. Stable event_id tie-breaker
|
|
382
|
+
candidates.sort(
|
|
383
|
+
key=lambda x: (
|
|
384
|
+
x.get("attr_match_count", 0),
|
|
385
|
+
x["score"],
|
|
386
|
+
x.get("lexical_hits", 0),
|
|
387
|
+
x["vector_score"],
|
|
388
|
+
x.get("event_id", x["id"]),
|
|
389
|
+
),
|
|
390
|
+
reverse=True,
|
|
391
|
+
)
|
|
392
|
+
return candidates
|
|
393
|
+
|
|
394
|
+
# 3. Epistemic progressive widening cascade (L0 -> L1 -> L2 -> L3 -> L4 bounded)
|
|
395
|
+
admitted, _ = self.widener.widen_and_retrieve(
|
|
396
|
+
query=query,
|
|
397
|
+
retrieval_fn=_multi_source_local_retrieval,
|
|
398
|
+
explicit_scope=effective_scope,
|
|
399
|
+
max_results=limit,
|
|
400
|
+
candidate_budget=500,
|
|
401
|
+
)
|
|
402
|
+
|
|
403
|
+
# Track winning candidate attribution
|
|
404
|
+
if admitted:
|
|
405
|
+
win_attr = admitted[0].get("attribution", "unknown")
|
|
406
|
+
self.winning_attribution_stats[win_attr] += 1
|
|
407
|
+
|
|
408
|
+
# 4. Late-binding fact hydration on admitted candidates
|
|
409
|
+
hydrated: list[dict[str, Any]] = []
|
|
410
|
+
for r in admitted:
|
|
411
|
+
text = hydrate_template(
|
|
412
|
+
text=r["template"],
|
|
413
|
+
kv_resolver=self.kv.resolve_ref,
|
|
414
|
+
chunk_domain=r.get("domain", "general"),
|
|
415
|
+
allow_secrets=False,
|
|
416
|
+
)
|
|
417
|
+
hydrated.append({
|
|
418
|
+
"id": r["id"],
|
|
419
|
+
"event_id": r.get("event_id", r["id"]),
|
|
420
|
+
"text": text,
|
|
421
|
+
"domain": r.get("domain", "general"),
|
|
422
|
+
"partition_id": r.get("partition_id", UNSCOPED_LEGACY.canonical_id),
|
|
423
|
+
"score": r.get("score", 0.0),
|
|
424
|
+
"evidence_score": r.get("evidence_score", 0.0),
|
|
425
|
+
"vector_score": r.get("vector_score", 0.0),
|
|
426
|
+
"attr_match_count": r.get("attr_match_count", 0),
|
|
427
|
+
"attribution": r.get("attribution", "unknown"),
|
|
428
|
+
"has_fact": bool(r.get("exact_rank") is not None),
|
|
429
|
+
"has_attr_match": bool(r.get("attr_match_count", 0) > 0),
|
|
430
|
+
"is_historical": "{{ref:" in r["template"] and "@rev:" in r["template"],
|
|
431
|
+
})
|
|
432
|
+
|
|
433
|
+
tok_sum = sum(len(h["text"].split()) for h in hydrated)
|
|
434
|
+
self.total_tokens += tok_sum
|
|
435
|
+
return hydrated
|