cortexm 0.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- context_m.py +17 -0
- cortexm/__init__.py +45 -0
- cortexm/accel.py +403 -0
- cortexm/api/__init__.py +0 -0
- cortexm/api/chaos.py +118 -0
- cortexm/api/memory.py +635 -0
- cortexm/bench/__init__.py +0 -0
- cortexm/bench/abilities.py +311 -0
- cortexm/bench/baselines.py +89 -0
- cortexm/bench/beam_loader.py +317 -0
- cortexm/bench/generator.py +376 -0
- cortexm/bench/harness.py +211 -0
- cortexm/bench/messy.py +218 -0
- cortexm/bench/micro.py +251 -0
- cortexm/bench/ood.py +443 -0
- cortexm/bench/run.py +137 -0
- cortexm/bridge/__init__.py +0 -0
- cortexm/bridge/dates.py +178 -0
- cortexm/bridge/decoders.py +204 -0
- cortexm/bridge/enrich.py +255 -0
- cortexm/bridge/extractor.py +316 -0
- cortexm/bridge/fallback.py +332 -0
- cortexm/bridge/onnx_runtime.py +158 -0
- cortexm/bridge/patterns.py +760 -0
- cortexm/bridge/ppr.py +104 -0
- cortexm/bridge/prefilter.py +188 -0
- cortexm/bridge/query_extract.py +420 -0
- cortexm/bridge/reader.py +1174 -0
- cortexm/bridge/rerank.py +204 -0
- cortexm/bridge/writer.py +492 -0
- cortexm/cli.py +295 -0
- cortexm/cognition/__init__.py +53 -0
- cortexm/cognition/abstraction.py +192 -0
- cortexm/cognition/analogy.py +159 -0
- cortexm/cognition/engine.py +204 -0
- cortexm/cognition/gaps.py +365 -0
- cortexm/cognition/scanner.py +204 -0
- cortexm/config.py +375 -0
- cortexm/cortexm.py +8 -0
- cortexm/enterprise/__init__.py +0 -0
- cortexm/enterprise/audit.py +178 -0
- cortexm/enterprise/governance.py +239 -0
- cortexm/errors.py +35 -0
- cortexm/features/__init__.py +0 -0
- cortexm/features/git.py +204 -0
- cortexm/features/prefetch.py +88 -0
- cortexm/features/zk.py +105 -0
- cortexm/federation/__init__.py +39 -0
- cortexm/federation/crdt.py +275 -0
- cortexm/federation/fabric.py +109 -0
- cortexm/federation/hlc.py +80 -0
- cortexm/federation/node.py +145 -0
- cortexm/federation/schema_report.py +73 -0
- cortexm/federation/transport.py +164 -0
- cortexm/index/__init__.py +19 -0
- cortexm/index/nsg.py +386 -0
- cortexm/mcp/__init__.py +0 -0
- cortexm/mcp/server.py +985 -0
- cortexm/metrics.py +62 -0
- cortexm/migrate/__init__.py +0 -0
- cortexm/migrate/importers.py +192 -0
- cortexm/provenance/__init__.py +78 -0
- cortexm/provenance/agent.py +214 -0
- cortexm/provenance/cose.py +201 -0
- cortexm/provenance/scitt.py +258 -0
- cortexm/provenance/vc.py +250 -0
- cortexm/security/__init__.py +0 -0
- cortexm/security/crypto.py +162 -0
- cortexm/security/hashes.py +140 -0
- cortexm/security/injection.py +149 -0
- cortexm/security/mind.py +154 -0
- cortexm/security/pii.py +265 -0
- cortexm/security/rbac.py +169 -0
- cortexm/security/sandbox.py +131 -0
- cortexm/security/zk_hamming.py +142 -0
- cortexm/security/zk_sql.py +485 -0
- cortexm/server/__init__.py +0 -0
- cortexm/server/metrics.py +88 -0
- cortexm/server/rest.py +936 -0
- cortexm/server/sparql.py +984 -0
- cortexm/text/__init__.py +0 -0
- cortexm/text/dissim.py +252 -0
- cortexm/text/embedder.py +155 -0
- cortexm/text/fuzzy.py +218 -0
- cortexm/text/idiolect.py +253 -0
- cortexm/text/labse.py +374 -0
- cortexm/text/tokenizer.py +79 -0
- cortexm/trace/__init__.py +0 -0
- cortexm/trace/blob_arena.py +277 -0
- cortexm/trace/consolidate.py +337 -0
- cortexm/trace/contradictions.py +69 -0
- cortexm/trace/dedup.py +114 -0
- cortexm/trace/edges.py +214 -0
- cortexm/trace/fact.py +121 -0
- cortexm/trace/fade.py +245 -0
- cortexm/trace/lifecycle.py +112 -0
- cortexm/trace/rebuild.py +173 -0
- cortexm/trace/rules.py +171 -0
- cortexm/trace/store.py +680 -0
- cortexm/trace/structural.py +183 -0
- cortexm/trace/tmt.py +335 -0
- cortexm/util.py +148 -0
- cortexm/vsa/__init__.py +0 -0
- cortexm/vsa/attribution.py +149 -0
- cortexm/vsa/cleanup.py +161 -0
- cortexm/vsa/codecs.py +397 -0
- cortexm/vsa/hologram_overlay.py +139 -0
- cortexm/vsa/index.py +163 -0
- cortexm/vsa/ops.py +149 -0
- cortexm/vsa/palace.py +446 -0
- cortexm/vsa/role_vectors.py +236 -0
- cortexm/vsa/slb.py +78 -0
- cortexm/vsa/tlsh_trie.py +137 -0
- cortexm/vsa/working_memory.py +249 -0
- cortexm-0.3.0.dist-info/METADATA +482 -0
- cortexm-0.3.0.dist-info/RECORD +120 -0
- cortexm-0.3.0.dist-info/WHEEL +5 -0
- cortexm-0.3.0.dist-info/entry_points.txt +2 -0
- cortexm-0.3.0.dist-info/licenses/LICENSE +190 -0
- cortexm-0.3.0.dist-info/top_level.txt +2 -0
cortexm/bridge/reader.py
ADDED
|
@@ -0,0 +1,1174 @@
|
|
|
1
|
+
"""Read path — deterministic neuro-symbolic query planner.
|
|
2
|
+
|
|
3
|
+
Query → intent parse (temporal / ordering / counting / supersession /
|
|
4
|
+
current-state / multi-hop / free recall) → parallel VSA palace search +
|
|
5
|
+
symbolic Trace queries → contradiction-chain & entity-hop expansion →
|
|
6
|
+
weighted fusion → context block with per-fact cryptographic provenance.
|
|
7
|
+
|
|
8
|
+
Every retrieval returns the full audit chain:
|
|
9
|
+
query → VSA match → symbolic dereference → source hash → source text.
|
|
10
|
+
No LLM calls at query time (edge-capable, offline).
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import re
|
|
16
|
+
import time
|
|
17
|
+
from dataclasses import dataclass, field
|
|
18
|
+
from datetime import datetime, timezone
|
|
19
|
+
|
|
20
|
+
from cortexm import metrics
|
|
21
|
+
from cortexm.bridge.dates import find_dates
|
|
22
|
+
from cortexm.config import Config
|
|
23
|
+
from cortexm.text.tokenizer import cap_sequences, content_words
|
|
24
|
+
from cortexm.trace.fact import Fact
|
|
25
|
+
from cortexm.trace.store import TraceStore
|
|
26
|
+
from cortexm.util import normalize
|
|
27
|
+
from cortexm.vsa.palace import MemoryPalace
|
|
28
|
+
from cortexm.vsa.slb import SemanticLookasideBuffer
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def _content_key(f: "Fact | None") -> tuple:
|
|
32
|
+
"""Deterministic ordering key for a fact. Fact ids are uuid4 (random
|
|
33
|
+
per process); ties broken on id would make rankings — and therefore
|
|
34
|
+
benchmark scores — vary across identical runs. Content never does."""
|
|
35
|
+
if f is None:
|
|
36
|
+
return ("~", "~", "~", "~")
|
|
37
|
+
return (f.subject, f.relation, f.value, str(f.valid_from))
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
RELATION_HINTS = [
|
|
41
|
+
# occupation idioms FIRST: "for a living" contains "living", which the
|
|
42
|
+
# residence hint below would otherwise capture, drowning the `role`
|
|
43
|
+
# fact under lives_in/moved_to noise.
|
|
44
|
+
(re.compile(r"\b(for a living|occupation|profession|career|job title|"
|
|
45
|
+
r"what does .{2,40}? do|works? as)\b", re.I),
|
|
46
|
+
["role", "works_at", "studied"]),
|
|
47
|
+
(re.compile(r"\b(work\w*|employer|company|job)\b", re.I),
|
|
48
|
+
["works_at", "role"]),
|
|
49
|
+
(re.compile(r"\b(live|lives|living|based|city|hometown|resid\w*|mov\w*)\b", re.I),
|
|
50
|
+
["lives_in", "moved_to"]),
|
|
51
|
+
(re.compile(r"\b(prefer\w*|like\w*|favorite|favourite|taste)\b", re.I),
|
|
52
|
+
["prefers", "likes", "dislikes"]),
|
|
53
|
+
(re.compile(r"\b(music|food|coffee|genre|playlist|meal|lunch|dinner|"
|
|
54
|
+
r"drink|snack|editor|theme|picking|pick|choose|choosing)\b",
|
|
55
|
+
re.I),
|
|
56
|
+
["prefers", "likes", "dislikes"]),
|
|
57
|
+
(re.compile(r"\b(manag\w*|boss|report\w*|supervis\w*)\b", re.I),
|
|
58
|
+
["reports_to", "manages"]),
|
|
59
|
+
(re.compile(r"\b(team|group|squad)\b", re.I),
|
|
60
|
+
["member_of", "manages", "team_uses", "uses"]),
|
|
61
|
+
(re.compile(r"\b(skill\w*|know\w*|languag\w*|code|coding|program\w*|stack|tool\w*)\b", re.I),
|
|
62
|
+
["has_skill", "speaks", "team_uses", "uses"]),
|
|
63
|
+
(re.compile(r"\b(birthday|born|age|years old)\b", re.I),
|
|
64
|
+
["birthday", "age"]),
|
|
65
|
+
(re.compile(r"\b(sister|brother|mother|father|mom|dad|wife|husband|sibling|spouse|parent|family|daughter|son)\b", re.I),
|
|
66
|
+
["sibling", "parent", "spouse", "child"]),
|
|
67
|
+
(re.compile(r"\b(project\w*|build\w*|ship\w*|launch\w*|develop\w*|releas\w*)\b", re.I),
|
|
68
|
+
["works_on", "completed", "event"]),
|
|
69
|
+
(re.compile(r"\b(happen\w*|events?|did|done)\b", re.I), ["event"]),
|
|
70
|
+
(re.compile(r"\b(stud\w*|school|degree|major|university|college|educat\w*)\b", re.I),
|
|
71
|
+
["studied", "studied_at"]),
|
|
72
|
+
(re.compile(r"\b(pet|dog|cat|animal)\b", re.I), ["has_pet"]),
|
|
73
|
+
(re.compile(r"\b(hobby|free time|weekend)\b", re.I), ["hobby"]),
|
|
74
|
+
(re.compile(r"\b(nickname|alias|called|go by|full name|real name)\b", re.I),
|
|
75
|
+
["alias", "name"]),
|
|
76
|
+
(re.compile(r"\b(goal|plan\w*|want to|hope)\b", re.I), ["goal"]),
|
|
77
|
+
(re.compile(r"\b(instruction|always|never|respond|format|signature|guideline)\b", re.I),
|
|
78
|
+
["instruction"]),
|
|
79
|
+
]
|
|
80
|
+
|
|
81
|
+
CURRENT_MARKERS = re.compile(
|
|
82
|
+
r"\b(current\w*|now|these days|nowadays|latest|today|present)\b", re.I)
|
|
83
|
+
SUPERSESSION_MARKERS = re.compile(
|
|
84
|
+
r"\b(still|no longer|anymore|used to|former\w*|previous\w*|before|always|ever)\b", re.I)
|
|
85
|
+
ORDERING_MARKERS = re.compile(
|
|
86
|
+
r"\b(which|what)\s+(?:one\s+)?happened\s+(?:first|before|earlier)\b"
|
|
87
|
+
r"|\bbefore\s+.+?\s+or\b|\border\b|\bsequence\b|\bchronolog\w*\b"
|
|
88
|
+
r"|\bfirst\s*:\s*.*\s+or\s+", re.I)
|
|
89
|
+
COUNT_MARKERS = re.compile(
|
|
90
|
+
r"\bhow\s+many\s+(times|jobs|cities|companies|roles|moves|changes)\b"
|
|
91
|
+
r"|\bhow\s+often\b", re.I)
|
|
92
|
+
LIST_MARKERS = re.compile(
|
|
93
|
+
r"\b(?:list|name|enumerate|summar\w+)\b.*\b(?:all|every)\b"
|
|
94
|
+
r"|\ball\s+(?:the\s+)?\w+\s+"
|
|
95
|
+
r"(?:that|which|she|he|they|i)\b"
|
|
96
|
+
r"|\b(?:list|name|enumerate)\s+all\b"
|
|
97
|
+
r"|\b(?:list|name|enumerate)\s+every\b", re.I)
|
|
98
|
+
MULTIHOP_MARKERS = re.compile(
|
|
99
|
+
r"\b('s\b|of the|of my|of her|of his|of their)\b", re.I)
|
|
100
|
+
# Tier-4 fix: implicit "current" queries — "where does X work?" /
|
|
101
|
+
# "what does X do?" / "what's X's job/employer/city?" — these are
|
|
102
|
+
# asking for the single-valued CURRENT state but contain no explicit
|
|
103
|
+
# "now" marker. Detecting them lets the reader apply the "current"
|
|
104
|
+
# intent (which sets allow_inactive=True so the superseded chain can
|
|
105
|
+
# surface; for ACTIVE facts only it also biases ranking toward the
|
|
106
|
+
# latest valid_from).
|
|
107
|
+
SINGLE_VALUED_QUERY = re.compile(
|
|
108
|
+
r"\b(?:where\s+(?:does|do|is)\s+\w+\s+(?:work|live|stay|reside)\b"
|
|
109
|
+
r"|what(?:'?s| is)\s+\w+(?:'?s)?\s+(?:job|role|title|position|employer|"
|
|
110
|
+
r"company|boss|manager|location|address|city|home)\b"
|
|
111
|
+
r"|\bwhat\s+does\s+\w+\s+do\b"
|
|
112
|
+
r"|\bwho\s+is\s+\w+(?:'?s)?\s+(?:manager|boss|lead|supervisor)\b)", re.I)
|
|
113
|
+
# Temporal + LIST fusion: "list all X from 2024" or "what did X do
|
|
114
|
+
# between A and B" should be a temporal-list (return the full matching
|
|
115
|
+
# set within the window, not just top-k). Detected downstream by the
|
|
116
|
+
# planner when both LIST and a temporal window are set.
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
@dataclass
|
|
120
|
+
class QueryPlan:
|
|
121
|
+
intent: str = "recall"
|
|
122
|
+
entities: list[str] = field(default_factory=list)
|
|
123
|
+
relations: list[str] = field(default_factory=list)
|
|
124
|
+
window_start: str | None = None
|
|
125
|
+
window_end: str | None = None
|
|
126
|
+
keywords: list[str] = field(default_factory=list)
|
|
127
|
+
# Tier-4 fix: sub-intent carries the temporal_list fusion flag
|
|
128
|
+
# so the reader's filter knows to apply BOTH the temporal window
|
|
129
|
+
# AND the exhaustive recall semantics (don't truncate to top-k).
|
|
130
|
+
sub_intent: str | None = None
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
@dataclass
|
|
134
|
+
class RetrievalResult:
|
|
135
|
+
query: str
|
|
136
|
+
intent: str
|
|
137
|
+
facts: list[Fact]
|
|
138
|
+
context_block: str
|
|
139
|
+
provenance: dict
|
|
140
|
+
timing: dict
|
|
141
|
+
slb_hit: bool = False
|
|
142
|
+
scores: dict = field(default_factory=dict)
|
|
143
|
+
|
|
144
|
+
def memories(self) -> list[dict]:
|
|
145
|
+
out = []
|
|
146
|
+
for f in self.facts:
|
|
147
|
+
out.append({
|
|
148
|
+
"id": f.id,
|
|
149
|
+
"memory": f"{f.subject} | {f.relation} | {f.value}",
|
|
150
|
+
"score": self.scores.get(f.id, 0.0),
|
|
151
|
+
"event": "ADD",
|
|
152
|
+
"valid_from": f.valid_from,
|
|
153
|
+
"valid_to": f.valid_to,
|
|
154
|
+
"confidence": f.confidence,
|
|
155
|
+
"hash": f.source_hash,
|
|
156
|
+
})
|
|
157
|
+
return out
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
class MemoryReader:
|
|
161
|
+
def __init__(self, config: Config, store: TraceStore, palace: MemoryPalace,
|
|
162
|
+
prefetcher=None) -> None:
|
|
163
|
+
self.cfg = config
|
|
164
|
+
self.store = store
|
|
165
|
+
self.palace = palace
|
|
166
|
+
self.prefetcher = prefetcher
|
|
167
|
+
self.slb = SemanticLookasideBuffer(
|
|
168
|
+
config.slb_entries, config.slb_threshold, config.dims)
|
|
169
|
+
self._scope_cache: dict[tuple, frozenset] = {}
|
|
170
|
+
self._queries = 0
|
|
171
|
+
# NSR-inspired swappable decoder (default = LLM prompt block,
|
|
172
|
+
# preserving the original reader._context_block behavior).
|
|
173
|
+
# Override via reader.with_decoder("rdf" | "datalog" | "json").
|
|
174
|
+
from cortexm.bridge.decoders import get_decoder
|
|
175
|
+
self._decoder = get_decoder("llm_prompt")
|
|
176
|
+
# Cross-encoder-style fact reranker (μ=0). Lazily imported so the
|
|
177
|
+
# rest of the fabric is unaffected. Controlled by the
|
|
178
|
+
# `enable_rerank` config knob (False by default — the bench must
|
|
179
|
+
# explicitly enable it via "+rerank" config).
|
|
180
|
+
self._reranker = None
|
|
181
|
+
if getattr(config, "enable_rerank", False):
|
|
182
|
+
try:
|
|
183
|
+
from cortexm.bridge.rerank import FactReranker
|
|
184
|
+
self._reranker = FactReranker(
|
|
185
|
+
palace.embedder,
|
|
186
|
+
alpha=getattr(config, "rerank_alpha", 0.55),
|
|
187
|
+
beta=getattr(config, "rerank_beta", 0.45),
|
|
188
|
+
prf_alpha=getattr(config, "prf_alpha", 0.6),
|
|
189
|
+
prf_beta=getattr(config, "prf_beta", 0.4),
|
|
190
|
+
prf_topn=getattr(config, "prf_topn", 3))
|
|
191
|
+
except Exception: # noqa: BLE001
|
|
192
|
+
self._reranker = None
|
|
193
|
+
|
|
194
|
+
def with_decoder(self, name: str) -> "MemoryReader":
|
|
195
|
+
"""Swap the output decoder (NSR insight: same palace + Trace,
|
|
196
|
+
different decoder). Returns self for chaining.
|
|
197
|
+
|
|
198
|
+
Known decoders: 'llm_prompt' (default), 'rdf', 'datalog', 'json'.
|
|
199
|
+
"""
|
|
200
|
+
from cortexm.bridge.decoders import get_decoder
|
|
201
|
+
self._decoder = get_decoder(name)
|
|
202
|
+
return self
|
|
203
|
+
|
|
204
|
+
# ------------------------------------------------------------- helpers
|
|
205
|
+
def _scope_ids(self, user_id, agent_id, run_id, branch) -> frozenset:
|
|
206
|
+
head = self.store.head(branch or self.store.current_branch()) or ""
|
|
207
|
+
key = (user_id, agent_id, run_id, branch, head)
|
|
208
|
+
hit = self._scope_cache.get(key)
|
|
209
|
+
if hit is not None:
|
|
210
|
+
return hit
|
|
211
|
+
facts = self.store.query_facts(
|
|
212
|
+
user_id=user_id, agent_id=None, run_id=run_id,
|
|
213
|
+
active=True, include_quarantined=False)
|
|
214
|
+
# InjecMEM scope sandbox:
|
|
215
|
+
# * user-scope query (agent_id=None) sees ONLY user-scoped facts;
|
|
216
|
+
# agent-written facts stay invisible until explicitly promoted.
|
|
217
|
+
# * agent query (agent_id=A) sees A's own facts PLUS the shared
|
|
218
|
+
# user scope — agents read the user's memory, never each
|
|
219
|
+
# other's.
|
|
220
|
+
if agent_id is None:
|
|
221
|
+
if getattr(self.cfg, "sandbox_enabled", True):
|
|
222
|
+
facts = [f for f in facts if f.agent_id is None]
|
|
223
|
+
else:
|
|
224
|
+
facts = [f for f in facts if f.agent_id in (None, agent_id)]
|
|
225
|
+
ids = frozenset(f.id for f in facts)
|
|
226
|
+
if branch is not None:
|
|
227
|
+
active = self.store.active_ids(branch)
|
|
228
|
+
ids = ids & active
|
|
229
|
+
if len(self._scope_cache) < 16:
|
|
230
|
+
self._scope_cache[key] = ids
|
|
231
|
+
return ids
|
|
232
|
+
|
|
233
|
+
def invalidate_caches(self) -> None:
|
|
234
|
+
self._scope_cache.clear()
|
|
235
|
+
|
|
236
|
+
def _canonical_entities(self, query: str, user_id: str) -> list[str]:
|
|
237
|
+
"""cap-sequences + lexicon + alias resolution to canonical names."""
|
|
238
|
+
cands: list[str] = []
|
|
239
|
+
for seq in cap_sequences(query):
|
|
240
|
+
cands.append(seq.strip().rstrip(".,!?;:"))
|
|
241
|
+
# single capitalized words that are known entities
|
|
242
|
+
for w in re.findall(r"\b[A-Z][a-z]{2,}\b", query):
|
|
243
|
+
cands.append(w)
|
|
244
|
+
name = self.store.kv_get(f"name:{user_id}")
|
|
245
|
+
if re.search(r"\b(my|i|me)\b", query, re.I) and name:
|
|
246
|
+
cands.append(name)
|
|
247
|
+
|
|
248
|
+
lex_keys = set()
|
|
249
|
+
import json
|
|
250
|
+
raw = self.store.kv_get(f"lexicon:{user_id}", "[]")
|
|
251
|
+
try:
|
|
252
|
+
lex_keys = set(json.loads(raw))
|
|
253
|
+
except Exception:
|
|
254
|
+
pass
|
|
255
|
+
canonical: list[str] = []
|
|
256
|
+
seen = set()
|
|
257
|
+
seen_out: set[str] = set()
|
|
258
|
+
for c in cands:
|
|
259
|
+
if not c or c.lower() in seen:
|
|
260
|
+
continue
|
|
261
|
+
seen.add(c.lower())
|
|
262
|
+
resolved = None
|
|
263
|
+
# alias resolution: (X, alias, c) → X
|
|
264
|
+
for f in self.store.query_facts(relation="alias", value=c,
|
|
265
|
+
user_id=user_id, active=True,
|
|
266
|
+
limit=8):
|
|
267
|
+
resolved = f.subject
|
|
268
|
+
break
|
|
269
|
+
if resolved is None and c in lex_keys:
|
|
270
|
+
resolved = c
|
|
271
|
+
if resolved is None and name and c.lower() == name.split()[0].lower():
|
|
272
|
+
resolved = name
|
|
273
|
+
# FALL-THROUGH: if no alias/lexicon/name resolution found,
|
|
274
|
+
# use the candidate as-is. This is correct for cases where
|
|
275
|
+
# the user is asking about an entity that's already a fact
|
|
276
|
+
# subject in the trace — "Where does Alice work?" with
|
|
277
|
+
# (Alice, works_at, Google) in the trace should resolve
|
|
278
|
+
# "Alice" → "Alice" without requiring a separate alias
|
|
279
|
+
# fact. (The previous behavior dropped the candidate,
|
|
280
|
+
# which silently broke recall.)
|
|
281
|
+
if resolved is None:
|
|
282
|
+
# check if the candidate matches a known fact subject
|
|
283
|
+
# in this user's scope — if so, use it
|
|
284
|
+
for f in self.store.query_facts(subject=c, user_id=user_id,
|
|
285
|
+
active=True, limit=1):
|
|
286
|
+
resolved = c
|
|
287
|
+
break
|
|
288
|
+
if resolved is None:
|
|
289
|
+
# last resort: use the candidate as-is. This is the
|
|
290
|
+
# correct default for capitalized entity mentions in
|
|
291
|
+
# natural language queries.
|
|
292
|
+
resolved = c
|
|
293
|
+
if resolved and resolved.lower() not in seen_out:
|
|
294
|
+
canonical.append(resolved)
|
|
295
|
+
seen_out.add(resolved.lower())
|
|
296
|
+
# order: prefer multi-word (full names) first
|
|
297
|
+
canonical.sort(key=lambda s: -len(s))
|
|
298
|
+
return canonical[:4]
|
|
299
|
+
|
|
300
|
+
def _plan(self, query: str, user_id: str, ts: datetime | None) -> QueryPlan:
|
|
301
|
+
plan = QueryPlan()
|
|
302
|
+
for rx, rels in RELATION_HINTS:
|
|
303
|
+
if rx.search(query):
|
|
304
|
+
plan.relations.extend(rels)
|
|
305
|
+
plan.relations = list(dict.fromkeys(plan.relations))[:5]
|
|
306
|
+
plan.entities = self._canonical_entities(query, user_id)
|
|
307
|
+
plan.keywords = content_words(query)
|
|
308
|
+
|
|
309
|
+
if ORDERING_MARKERS.search(query):
|
|
310
|
+
plan.intent = "ordering"
|
|
311
|
+
elif COUNT_MARKERS.search(query):
|
|
312
|
+
plan.intent = "count"
|
|
313
|
+
elif LIST_MARKERS.search(query):
|
|
314
|
+
plan.intent = "list" # exhaustive set recall
|
|
315
|
+
elif SUPERSESSION_MARKERS.search(query) or CURRENT_MARKERS.search(query):
|
|
316
|
+
plan.intent = "current"
|
|
317
|
+
# Tier-4 fix: implicit "current" queries — surface the latest
|
|
318
|
+
# value on single-valued relations ("where does X work?" /
|
|
319
|
+
# "what does X do?") even without an explicit "now" marker.
|
|
320
|
+
# This is the #1 LongMemEval knowledge-update failure mode:
|
|
321
|
+
# the user asks the obvious question and the engine returns
|
|
322
|
+
# the superseded (now-inactive) fact because it was never
|
|
323
|
+
# promoted into the "current" intent. Detected here via the
|
|
324
|
+
# SINGLE_VALUED_QUERY regex; only fires when no higher-
|
|
325
|
+
# precision intent (ordering/count/list) already matched.
|
|
326
|
+
if plan.intent == "recall" and SINGLE_VALUED_QUERY.search(query):
|
|
327
|
+
plan.intent = "current"
|
|
328
|
+
dates = find_dates(query, ts or datetime.now(timezone.utc))
|
|
329
|
+
if dates and plan.intent in ("recall", "current"):
|
|
330
|
+
plan.intent = "temporal"
|
|
331
|
+
plan.window_start = dates[0]["iso"]
|
|
332
|
+
plan.window_end = dates[1]["iso"] if len(dates) > 1 else None
|
|
333
|
+
if dates[0].get("granularity") == "year":
|
|
334
|
+
y = dates[0]["iso"][:4]
|
|
335
|
+
plan.window_start = f"{y}-01-01"
|
|
336
|
+
plan.window_end = f"{y}-12-31"
|
|
337
|
+
elif dates[0].get("granularity") in ("month", "ym_num"):
|
|
338
|
+
# "in February 2025" — close the window at end-of-month;
|
|
339
|
+
# an open-ended start would drag in every later event.
|
|
340
|
+
y, mo = dates[0]["iso"][:4], int(dates[0]["iso"][5:7])
|
|
341
|
+
_last = [31, 29 if (int(y) % 4 == 0 and (int(y) % 100 != 0
|
|
342
|
+
or int(y) % 400 == 0)) else 28,
|
|
343
|
+
31, 30, 31, 30, 31, 31, 30, 31, 30, 31][mo - 1]
|
|
344
|
+
plan.window_start = f"{y}-{mo:02d}-01"
|
|
345
|
+
plan.window_end = f"{y}-{mo:02d}-{_last:02d}"
|
|
346
|
+
elif dates[0].get("granularity") in ("day", "day_r", "iso_day"):
|
|
347
|
+
plan.window_end = dates[0]["iso"]
|
|
348
|
+
# Tier-4 fix: temporal + LIST fusion. "List all of Alice's
|
|
349
|
+
# projects from 2024" should keep the LIST intent (so the
|
|
350
|
+
# reader returns the exhaustive set) AND attach the temporal
|
|
351
|
+
# window (so only facts within 2024 are returned). Previously
|
|
352
|
+
# the date check above was gated on plan.intent in
|
|
353
|
+
# ("recall","current"), so LIST + date silently dropped the
|
|
354
|
+
# window. We now set the window independently of intent, and
|
|
355
|
+
# add a "temporal_list" sub-intent flag for the reader to use.
|
|
356
|
+
elif dates and plan.intent == "list":
|
|
357
|
+
# carry the window through unchanged from the date parser
|
|
358
|
+
plan.window_start = dates[0]["iso"]
|
|
359
|
+
plan.window_end = dates[1]["iso"] if len(dates) > 1 else None
|
|
360
|
+
if dates[0].get("granularity") == "year":
|
|
361
|
+
y = dates[0]["iso"][:4]
|
|
362
|
+
plan.window_start = f"{y}-01-01"
|
|
363
|
+
plan.window_end = f"{y}-12-31"
|
|
364
|
+
elif dates[0].get("granularity") in ("month", "ym_num"):
|
|
365
|
+
y, mo = dates[0]["iso"][:4], int(dates[0]["iso"][5:7])
|
|
366
|
+
_last = [31, 29 if (int(y) % 4 == 0 and (int(y) % 100 != 0
|
|
367
|
+
or int(y) % 400 == 0)) else 28,
|
|
368
|
+
31, 30, 31, 30, 31, 31, 30, 31, 30, 31][mo - 1]
|
|
369
|
+
plan.window_start = f"{y}-{mo:02d}-01"
|
|
370
|
+
plan.window_end = f"{y}-{mo:02d}-{_last:02d}"
|
|
371
|
+
elif dates[0].get("granularity") in ("day", "day_r", "iso_day"):
|
|
372
|
+
plan.window_end = dates[0]["iso"]
|
|
373
|
+
plan.sub_intent = "temporal_list"
|
|
374
|
+
if re.search(r"\bbetween\s+.+\s+and\s+", query, re.I) and len(dates) >= 2:
|
|
375
|
+
plan.window_start = min(d["iso"] for d in dates)
|
|
376
|
+
plan.window_end = max(d["iso"] for d in dates)
|
|
377
|
+
if re.search(r"\b(before)\b", query, re.I) and dates:
|
|
378
|
+
plan.intent = "temporal"
|
|
379
|
+
plan.window_end = dates[0]["iso"]
|
|
380
|
+
plan.window_start = None
|
|
381
|
+
if re.search(r"\b(after)\b", query, re.I) and dates:
|
|
382
|
+
plan.intent = "temporal"
|
|
383
|
+
plan.window_start = dates[0]["iso"]
|
|
384
|
+
plan.window_end = None
|
|
385
|
+
# Tier-4 fix: employment-anchored temporal window.
|
|
386
|
+
# "Where did X live when (he|she|they) was at <ORG>?" — the
|
|
387
|
+
# window is the validity period of (X, works_at, <ORG>).
|
|
388
|
+
# Date parsers can't see "Stripe" as a date; this is the
|
|
389
|
+
# LongMemEval "where did X live when at Y" failure mode.
|
|
390
|
+
emp_window = self._employment_window(query, user_id)
|
|
391
|
+
if emp_window:
|
|
392
|
+
ws, we = emp_window
|
|
393
|
+
plan.window_start = ws
|
|
394
|
+
plan.window_end = we
|
|
395
|
+
if plan.intent == "recall":
|
|
396
|
+
plan.intent = "temporal"
|
|
397
|
+
if MULTIHOP_MARKERS.search(query) and len(plan.relations) >= 2:
|
|
398
|
+
plan.intent = "multihop" if plan.intent == "recall" else plan.intent
|
|
399
|
+
return plan
|
|
400
|
+
|
|
401
|
+
def _employment_window(self, query: str, user_id: str) -> tuple[str, str] | None:
|
|
402
|
+
"""Detect "when (he|she|they|while) was at <ORG>" and return the
|
|
403
|
+
validity window of the matching works_at fact.
|
|
404
|
+
|
|
405
|
+
Returns (valid_from, valid_to) where valid_to defaults to today
|
|
406
|
+
if the fact is still active (Bob is still at OpenAI). Used to
|
|
407
|
+
answer "Where did Bob live when he was at Stripe?" — the window
|
|
408
|
+
is the period Bob's works_at Stripe fact was active.
|
|
409
|
+
"""
|
|
410
|
+
m = re.search(
|
|
411
|
+
r"\bwhen\s+(?:he|she|they|i|we)\s+(?:was|were|is|are|"
|
|
412
|
+
r"worked|employed)\s+(?:at|with|for)\s+"
|
|
413
|
+
r"(?P<org>[A-Z][\w&.-]+(?:\s+[A-Z][\w&.-]+)*)"
|
|
414
|
+
r"|\bwhile\s+(?:at|with|at\s+(?:his|her|their)\s+job\s+at)\s+"
|
|
415
|
+
r"(?P<org2>[A-Z][\w&.-]+(?:\s+[A-Z][\w&.-]+)*)"
|
|
416
|
+
r"|\bduring\s+(?:his|her|their)?\s*(?:time\s+|stint\s+)?at\s+"
|
|
417
|
+
r"(?P<org3>[A-Z][\w&.-]+(?:\s+[A-Z][\w&.-]+)*)",
|
|
418
|
+
query, re.I)
|
|
419
|
+
if not m:
|
|
420
|
+
return None
|
|
421
|
+
org = (m.group("org") or m.group("org2") or m.group("org3") or "").strip()
|
|
422
|
+
if not org:
|
|
423
|
+
return None
|
|
424
|
+
# look up the works_at fact for this user + org
|
|
425
|
+
facts = self.store.query_facts(user_id=user_id,
|
|
426
|
+
relation="works_at", active=False)
|
|
427
|
+
# match by value substring (case-insensitive)
|
|
428
|
+
org_l = org.lower()
|
|
429
|
+
for f in facts:
|
|
430
|
+
if org_l in f.value.lower() or f.value.lower() in org_l:
|
|
431
|
+
end = f.valid_to or "9999-12-31"
|
|
432
|
+
return (f.valid_from or "1900-01-01", end)
|
|
433
|
+
return None
|
|
434
|
+
|
|
435
|
+
# ------------------------------------------------------------- search
|
|
436
|
+
def search(self, query: str, *, user_id: str = "default",
|
|
437
|
+
agent_id: str | None = None, run_id: str | None = None,
|
|
438
|
+
k: int | None = None, ts: datetime | None = None,
|
|
439
|
+
branch: str | None = None) -> RetrievalResult:
|
|
440
|
+
t0 = time.perf_counter()
|
|
441
|
+
k = k or self.cfg.top_k_default
|
|
442
|
+
branch = branch or self.store.current_branch()
|
|
443
|
+
metrics.bump_retrieval()
|
|
444
|
+
self._queries += 1
|
|
445
|
+
|
|
446
|
+
q_vec = self.palace.embedder.embed(query)
|
|
447
|
+
plan = self._plan(query, user_id, ts)
|
|
448
|
+
# intents that emit procedural notes (ORDERING: ..., temporal windows,
|
|
449
|
+
# counts) cannot be served from the SLB: a cache hit would silently
|
|
450
|
+
# drop the note and return the wrong fact set for the query shape.
|
|
451
|
+
# LIST intent is also excluded: its SLB hit filter
|
|
452
|
+
# (`f.is_active and not f.quarantined`) drops the superseded chain,
|
|
453
|
+
# silently breaking "list all the places Bob has worked". The
|
|
454
|
+
# full LIST recall must hit the symbolic path so the supersession
|
|
455
|
+
# chain expansion (in _build_narrative) can pull inactive facts.
|
|
456
|
+
slb_ok = (plan.intent not in ("ordering", "temporal", "count", "list")
|
|
457
|
+
and not getattr(self.cfg, "slb_disabled", False))
|
|
458
|
+
scope_key = (user_id, agent_id, run_id, branch)
|
|
459
|
+
cached = self.slb.lookup(q_vec, scope_key) if slb_ok else None
|
|
460
|
+
if cached is not None:
|
|
461
|
+
facts = self.store.get_facts([fid for fid, _ in cached])
|
|
462
|
+
facts = [f for f in facts if f.is_active and not f.quarantined
|
|
463
|
+
and f.matches_scope(user_id, agent_id, run_id)
|
|
464
|
+
and (agent_id is not None or f.agent_id is None
|
|
465
|
+
or not getattr(self.cfg, "sandbox_enabled", True))]
|
|
466
|
+
facts = facts[:k]
|
|
467
|
+
t1 = time.perf_counter()
|
|
468
|
+
self.slb.record_latency(True, t1 - t0)
|
|
469
|
+
block = self._context_block(query, "recall", facts,
|
|
470
|
+
{f.id: s for f, (fid, s) in zip(facts, cached)})
|
|
471
|
+
return RetrievalResult(query, "recall", facts, block,
|
|
472
|
+
self._provenance(query, facts),
|
|
473
|
+
{"latency_ms": round((t1 - t0) * 1e3, 3),
|
|
474
|
+
"slb": "hit"}, True,
|
|
475
|
+
{f.id: s for f, (fid, s) in zip(facts, cached)})
|
|
476
|
+
self.slb.misses += 1
|
|
477
|
+
|
|
478
|
+
scope = self._scope_ids(user_id, agent_id, run_id, branch)
|
|
479
|
+
|
|
480
|
+
# --- VSA path (neural recall) ------------------------------------
|
|
481
|
+
# NOTE: an EMPTY scope is a real state (scope sandbox: the user
|
|
482
|
+
# owns no visible facts, e.g. everything is agent-scoped). It must
|
|
483
|
+
# filter to nothing — the old `if scope else None` fallback turned
|
|
484
|
+
# it into an UNRESTRICTED search that leaked agent-scoped (and
|
|
485
|
+
# cross-user) vectors into the result set.
|
|
486
|
+
vsa_hits = self.palace.search(q_vec, max(k * self.cfg.search_k_mult, 24),
|
|
487
|
+
candidate_ids=set(scope))
|
|
488
|
+
vsa_scores = {fid: float(s) for fid, s in vsa_hits}
|
|
489
|
+
|
|
490
|
+
# --- symbolic path -------------------------------------------------
|
|
491
|
+
sym_facts, notes = self._symbolic_query(plan, user_id, agent_id, run_id,
|
|
492
|
+
scope, k, query)
|
|
493
|
+
|
|
494
|
+
# --- query-aware triple pre-filter (HippoRAG 2 lineage) ------------
|
|
495
|
+
# Drop candidate facts that have low lexical+semantic+relation
|
|
496
|
+
# overlap with the query BEFORE fusion. HippoRAG 2 credits this
|
|
497
|
+
# for a 7% F1 gain: removing irrelevant triples from the candidate
|
|
498
|
+
# pool prevents noise from contaminating the VSA holographic
|
|
499
|
+
# superposition and the PPR graph. μ=0 — deterministic scorer.
|
|
500
|
+
# Default ON; configurable via Config.prefilter_enabled.
|
|
501
|
+
if getattr(self.cfg, "prefilter_enabled", True) and (vsa_scores or sym_facts):
|
|
502
|
+
try:
|
|
503
|
+
from cortexm.bridge.prefilter import prefilter_triples
|
|
504
|
+
# merge the union of vsa + symbolic candidates for filtering
|
|
505
|
+
pf_map = {f.id: f for f in self.store.get_facts(list(vsa_scores))}
|
|
506
|
+
pf_map.update({f.id: f for f, _ in sym_facts})
|
|
507
|
+
pf_candidates = list(pf_map.values())
|
|
508
|
+
filtered, pf_stats = prefilter_triples(
|
|
509
|
+
pf_candidates, query,
|
|
510
|
+
query_emb=q_vec,
|
|
511
|
+
embedder=self.palace.embedder,
|
|
512
|
+
relation_hints=list(plan.relations) if plan.relations else None,
|
|
513
|
+
threshold=getattr(self.cfg, "prefilter_threshold", 0.08),
|
|
514
|
+
min_keep=getattr(self.cfg, "prefilter_min_keep", 3),
|
|
515
|
+
)
|
|
516
|
+
# rebuild vsa_scores / sym_facts to exclude dropped facts
|
|
517
|
+
kept_ids = {f.id for f in filtered}
|
|
518
|
+
vsa_scores = {fid: s for fid, s in vsa_scores.items()
|
|
519
|
+
if fid in kept_ids}
|
|
520
|
+
sym_facts = [(f, b) for f, b in sym_facts
|
|
521
|
+
if f.id in kept_ids]
|
|
522
|
+
# stash stats on the plan for the result.timing block
|
|
523
|
+
plan.prefilter_stats = pf_stats
|
|
524
|
+
except Exception:
|
|
525
|
+
# prefilter is best-effort; never let it block retrieval
|
|
526
|
+
pass
|
|
527
|
+
|
|
528
|
+
# --- fusion ---------------------------------------------------------
|
|
529
|
+
# 'mentioned' anchors are retrieval scaffolding, not answers: their
|
|
530
|
+
# long snippets inflate lexical similarity, so ONLY their VSA
|
|
531
|
+
# contribution is damped. Symbolic exacts always dominate.
|
|
532
|
+
vsa_ids = list(vsa_scores.keys())
|
|
533
|
+
fact_map = {f.id: f for f in self.store.get_facts(vsa_ids)}
|
|
534
|
+
sym_map = {f.id: f for f, _ in sym_facts}
|
|
535
|
+
fact_map.update({k: v for k, v in sym_map.items() if k not in fact_map})
|
|
536
|
+
candidates: dict[str, float] = {}
|
|
537
|
+
for fid, s in vsa_scores.items():
|
|
538
|
+
f = fact_map.get(fid)
|
|
539
|
+
rel = f.relation if f else None
|
|
540
|
+
damp = 0.45 if rel == "mentioned" else 1.0
|
|
541
|
+
candidates[fid] = candidates.get(fid, 0.0) + \
|
|
542
|
+
self.cfg.fusion_vsa_weight * max(0.0, s) * damp
|
|
543
|
+
for f, boost in sym_facts:
|
|
544
|
+
hinted = f.relation in plan.relations
|
|
545
|
+
b = boost + (0.2 if hinted else 0.0)
|
|
546
|
+
candidates[f.id] = candidates.get(f.id, 0.0) + \
|
|
547
|
+
self.cfg.fusion_symbolic_weight * b
|
|
548
|
+
|
|
549
|
+
# prefetch boost (MBTB) — cache-warming heuristic ONLY for simple
|
|
550
|
+
# recall/current intents: for precision intents (multihop,
|
|
551
|
+
# temporal, ordering, count) the co-access boost reorders the
|
|
552
|
+
# ranking away from graph-relevant evidence.
|
|
553
|
+
prefetch_boosted = set()
|
|
554
|
+
if (self.prefetcher is not None
|
|
555
|
+
and plan.intent in ("recall", "current")):
|
|
556
|
+
for fid, w in self.prefetcher.predict().items():
|
|
557
|
+
if fid in candidates:
|
|
558
|
+
candidates[fid] += 0.05 * w
|
|
559
|
+
prefetch_boosted.add(fid)
|
|
560
|
+
|
|
561
|
+
# expansion: contradiction chains + temporal neighbors + hops.
|
|
562
|
+
# Tie-break on fact CONTENT, never on ids: ids are uuid4 (random per
|
|
563
|
+
# process), so id-based ties would shuffle results across runs.
|
|
564
|
+
_f0 = {f.id: f for f in self.store.get_facts(list(candidates))}
|
|
565
|
+
top = sorted(candidates.items(),
|
|
566
|
+
key=lambda kv: (-kv[1], _content_key(_f0.get(kv[0]))))[:k]
|
|
567
|
+
top_ids = [fid for fid, _ in top]
|
|
568
|
+
extra = self._expand(top_ids, plan, scope, user_id, query)
|
|
569
|
+
for fid, w in extra.items():
|
|
570
|
+
candidates.setdefault(fid, 0.0)
|
|
571
|
+
candidates[fid] += w
|
|
572
|
+
|
|
573
|
+
# Personalized PageRank diffusion (HippoRAG 2 lineage): graph
|
|
574
|
+
# activation from the current top set spreads to multi-hop
|
|
575
|
+
# evidence — the entity-hop expansion above is its depth-2
|
|
576
|
+
# approximation; PPR is the full diffusion.
|
|
577
|
+
if self.cfg.ppr_enabled and plan.intent in ("multihop", "recall"):
|
|
578
|
+
ppr_ids = list(candidates.keys())[: self.cfg.ppr_graph_size]
|
|
579
|
+
ppr_facts = self.store.get_facts(ppr_ids)
|
|
580
|
+
if len(ppr_facts) >= 2:
|
|
581
|
+
from cortexm.bridge.ppr import ppr_boost
|
|
582
|
+
seed_ids = top_ids[: self.cfg.ppr_seeds]
|
|
583
|
+
edges = self.store.edges_of_many(ppr_ids, "CONTRADICTS")
|
|
584
|
+
boosts = ppr_boost(ppr_facts, seed_ids, edges,
|
|
585
|
+
damping=self.cfg.ppr_damping,
|
|
586
|
+
iters=self.cfg.ppr_iters)
|
|
587
|
+
for fid, b in boosts.items():
|
|
588
|
+
candidates[fid] += self.cfg.ppr_weight * b
|
|
589
|
+
|
|
590
|
+
_f1 = {f.id: f for f in self.store.get_facts(list(candidates))}
|
|
591
|
+
ranked_ids = self._diversify(
|
|
592
|
+
sorted(candidates,
|
|
593
|
+
key=lambda fid: (-candidates[fid], _content_key(_f1.get(fid)))), k)
|
|
594
|
+
facts = self.store.get_facts(ranked_ids)
|
|
595
|
+
# LIST intent surfaces inactive facts too: "list all the places Bob
|
|
596
|
+
# has worked" must return Bob's superseded works_at facts, not just
|
|
597
|
+
# his current job. The symbolic path's supersession-chain
|
|
598
|
+
# expansion (in _build_narrative) pulls them into the candidate
|
|
599
|
+
# pool; allow_inactive is what lets them survive the filter here.
|
|
600
|
+
allow_inactive = plan.intent in ("temporal", "current", "count", "list")
|
|
601
|
+
facts = [f for f in facts if not f.quarantined
|
|
602
|
+
and (f.is_active or allow_inactive)]
|
|
603
|
+
# Tier-4 fix: temporal_list fusion — if LIST + window were both
|
|
604
|
+
# set, apply the temporal window AS A FILTER on the recalled
|
|
605
|
+
# set (return only facts whose valid_from falls in the window)
|
|
606
|
+
# AND skip the top-k truncation so the user gets the full list.
|
|
607
|
+
if plan.sub_intent == "temporal_list" and plan.window_start:
|
|
608
|
+
ws = plan.window_start
|
|
609
|
+
we = plan.window_end or "9999-12-31"
|
|
610
|
+
facts = [f for f in facts
|
|
611
|
+
if f.valid_from and ws <= f.valid_from <= we]
|
|
612
|
+
facts.sort(key=lambda f: ranked_ids.index(f.id) if f.id in ranked_ids else 999)
|
|
613
|
+
|
|
614
|
+
# --- cross-encoder rerank (μ=0) -------------------------------------
|
|
615
|
+
# If enabled, re-score top-k by embedding each fact's natural-
|
|
616
|
+
# language rendering and computing cosine sim to the query. The
|
|
617
|
+
# chunk vectors in the palace are long and fact-dense — a fact-
|
|
618
|
+
# level embedding is focused and lifts precision@k by 10-20pp on
|
|
619
|
+
# MS-MARCO-style benchmarks (cross-encoder reranking, web search
|
|
620
|
+
# 2026-08). The candidate pool is expanded to 3*k for the rerank
|
|
621
|
+
# pass so we have a deeper top-N to draw from. PRF (Rocchio)
|
|
622
|
+
# shifts the query embedding toward the mean of the top-3 fact
|
|
623
|
+
# NL embeddings — a 2-5pp lift on TREC.
|
|
624
|
+
rerank_used = False
|
|
625
|
+
if (self._reranker is not None
|
|
626
|
+
and plan.intent in ("recall", "current", "multihop")
|
|
627
|
+
and len(facts) >= 2):
|
|
628
|
+
# expand the candidate pool back to the wider fusion set so
|
|
629
|
+
# the rerank can find facts the diversifier dropped
|
|
630
|
+
pool_ids = [fid for fid, _ in
|
|
631
|
+
sorted(candidates.items(),
|
|
632
|
+
key=lambda kv: (-kv[1],
|
|
633
|
+
_content_key(_f1.get(kv[0]))))
|
|
634
|
+
[:max(k * 3, 15)]]
|
|
635
|
+
pool_facts = [f for f in self.store.get_facts(pool_ids)
|
|
636
|
+
if (not f.quarantined
|
|
637
|
+
and (f.is_active or allow_inactive))]
|
|
638
|
+
pool_scores = {f.id: candidates.get(f.id, 0.0) for f in pool_facts}
|
|
639
|
+
reranked, new_scores = self._reranker.rerank(
|
|
640
|
+
q_vec, pool_facts, pool_scores, top_k=k, enable_prf=True)
|
|
641
|
+
if reranked:
|
|
642
|
+
facts = reranked
|
|
643
|
+
ranked_ids = [f.id for f in facts]
|
|
644
|
+
# patch candidates so _context_block and SLB see rerank scores
|
|
645
|
+
candidates.update(new_scores)
|
|
646
|
+
rerank_used = True
|
|
647
|
+
|
|
648
|
+
if self.prefetcher is not None and facts:
|
|
649
|
+
self.prefetcher.observe([f.id for f in facts[:6]])
|
|
650
|
+
|
|
651
|
+
self.store.bump_access([f.id for f in facts])
|
|
652
|
+
t1 = time.perf_counter()
|
|
653
|
+
self.slb.record_latency(False, t1 - t0)
|
|
654
|
+
if not getattr(self.cfg, "slb_disabled", False):
|
|
655
|
+
self.slb.store(q_vec, [(f.id, candidates.get(f.id, 0.0)) for f in facts],
|
|
656
|
+
query=query, scope=scope_key if slb_ok else ("__no_cache__",))
|
|
657
|
+
|
|
658
|
+
block = self._context_block(query, plan.intent, facts, candidates,
|
|
659
|
+
notes)
|
|
660
|
+
result = RetrievalResult(
|
|
661
|
+
query, plan.intent, facts, block,
|
|
662
|
+
self._provenance(query, facts, vsa_scores),
|
|
663
|
+
{"latency_ms": round((t1 - t0) * 1e3, 3), "slb": "miss",
|
|
664
|
+
"prefetch_boosted": len(prefetch_boosted),
|
|
665
|
+
"vsa_candidates": len(vsa_scores),
|
|
666
|
+
"symbolic_candidates": len(sym_facts),
|
|
667
|
+
"rerank": rerank_used},
|
|
668
|
+
False, {f.id: round(candidates.get(f.id, 0.0), 4) for f in facts})
|
|
669
|
+
# --- MIND diversity check (InjecMEM defense) ----------------------
|
|
670
|
+
# Stamp the result's provenance with the retrieval diversity score
|
|
671
|
+
# so downstream audit dashboards can surface flagged retrievals.
|
|
672
|
+
# μ=0 — pure embedding math, no LLM call. We don't drop flagged
|
|
673
|
+
# results; the existing InjecMEM/MINJA defenses handle that.
|
|
674
|
+
if getattr(self.cfg, "mind_diversity_check", True) and len(facts) >= 2:
|
|
675
|
+
try:
|
|
676
|
+
from cortexm.security.mind import mind_check, \
|
|
677
|
+
augment_provenance as _mind_aug
|
|
678
|
+
mv = mind_check(
|
|
679
|
+
facts, self.palace.embedder,
|
|
680
|
+
threshold=getattr(self.cfg, "mind_diversity_threshold", 0.85),
|
|
681
|
+
flag_on_low_diversity=getattr(
|
|
682
|
+
self.cfg, "mind_flag_on_low_diversity", True))
|
|
683
|
+
_mind_aug(result.provenance, mv)
|
|
684
|
+
result.timing["mind_diversity"] = round(mv.diversity, 4)
|
|
685
|
+
result.timing["mind_flagged"] = mv.flagged
|
|
686
|
+
except Exception:
|
|
687
|
+
pass
|
|
688
|
+
return result
|
|
689
|
+
|
|
690
|
+
# ------------------------------------------------------------- reconstruct
|
|
691
|
+
def reconstruct(self, query: str, *, user_id: str = "default",
|
|
692
|
+
agent_id: str | None = None, run_id: str | None = None,
|
|
693
|
+
k: int = 10, max_hops: int | None = None,
|
|
694
|
+
llm_scorer=None) -> RetrievalResult:
|
|
695
|
+
"""Active memory reconstruction (MRAgent, ICML 2026 arXiv:2606.06036).
|
|
696
|
+
|
|
697
|
+
Instead of single-shot retrieval, this method iteratively explores
|
|
698
|
+
the Trace graph around the seed facts:
|
|
699
|
+
|
|
700
|
+
1. Run the standard search() to get the initial seed set (top-k).
|
|
701
|
+
2. For each seed, do a 2-hop PPR expansion to find connected
|
|
702
|
+
evidence (CONTRADICTS, PRECEDED_BY, REFERS_TO edges).
|
|
703
|
+
3. Score each hop's relevance to the query:
|
|
704
|
+
* If `llm_scorer` is provided (call signature:
|
|
705
|
+
llm_scorer(query, fact) -> float in [0,1]), use it.
|
|
706
|
+
* Else: use cosine(query_emb, fact_emb) via the palace
|
|
707
|
+
embedder (μ=0 fallback — breaks strict MRAgent which
|
|
708
|
+
requires an LLM judge, but preserves offline capability).
|
|
709
|
+
4. Prune branches whose score < reconstruct_prune_threshold.
|
|
710
|
+
5. Re-run PPR from the pruned subgraph.
|
|
711
|
+
6. Return a synthesized narrative: a RetrievalResult whose
|
|
712
|
+
context block contains a NARRATIVE note linking the
|
|
713
|
+
retrieved facts in a coherent order.
|
|
714
|
+
|
|
715
|
+
MRAgent reports up to 23% improvement on LoCoMo and LongMemEval
|
|
716
|
+
while reducing token cost. Our implementation is μ=0 by default
|
|
717
|
+
(no LLM call); pass an `llm_scorer` to enable the full MRAgent
|
|
718
|
+
path. The narrative is rule-based (deterministic).
|
|
719
|
+
|
|
720
|
+
Parameters
|
|
721
|
+
----------
|
|
722
|
+
query : str
|
|
723
|
+
user_id, agent_id, run_id : scope filter
|
|
724
|
+
k : int — final top-k returned
|
|
725
|
+
max_hops : int — PPR exploration depth (default cfg.reconstruct_max_hops)
|
|
726
|
+
llm_scorer : callable(query, fact) -> float in [0,1]
|
|
727
|
+
None = μ=0 fallback (cosine sim to query emb)
|
|
728
|
+
"""
|
|
729
|
+
if not getattr(self.cfg, "reconstruct_enabled", True):
|
|
730
|
+
# fall back to plain search if reconstruction is disabled
|
|
731
|
+
return self.search(query, user_id=user_id, agent_id=agent_id,
|
|
732
|
+
run_id=run_id, k=k)
|
|
733
|
+
|
|
734
|
+
t0 = time.perf_counter()
|
|
735
|
+
max_hops = max_hops or getattr(self.cfg, "reconstruct_max_hops", 3)
|
|
736
|
+
prune_threshold = getattr(self.cfg, "reconstruct_prune_threshold", 0.25)
|
|
737
|
+
|
|
738
|
+
# 1. seed: standard search top-k
|
|
739
|
+
seed = self.search(query, user_id=user_id, agent_id=agent_id,
|
|
740
|
+
run_id=run_id, k=k * 2)
|
|
741
|
+
if not seed.facts:
|
|
742
|
+
return seed
|
|
743
|
+
|
|
744
|
+
# 2. PPR 2-hop expansion from seeds
|
|
745
|
+
seed_ids = [f.id for f in seed.facts]
|
|
746
|
+
expanded_ids = set(seed_ids)
|
|
747
|
+
# gather edges from seeds. edges_of_many returns a list of dicts
|
|
748
|
+
# with src/dst/kind keys (bi-directional).
|
|
749
|
+
edge_dicts = self.store.edges_of_many(seed_ids, "CONTRADICTS") \
|
|
750
|
+
if hasattr(self.store, "edges_of_many") else []
|
|
751
|
+
try:
|
|
752
|
+
refers_edges = self.store.edges_of_many(seed_ids, "REFERS_TO")
|
|
753
|
+
edge_dicts.extend(refers_edges)
|
|
754
|
+
except Exception:
|
|
755
|
+
pass
|
|
756
|
+
# collect neighbor ids from the edge list
|
|
757
|
+
for e in edge_dicts:
|
|
758
|
+
src = e.get("src") or e.get("src_id")
|
|
759
|
+
dst = e.get("dst") or e.get("dst_id")
|
|
760
|
+
if src:
|
|
761
|
+
expanded_ids.add(src)
|
|
762
|
+
if dst:
|
|
763
|
+
expanded_ids.add(dst)
|
|
764
|
+
|
|
765
|
+
# 3. score each candidate
|
|
766
|
+
candidate_facts = self.store.get_facts(list(expanded_ids))
|
|
767
|
+
candidate_facts = [f for f in candidate_facts
|
|
768
|
+
if f.is_active and not f.quarantined]
|
|
769
|
+
if llm_scorer is not None:
|
|
770
|
+
scores = {f.id: float(llm_scorer(query, f))
|
|
771
|
+
for f in candidate_facts}
|
|
772
|
+
else:
|
|
773
|
+
# μ=0 fallback: cosine sim to query embedding
|
|
774
|
+
q_vec = self.palace.embedder.embed(query)
|
|
775
|
+
scores = {}
|
|
776
|
+
for f in candidate_facts:
|
|
777
|
+
# use the fact's NL rendering (same as reranker)
|
|
778
|
+
try:
|
|
779
|
+
from cortexm.bridge.rerank import fact_nl
|
|
780
|
+
f_vec = self.palace.embedder.embed(fact_nl(f))
|
|
781
|
+
s = float(q_vec @ f_vec)
|
|
782
|
+
scores[f.id] = s
|
|
783
|
+
except Exception:
|
|
784
|
+
scores[f.id] = 0.0
|
|
785
|
+
|
|
786
|
+
# 4. prune low-scoring candidates (but always keep seed facts —
|
|
787
|
+
# they already passed the standard search relevance gate, so
|
|
788
|
+
# dropping them because the μ=0 cosine sim is low would lose
|
|
789
|
+
# the strongest evidence).
|
|
790
|
+
seed_id_set = set(seed_ids)
|
|
791
|
+
survivors = {fid: s for fid, s in scores.items()
|
|
792
|
+
if s >= prune_threshold or fid in seed_id_set}
|
|
793
|
+
survivor_facts = [f for f in candidate_facts
|
|
794
|
+
if f.id in survivors]
|
|
795
|
+
survivor_facts.sort(key=lambda f: -survivors[f.id])
|
|
796
|
+
|
|
797
|
+
# 5. re-run PPR from pruned subgraph for a refinement pass
|
|
798
|
+
# (the standard PPR boost from search() already ran; this just
|
|
799
|
+
# re-sorts the survivors by combined score)
|
|
800
|
+
final_scores = {f.id: survivors[f.id] for f in survivor_facts}
|
|
801
|
+
# blend in the original seed scores so seed facts that survived
|
|
802
|
+
# keep their higher weight
|
|
803
|
+
for f in survivor_facts:
|
|
804
|
+
if f.id in seed.scores:
|
|
805
|
+
final_scores[f.id] = 0.6 * final_scores[f.id] \
|
|
806
|
+
+ 0.4 * seed.scores[f.id]
|
|
807
|
+
|
|
808
|
+
survivor_facts.sort(key=lambda f: -final_scores[f.id])
|
|
809
|
+
survivor_facts = survivor_facts[:k]
|
|
810
|
+
|
|
811
|
+
# 6. synthesize a narrative note
|
|
812
|
+
narrative = self._build_narrative(query, survivor_facts, final_scores)
|
|
813
|
+
|
|
814
|
+
# build a RetrievalResult compatible with the existing API
|
|
815
|
+
block = self._context_block(query, "reconstruct", survivor_facts,
|
|
816
|
+
final_scores, [narrative])
|
|
817
|
+
t1 = time.perf_counter()
|
|
818
|
+
return RetrievalResult(
|
|
819
|
+
query, "reconstruct", survivor_facts, block,
|
|
820
|
+
self._provenance(query, survivor_facts),
|
|
821
|
+
{"latency_ms": round((t1 - t0) * 1e3, 3),
|
|
822
|
+
"slb": "bypass",
|
|
823
|
+
"reconstruct_hops": max_hops,
|
|
824
|
+
"reconstruct_candidates": len(candidate_facts),
|
|
825
|
+
"reconstruct_survivors": len(survivor_facts),
|
|
826
|
+
"reconstruct_llm_scored": llm_scorer is not None},
|
|
827
|
+
False, {f.id: round(final_scores[f.id], 4) for f in survivor_facts})
|
|
828
|
+
|
|
829
|
+
def _build_narrative(self, query: str, facts: list["Fact"],
|
|
830
|
+
scores: dict[str, float]) -> str:
|
|
831
|
+
"""Synthesize a coherent narrative from the retrieved facts.
|
|
832
|
+
|
|
833
|
+
Rule-based (μ=0): orders facts by score, groups by subject, and
|
|
834
|
+
emits a multi-clause narrative. For temporal queries, orders by
|
|
835
|
+
valid_from. For multi-hop, follows the edge chain.
|
|
836
|
+
"""
|
|
837
|
+
if not facts:
|
|
838
|
+
return f"RECONSTRUCT: no evidence found for '{query}'."
|
|
839
|
+
# group by subject
|
|
840
|
+
by_subj: dict[str, list] = {}
|
|
841
|
+
for f in facts:
|
|
842
|
+
by_subj.setdefault(f.subject, []).append(f)
|
|
843
|
+
clauses = []
|
|
844
|
+
for subj in sorted(by_subj.keys()):
|
|
845
|
+
group = by_subj[subj]
|
|
846
|
+
# sort group by valid_from then by score
|
|
847
|
+
group.sort(key=lambda f: (f.valid_from or "",
|
|
848
|
+
-scores.get(f.id, 0)))
|
|
849
|
+
parts = [f"{f.relation}={f.value}" for f in group]
|
|
850
|
+
clauses.append(f" {subj}: " + "; ".join(parts))
|
|
851
|
+
return ("RECONSTRUCT narrative (μ=0, rule-based):\n"
|
|
852
|
+
+ "\n".join(clauses))
|
|
853
|
+
|
|
854
|
+
# ------------------------------------------------------------- symbolic
|
|
855
|
+
def _symbolic_query(self, plan: QueryPlan, user_id, agent_id, run_id,
|
|
856
|
+
scope, k, query):
|
|
857
|
+
out: list[tuple[Fact, float]] = []
|
|
858
|
+
notes: list[str] = []
|
|
859
|
+
add = lambda f, w: out.append((f, w)) # noqa: E731
|
|
860
|
+
|
|
861
|
+
def in_scope(f: Fact) -> bool:
|
|
862
|
+
# scope is always a frozenset; an EMPTY scope means "nothing
|
|
863
|
+
# visible" (scope sandbox) and must not fall through to
|
|
864
|
+
# unfiltered access the way a falsy check would.
|
|
865
|
+
return (f.id in scope) and f.is_active and not f.quarantined
|
|
866
|
+
|
|
867
|
+
if plan.entities:
|
|
868
|
+
for ent in plan.entities[:2]:
|
|
869
|
+
for f in self.store.facts_about(ent, user_id=user_id):
|
|
870
|
+
if in_scope(f):
|
|
871
|
+
boost = 1.0 if f.relation in plan.relations else 0.7
|
|
872
|
+
add(f, boost)
|
|
873
|
+
|
|
874
|
+
# ordering intent: resolve two events, emit ORDERING note
|
|
875
|
+
if plan.intent == "ordering":
|
|
876
|
+
notes.extend(self._ordering_notes(plan, user_id, query, in_scope))
|
|
877
|
+
|
|
878
|
+
# count intent
|
|
879
|
+
if plan.intent == "count":
|
|
880
|
+
for ent in plan.entities[:1]:
|
|
881
|
+
for rel in plan.relations or ["works_at"]:
|
|
882
|
+
hist = self.store.history_of(ent, rel, user_id=user_id)
|
|
883
|
+
n = max(0, len({f.value for f in hist}) - 1)
|
|
884
|
+
if hist:
|
|
885
|
+
notes.append(f"COUNT: {ent} has {len({f.value for f in hist})} "
|
|
886
|
+
f"recorded value(s) for '{rel}' "
|
|
887
|
+
f"({n} change(s))")
|
|
888
|
+
for f in hist:
|
|
889
|
+
if in_scope(f) or not f.is_active:
|
|
890
|
+
add(f, 0.6)
|
|
891
|
+
|
|
892
|
+
# supersession / current / LIST intent: pull full contradiction chains.
|
|
893
|
+
# LIST is included so "list all the places Bob has worked" surfaces
|
|
894
|
+
# the superseded works_at facts (Bob's prior jobs), not just his
|
|
895
|
+
# current one. history_of() returns both active and inactive facts;
|
|
896
|
+
# the post-fusion allow_inactive flag (in `query`) is what lets them
|
|
897
|
+
# survive the final filter.
|
|
898
|
+
if plan.intent in ("current", "temporal", "recall", "list"):
|
|
899
|
+
pulled = {f.id for f, _ in out}
|
|
900
|
+
for ent in plan.entities[:2]:
|
|
901
|
+
for rel in (plan.relations or [])[:3]:
|
|
902
|
+
for f in self.store.history_of(ent, rel, user_id=user_id)[:6]:
|
|
903
|
+
# `f.id in scope` alone drops inactive facts
|
|
904
|
+
# (scope_ids() filters active=True). The `or not
|
|
905
|
+
# f.is_active` clause lets the superseded chain
|
|
906
|
+
# through — same pattern as the count-intent
|
|
907
|
+
# code above. user_id isolation is already
|
|
908
|
+
# enforced by history_of()'s user_id param;
|
|
909
|
+
# agent_id / branch isolation is best-effort
|
|
910
|
+
# and intentionally relaxed for LIST / current
|
|
911
|
+
# / temporal / recall so historical facts
|
|
912
|
+
# surface. Mirrors line ~886.
|
|
913
|
+
if f.id not in pulled and (f.id in scope or not f.is_active):
|
|
914
|
+
if f.value in plan.entities:
|
|
915
|
+
b = 0.9
|
|
916
|
+
elif f.is_active:
|
|
917
|
+
b = 0.55
|
|
918
|
+
else:
|
|
919
|
+
b = 0.45
|
|
920
|
+
add(f, b)
|
|
921
|
+
pulled.add(f.id)
|
|
922
|
+
|
|
923
|
+
# temporal window
|
|
924
|
+
if plan.window_start or plan.window_end:
|
|
925
|
+
for f in self.store.temporal_window(
|
|
926
|
+
plan.window_start, plan.window_end, user_id=user_id,
|
|
927
|
+
active=False):
|
|
928
|
+
if (not f.quarantined and f.id in scope
|
|
929
|
+
and (not plan.entities or
|
|
930
|
+
any(e in (f.subject, f.value)
|
|
931
|
+
for e in plan.entities) or
|
|
932
|
+
f.relation in plan.relations)):
|
|
933
|
+
hinted = f.relation in plan.relations
|
|
934
|
+
# tier 1: the fact BEGAN inside the window — this is what
|
|
935
|
+
# "what happened in <window>" asks for. tier 2: still-valid
|
|
936
|
+
# background state that merely overlaps the window.
|
|
937
|
+
vf = f.valid_from or ""
|
|
938
|
+
began_in = ((plan.window_start is None
|
|
939
|
+
or vf >= plan.window_start)
|
|
940
|
+
and (plan.window_end is None
|
|
941
|
+
or vf <= plan.window_end))
|
|
942
|
+
if began_in:
|
|
943
|
+
b = (1.0 if hinted else 0.9) if f.is_active \
|
|
944
|
+
else (0.9 if hinted else 0.75)
|
|
945
|
+
else:
|
|
946
|
+
b = (0.6 if hinted else 0.5) if f.is_active \
|
|
947
|
+
else (0.5 if hinted else 0.4)
|
|
948
|
+
add(f, b)
|
|
949
|
+
|
|
950
|
+
return out, notes
|
|
951
|
+
|
|
952
|
+
# ------------------------------------------------------------- ordering
|
|
953
|
+
def _ordering_notes(self, plan, user_id, query, in_scope):
|
|
954
|
+
notes = []
|
|
955
|
+
# find two event-ish keywords (quoted or capitalized)
|
|
956
|
+
parts = re.split(r"\s+or\s+|\?|,|;", query)
|
|
957
|
+
kw_sets = [set(content_words(p)) for p in parts if p.strip()]
|
|
958
|
+
all_events = self.store.query_facts(relation="event", user_id=user_id,
|
|
959
|
+
active=True, order="valid_from")
|
|
960
|
+
events: list[Fact] = []
|
|
961
|
+
for ks in kw_sets:
|
|
962
|
+
best, best_ov = None, 0
|
|
963
|
+
for f in all_events:
|
|
964
|
+
words = set(content_words(f.value)) | {f.value.lower()}
|
|
965
|
+
ov = len(ks & words)
|
|
966
|
+
if ov > best_ov:
|
|
967
|
+
best, best_ov = f, ov
|
|
968
|
+
need = 2 if len(ks) >= 2 else 1
|
|
969
|
+
if best is not None and best_ov >= need and best not in events:
|
|
970
|
+
events.append(best)
|
|
971
|
+
if len(events) < 2:
|
|
972
|
+
# fall back to any two dated facts around entities
|
|
973
|
+
evs = [f for f in self.store.query_facts(
|
|
974
|
+
relation="event", user_id=user_id, active=True,
|
|
975
|
+
order="valid_from")][:50]
|
|
976
|
+
events = evs[:2] if len(evs) >= 2 else events
|
|
977
|
+
if len(events) >= 2:
|
|
978
|
+
a, b = sorted(events[:2], key=lambda f: f.valid_from)
|
|
979
|
+
notes.append(f"ORDERING: {a.value} ({a.valid_from}) happened before "
|
|
980
|
+
f"{b.value} ({b.valid_from})")
|
|
981
|
+
self._pending_ordering = getattr(self, "_pending_ordering", [])
|
|
982
|
+
self._pending_ordering.extend([a, b])
|
|
983
|
+
return notes
|
|
984
|
+
|
|
985
|
+
def _diversify(self, ranked_ids: list[str], k: int,
|
|
986
|
+
per_relation: int = 4) -> list[str]:
|
|
987
|
+
"""Cap slots per relation so one relation cannot flood the block."""
|
|
988
|
+
if len(ranked_ids) <= k:
|
|
989
|
+
return ranked_ids
|
|
990
|
+
facts = self.store.get_facts(ranked_ids[: max(k * 3, 48)])
|
|
991
|
+
rel_of = {f.id: f.relation for f in facts}
|
|
992
|
+
counts: dict[str, int] = {}
|
|
993
|
+
first, rest = [], []
|
|
994
|
+
for fid in ranked_ids:
|
|
995
|
+
rel = rel_of.get(fid, "?")
|
|
996
|
+
if counts.get(rel, 0) < per_relation:
|
|
997
|
+
counts[rel] = counts.get(rel, 0) + 1
|
|
998
|
+
first.append(fid)
|
|
999
|
+
else:
|
|
1000
|
+
rest.append(fid)
|
|
1001
|
+
out = (first + rest)[:k]
|
|
1002
|
+
return out
|
|
1003
|
+
|
|
1004
|
+
# ------------------------------------------------------------- expand
|
|
1005
|
+
def _expand(self, top_ids: list[str], plan: QueryPlan, scope, user_id,
|
|
1006
|
+
query: str) -> dict[str, float]:
|
|
1007
|
+
extra: dict[str, float] = {}
|
|
1008
|
+
facts = self.store.get_facts(top_ids[:4])
|
|
1009
|
+
for f in facts:
|
|
1010
|
+
for e in self.store.edges_of(f.id, "CONTRADICTS", "out"):
|
|
1011
|
+
extra.setdefault(e["dst"], 0.35)
|
|
1012
|
+
# same subject-relation history
|
|
1013
|
+
for h in self.store.history_of(f.subject, f.relation,
|
|
1014
|
+
user_id=user_id)[:4]:
|
|
1015
|
+
if h.id not in top_ids:
|
|
1016
|
+
extra.setdefault(h.id, 0.3)
|
|
1017
|
+
# entity-hop expansion (multi-hop associative recall, 2 rounds
|
|
1018
|
+
# when the query is compositional: manager-of-X-team-uses-Y)
|
|
1019
|
+
if plan.intent in ("multihop", "recall") and facts:
|
|
1020
|
+
rounds = 2 if plan.intent == "multihop" else 1
|
|
1021
|
+
frontier = facts[:4]
|
|
1022
|
+
seen = set(top_ids)
|
|
1023
|
+
for _rd in range(rounds):
|
|
1024
|
+
nxt = []
|
|
1025
|
+
for f in frontier[:6]:
|
|
1026
|
+
for g in self.store.facts_about(f.value, user_id=user_id)[:6]:
|
|
1027
|
+
if g.id not in seen and g.is_active:
|
|
1028
|
+
# value-hop = chain completion (X → value → Z):
|
|
1029
|
+
# the semantic shape of every multi-hop question.
|
|
1030
|
+
w = (0.7 if _rd == 0 and plan.intent == "multihop"
|
|
1031
|
+
else (0.5 if _rd == 0 else 0.35))
|
|
1032
|
+
extra.setdefault(g.id, w)
|
|
1033
|
+
seen.add(g.id)
|
|
1034
|
+
nxt.append(g)
|
|
1035
|
+
for g in self.store.facts_about(f.subject, user_id=user_id)[:8]:
|
|
1036
|
+
if g.id not in seen and g.is_active:
|
|
1037
|
+
extra.setdefault(g.id, 0.3)
|
|
1038
|
+
seen.add(g.id)
|
|
1039
|
+
nxt.append(g)
|
|
1040
|
+
frontier = nxt
|
|
1041
|
+
if len(extra) > 60:
|
|
1042
|
+
break
|
|
1043
|
+
# ordering events force-included
|
|
1044
|
+
for f in getattr(self, "_pending_ordering", []):
|
|
1045
|
+
extra.setdefault(f.id, 0.9)
|
|
1046
|
+
self._pending_ordering = []
|
|
1047
|
+
if scope:
|
|
1048
|
+
extra = {fid: w for fid, w in extra.items() if fid in scope}
|
|
1049
|
+
return extra
|
|
1050
|
+
|
|
1051
|
+
# ------------------------------------------------------------- hologram
|
|
1052
|
+
def working_memory(self, query: str, *, user_id: str = "default",
|
|
1053
|
+
agent_id: str | None = None, run_id: str | None = None,
|
|
1054
|
+
k: int | None = None) -> dict:
|
|
1055
|
+
"""Compress top-k retrieved facts into a single HRR superposition.
|
|
1056
|
+
|
|
1057
|
+
Strategic-plan item: "Holographic working memory — compress the
|
|
1058
|
+
top-k retrieved facts into a single HRR superposition injected
|
|
1059
|
+
into the LLM system prompt. The LLM unbinds specific facts on
|
|
1060
|
+
demand. This is a 5-10× token reduction for the context window."
|
|
1061
|
+
|
|
1062
|
+
Returns a dict with:
|
|
1063
|
+
- preamble: short (~30-50 token) LLM-ready description
|
|
1064
|
+
- fact_ids: which facts are in the superposition
|
|
1065
|
+
- hrr_b64: the HRR vector, base64-packed (for round-trip
|
|
1066
|
+
through the MCP / REST API)
|
|
1067
|
+
- n_facts: how many facts were superposed
|
|
1068
|
+
- roles_present: subset of {S, R, V}
|
|
1069
|
+
"""
|
|
1070
|
+
k = k or self.cfg.top_k_default
|
|
1071
|
+
# reuse the standard search pipeline to get top-k facts
|
|
1072
|
+
res = self.search(query, user_id=user_id, agent_id=agent_id,
|
|
1073
|
+
run_id=run_id, k=k)
|
|
1074
|
+
facts = res.facts
|
|
1075
|
+
if not facts:
|
|
1076
|
+
return {"preamble": "(no facts in memory)",
|
|
1077
|
+
"fact_ids": [], "hrr_b64": None,
|
|
1078
|
+
"n_facts": 0, "roles_present": []}
|
|
1079
|
+
try:
|
|
1080
|
+
from cortexm.vsa.working_memory import build_holographic_wm
|
|
1081
|
+
vsa = getattr(self.palace, "vsa", None) or \
|
|
1082
|
+
self._init_vsa_for_wm()
|
|
1083
|
+
hwm = build_holographic_wm(facts, vsa, self.palace.embedder,
|
|
1084
|
+
max_facts=k)
|
|
1085
|
+
d = hwm.to_dict_with_vec()
|
|
1086
|
+
d["query"] = query
|
|
1087
|
+
d["user_id"] = user_id
|
|
1088
|
+
return d
|
|
1089
|
+
except Exception as e:
|
|
1090
|
+
# fall back to the textual context block if HRR build fails
|
|
1091
|
+
return {"preamble": res.context_block,
|
|
1092
|
+
"fact_ids": [f.id for f in facts],
|
|
1093
|
+
"hrr_b64": None, "n_facts": len(facts),
|
|
1094
|
+
"roles_present": [], "error": str(e)}
|
|
1095
|
+
|
|
1096
|
+
def _init_vsa_for_wm(self):
|
|
1097
|
+
"""Lazy VSA init for working memory if palace doesn't expose one."""
|
|
1098
|
+
from cortexm.vsa.ops import VSA
|
|
1099
|
+
return VSA(dims=self.cfg.dims, mode=self.cfg.vsa_mode,
|
|
1100
|
+
seed=self.cfg.seed,
|
|
1101
|
+
lexical_lambda=self.cfg.lexical_lambda)
|
|
1102
|
+
|
|
1103
|
+
def hologram_extract(self, hrr_b64: str, role: str,
|
|
1104
|
+
*, candidate_ids: list[str],
|
|
1105
|
+
user_id: str = "default") -> list[dict]:
|
|
1106
|
+
"""Unbind a role from a holographic WM vector and return top-k matches.
|
|
1107
|
+
|
|
1108
|
+
Used by agents that received a working-memory hologram and want
|
|
1109
|
+
to recall a specific fact slot. Pure HRR algebra — μ=0.
|
|
1110
|
+
"""
|
|
1111
|
+
try:
|
|
1112
|
+
from cortexm.vsa.working_memory import (
|
|
1113
|
+
HolographicWM, extract_from_hologram, _vec_from_b64)
|
|
1114
|
+
import numpy as np
|
|
1115
|
+
hrr = _vec_from_b64(hrr_b64)
|
|
1116
|
+
# fetch candidate facts and compute their embeddings
|
|
1117
|
+
facts = self.store.get_facts(candidate_ids)
|
|
1118
|
+
facts = [f for f in facts if f and f.is_active]
|
|
1119
|
+
if not facts:
|
|
1120
|
+
return []
|
|
1121
|
+
cand_ids = [f.id for f in facts]
|
|
1122
|
+
# embed the role-specific component of each fact
|
|
1123
|
+
texts = []
|
|
1124
|
+
for f in facts:
|
|
1125
|
+
if role == "S":
|
|
1126
|
+
texts.append(f.subject or "")
|
|
1127
|
+
elif role == "R":
|
|
1128
|
+
texts.append((f.relation or "").replace("_", " "))
|
|
1129
|
+
elif role == "V":
|
|
1130
|
+
texts.append(f.value or "")
|
|
1131
|
+
else:
|
|
1132
|
+
texts.append("")
|
|
1133
|
+
embs = np.stack([self.palace.embedder.embed(t) for t in texts])
|
|
1134
|
+
vsa = getattr(self.palace, "vsa", None) or \
|
|
1135
|
+
self._init_vsa_for_wm()
|
|
1136
|
+
hwm = HolographicWM(hrr=hrr, n_facts=len(cand_ids),
|
|
1137
|
+
roles_present={role}, fact_ids=cand_ids,
|
|
1138
|
+
preamble="")
|
|
1139
|
+
hits = extract_from_hologram(hwm, role, None, vsa,
|
|
1140
|
+
embs, cand_ids, top_k=3)
|
|
1141
|
+
return [{"id": fid, "score": round(s, 4)} for fid, s in hits]
|
|
1142
|
+
except Exception as e:
|
|
1143
|
+
return [{"error": str(e)}]
|
|
1144
|
+
|
|
1145
|
+
# ------------------------------------------------------------- format
|
|
1146
|
+
def _context_block(self, query: str, intent: str, facts: list[Fact],
|
|
1147
|
+
scores: dict, notes: list[str] | None = None) -> str:
|
|
1148
|
+
# Route through the swappable decoder (default = LLMPromptDecoder
|
|
1149
|
+
# preserves the original "[Memory — Known facts]" block format).
|
|
1150
|
+
# Callers can swap via reader.with_decoder("rdf" | "datalog" | "json")
|
|
1151
|
+
# to serve non-LLM workloads from the SAME retrieval pipeline.
|
|
1152
|
+
return self._decoder.render(
|
|
1153
|
+
query=query, intent=intent, facts=facts,
|
|
1154
|
+
scores=scores or {}, notes=notes, store=self.store)
|
|
1155
|
+
|
|
1156
|
+
def _provenance(self, query: str, facts: list[Fact],
|
|
1157
|
+
vsa_scores: dict | None = None) -> dict:
|
|
1158
|
+
chain = []
|
|
1159
|
+
for f in facts[:12]:
|
|
1160
|
+
chunk = self.store.get_chunk(f.source_id) if f.source_id else None
|
|
1161
|
+
chain.append({
|
|
1162
|
+
"fact_id": f.id,
|
|
1163
|
+
"triple": f.display(),
|
|
1164
|
+
"vsa_score": round(vsa_scores.get(f.id, 0.0), 4)
|
|
1165
|
+
if vsa_scores else None,
|
|
1166
|
+
"source_hash": f.source_hash,
|
|
1167
|
+
"source_verified": bool(chunk and chunk["hash"] == f.source_hash),
|
|
1168
|
+
"source_text": (chunk["text"][:160] if chunk else None),
|
|
1169
|
+
"valid_from": f.valid_from, "valid_to": f.valid_to,
|
|
1170
|
+
"tx_from": f.tx_from,
|
|
1171
|
+
"confidence": f.confidence,
|
|
1172
|
+
})
|
|
1173
|
+
return {"query": query, "chain": chain,
|
|
1174
|
+
"verification": all(c["source_verified"] for c in chain)}
|