cortexm 0.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- context_m.py +17 -0
- cortexm/__init__.py +45 -0
- cortexm/accel.py +403 -0
- cortexm/api/__init__.py +0 -0
- cortexm/api/chaos.py +118 -0
- cortexm/api/memory.py +635 -0
- cortexm/bench/__init__.py +0 -0
- cortexm/bench/abilities.py +311 -0
- cortexm/bench/baselines.py +89 -0
- cortexm/bench/beam_loader.py +317 -0
- cortexm/bench/generator.py +376 -0
- cortexm/bench/harness.py +211 -0
- cortexm/bench/messy.py +218 -0
- cortexm/bench/micro.py +251 -0
- cortexm/bench/ood.py +443 -0
- cortexm/bench/run.py +137 -0
- cortexm/bridge/__init__.py +0 -0
- cortexm/bridge/dates.py +178 -0
- cortexm/bridge/decoders.py +204 -0
- cortexm/bridge/enrich.py +255 -0
- cortexm/bridge/extractor.py +316 -0
- cortexm/bridge/fallback.py +332 -0
- cortexm/bridge/onnx_runtime.py +158 -0
- cortexm/bridge/patterns.py +760 -0
- cortexm/bridge/ppr.py +104 -0
- cortexm/bridge/prefilter.py +188 -0
- cortexm/bridge/query_extract.py +420 -0
- cortexm/bridge/reader.py +1174 -0
- cortexm/bridge/rerank.py +204 -0
- cortexm/bridge/writer.py +492 -0
- cortexm/cli.py +295 -0
- cortexm/cognition/__init__.py +53 -0
- cortexm/cognition/abstraction.py +192 -0
- cortexm/cognition/analogy.py +159 -0
- cortexm/cognition/engine.py +204 -0
- cortexm/cognition/gaps.py +365 -0
- cortexm/cognition/scanner.py +204 -0
- cortexm/config.py +375 -0
- cortexm/cortexm.py +8 -0
- cortexm/enterprise/__init__.py +0 -0
- cortexm/enterprise/audit.py +178 -0
- cortexm/enterprise/governance.py +239 -0
- cortexm/errors.py +35 -0
- cortexm/features/__init__.py +0 -0
- cortexm/features/git.py +204 -0
- cortexm/features/prefetch.py +88 -0
- cortexm/features/zk.py +105 -0
- cortexm/federation/__init__.py +39 -0
- cortexm/federation/crdt.py +275 -0
- cortexm/federation/fabric.py +109 -0
- cortexm/federation/hlc.py +80 -0
- cortexm/federation/node.py +145 -0
- cortexm/federation/schema_report.py +73 -0
- cortexm/federation/transport.py +164 -0
- cortexm/index/__init__.py +19 -0
- cortexm/index/nsg.py +386 -0
- cortexm/mcp/__init__.py +0 -0
- cortexm/mcp/server.py +985 -0
- cortexm/metrics.py +62 -0
- cortexm/migrate/__init__.py +0 -0
- cortexm/migrate/importers.py +192 -0
- cortexm/provenance/__init__.py +78 -0
- cortexm/provenance/agent.py +214 -0
- cortexm/provenance/cose.py +201 -0
- cortexm/provenance/scitt.py +258 -0
- cortexm/provenance/vc.py +250 -0
- cortexm/security/__init__.py +0 -0
- cortexm/security/crypto.py +162 -0
- cortexm/security/hashes.py +140 -0
- cortexm/security/injection.py +149 -0
- cortexm/security/mind.py +154 -0
- cortexm/security/pii.py +265 -0
- cortexm/security/rbac.py +169 -0
- cortexm/security/sandbox.py +131 -0
- cortexm/security/zk_hamming.py +142 -0
- cortexm/security/zk_sql.py +485 -0
- cortexm/server/__init__.py +0 -0
- cortexm/server/metrics.py +88 -0
- cortexm/server/rest.py +936 -0
- cortexm/server/sparql.py +984 -0
- cortexm/text/__init__.py +0 -0
- cortexm/text/dissim.py +252 -0
- cortexm/text/embedder.py +155 -0
- cortexm/text/fuzzy.py +218 -0
- cortexm/text/idiolect.py +253 -0
- cortexm/text/labse.py +374 -0
- cortexm/text/tokenizer.py +79 -0
- cortexm/trace/__init__.py +0 -0
- cortexm/trace/blob_arena.py +277 -0
- cortexm/trace/consolidate.py +337 -0
- cortexm/trace/contradictions.py +69 -0
- cortexm/trace/dedup.py +114 -0
- cortexm/trace/edges.py +214 -0
- cortexm/trace/fact.py +121 -0
- cortexm/trace/fade.py +245 -0
- cortexm/trace/lifecycle.py +112 -0
- cortexm/trace/rebuild.py +173 -0
- cortexm/trace/rules.py +171 -0
- cortexm/trace/store.py +680 -0
- cortexm/trace/structural.py +183 -0
- cortexm/trace/tmt.py +335 -0
- cortexm/util.py +148 -0
- cortexm/vsa/__init__.py +0 -0
- cortexm/vsa/attribution.py +149 -0
- cortexm/vsa/cleanup.py +161 -0
- cortexm/vsa/codecs.py +397 -0
- cortexm/vsa/hologram_overlay.py +139 -0
- cortexm/vsa/index.py +163 -0
- cortexm/vsa/ops.py +149 -0
- cortexm/vsa/palace.py +446 -0
- cortexm/vsa/role_vectors.py +236 -0
- cortexm/vsa/slb.py +78 -0
- cortexm/vsa/tlsh_trie.py +137 -0
- cortexm/vsa/working_memory.py +249 -0
- cortexm-0.3.0.dist-info/METADATA +482 -0
- cortexm-0.3.0.dist-info/RECORD +120 -0
- cortexm-0.3.0.dist-info/WHEEL +5 -0
- cortexm-0.3.0.dist-info/entry_points.txt +2 -0
- cortexm-0.3.0.dist-info/licenses/LICENSE +190 -0
- cortexm-0.3.0.dist-info/top_level.txt +2 -0
cortexm/api/memory.py
ADDED
|
@@ -0,0 +1,635 @@
|
|
|
1
|
+
"""Context-M Memory — the Mem0-compatible facade.
|
|
2
|
+
|
|
3
|
+
Drop-in surface (``pip install cortexm`` → ``from cortexm import Memory``):
|
|
4
|
+
|
|
5
|
+
m = Memory()
|
|
6
|
+
m.add("I work at Google", user_id="alice")
|
|
7
|
+
m.search("Where does Alice work?", user_id="alice")
|
|
8
|
+
m.get_all(user_id="alice")
|
|
9
|
+
m.history(memory_id)
|
|
10
|
+
|
|
11
|
+
…plus everything Mem0 does not ship: Zep-compatible temporal queries,
|
|
12
|
+
cryptographic provenance on every retrieval, Memory Git (branch/merge/
|
|
13
|
+
diff/blame), ZK-lite proofs, self-healing vector storage, predictive
|
|
14
|
+
prefetching, federation-ready schema export, and a μ=0 ingest counter
|
|
15
|
+
that proves zero LLM calls.
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
import datetime as _dt
|
|
21
|
+
import os
|
|
22
|
+
from datetime import datetime, timezone
|
|
23
|
+
|
|
24
|
+
import numpy as np
|
|
25
|
+
|
|
26
|
+
from cortexm import metrics
|
|
27
|
+
from cortexm.bridge.extractor import Extractor
|
|
28
|
+
from cortexm.bridge.reader import MemoryReader, RetrievalResult
|
|
29
|
+
from cortexm.bridge.writer import MemoryWriter
|
|
30
|
+
from cortexm.config import Config
|
|
31
|
+
from cortexm.errors import ContextMError
|
|
32
|
+
from cortexm.features.git import MemoryGit
|
|
33
|
+
from cortexm.features.prefetch import Prefetcher
|
|
34
|
+
from cortexm.features.zk import ZKProver
|
|
35
|
+
from cortexm.federation import export_schema_report, merge_schema_reports
|
|
36
|
+
from cortexm.security.hashes import HashProvider
|
|
37
|
+
from cortexm.trace import lifecycle
|
|
38
|
+
from cortexm.trace.store import TraceStore
|
|
39
|
+
from cortexm.util import parse_ts
|
|
40
|
+
from cortexm.vsa.palace import MemoryPalace
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
class Memory:
|
|
44
|
+
"""The Universal Neuro-Symbolic Memory Fabric — one object, every layer."""
|
|
45
|
+
|
|
46
|
+
def __init__(self, config: Config | None = None, **overrides) -> None:
|
|
47
|
+
if config is None:
|
|
48
|
+
config = Config.from_env(**overrides)
|
|
49
|
+
elif overrides:
|
|
50
|
+
import dataclasses
|
|
51
|
+
changes = {k: v for k, v in overrides.items()
|
|
52
|
+
if hasattr(config, k)}
|
|
53
|
+
config = dataclasses.replace(config, **changes)
|
|
54
|
+
self.config = config
|
|
55
|
+
self.store = TraceStore(config.db_path, HashProvider(config.hash_provider),
|
|
56
|
+
wal_sync=getattr(config, "wal_sync", "normal"))
|
|
57
|
+
self.palace = MemoryPalace(config, self.store)
|
|
58
|
+
self.extractor = Extractor(config)
|
|
59
|
+
self.prefetcher = Prefetcher()
|
|
60
|
+
self.writer = MemoryWriter(config, self.store, self.palace, self.extractor)
|
|
61
|
+
self.reader = MemoryReader(config, self.store, self.palace,
|
|
62
|
+
self.prefetcher)
|
|
63
|
+
self.git = MemoryGit(self.store, self.palace)
|
|
64
|
+
self.zk = ZKProver(self.store, self.reader)
|
|
65
|
+
|
|
66
|
+
# --- enterprise layer (PII, crypto, RBAC, audit, governance) ---------
|
|
67
|
+
from cortexm.security.crypto import AESGCMCipher, load_master_key
|
|
68
|
+
from cortexm.security.pii import PIIGuard, PIIVault
|
|
69
|
+
from cortexm.security.rbac import APIKeyStore
|
|
70
|
+
from cortexm.enterprise.audit import AuditLog
|
|
71
|
+
from cortexm.enterprise.governance import Governance
|
|
72
|
+
self.cipher = None
|
|
73
|
+
if config.encryption_at_rest:
|
|
74
|
+
key = load_master_key(config.master_key_path)
|
|
75
|
+
if key is None:
|
|
76
|
+
sidecar = ("" if config.db_path == ":memory:"
|
|
77
|
+
else config.db_path + ".key")
|
|
78
|
+
if sidecar and os.path.exists(sidecar):
|
|
79
|
+
key = load_master_key(sidecar)
|
|
80
|
+
if key is None:
|
|
81
|
+
from cortexm.security.crypto import generate_master_key
|
|
82
|
+
if config.db_path != ":memory:":
|
|
83
|
+
key = generate_master_key(config.db_path + ".key")
|
|
84
|
+
if key is not None:
|
|
85
|
+
self.cipher = AESGCMCipher(key, store=self.store)
|
|
86
|
+
self.pii_vault = PIIVault(self.store, self.cipher)
|
|
87
|
+
self.pii_guard = PIIGuard(config.pii_mode, self.pii_vault)
|
|
88
|
+
self.keys = APIKeyStore(self.store)
|
|
89
|
+
self.audit_log = AuditLog(self.store, enabled=config.audit_enabled)
|
|
90
|
+
self.governance = Governance(self)
|
|
91
|
+
|
|
92
|
+
# ------------------------------------------------------------ Mem0 API
|
|
93
|
+
def add(self, messages, *, user_id: str | None = None,
|
|
94
|
+
agent_id: str | None = None, run_id: str | None = None,
|
|
95
|
+
metadata: dict | None = None, timestamp=None, **kw) -> dict:
|
|
96
|
+
"""μ=0 ingest. Accepts str | list[str] | mem0-style message dicts.
|
|
97
|
+
|
|
98
|
+
When ``pii_mode`` is ``redact``/``block``, the write path passes
|
|
99
|
+
through the PII guard BEFORE extraction — raw personal data never
|
|
100
|
+
reaches facts, chunks, or vectors."""
|
|
101
|
+
user_id = user_id or self.config.default_user_id
|
|
102
|
+
ts = parse_ts(timestamp) if timestamp else None
|
|
103
|
+
if self.pii_guard.mode != "off":
|
|
104
|
+
messages = self._apply_pii(messages)
|
|
105
|
+
if messages is None:
|
|
106
|
+
self.audit_log.log("memory.add", resource=user_id,
|
|
107
|
+
outcome="blocked_pii",
|
|
108
|
+
meta={"reason": "pii_mode=block"})
|
|
109
|
+
return {"results": [], "blocked": "pii_policy"}
|
|
110
|
+
out = self.writer.add(messages, user_id=user_id, agent_id=agent_id,
|
|
111
|
+
run_id=run_id, ts=ts, metadata=metadata, **kw)
|
|
112
|
+
self.reader.invalidate_caches()
|
|
113
|
+
if self.config.audit_actions == "all":
|
|
114
|
+
self.audit_log.log("memory.add", resource=user_id,
|
|
115
|
+
meta={"facts": len(out.get("results", []))})
|
|
116
|
+
return out
|
|
117
|
+
|
|
118
|
+
def _apply_pii(self, messages):
|
|
119
|
+
"""Run the PII guard over every message text. Returns redacted
|
|
120
|
+
messages, or None when the policy blocks the write."""
|
|
121
|
+
if isinstance(messages, str):
|
|
122
|
+
res = self.pii_guard.process(messages)
|
|
123
|
+
if res.blocked:
|
|
124
|
+
return None
|
|
125
|
+
return res.redacted_text
|
|
126
|
+
if isinstance(messages, list):
|
|
127
|
+
out = []
|
|
128
|
+
for m in messages:
|
|
129
|
+
if isinstance(m, dict) and "content" in m:
|
|
130
|
+
res = self.pii_guard.process(str(m["content"]))
|
|
131
|
+
if res.blocked:
|
|
132
|
+
return None
|
|
133
|
+
m = {**m, "content": res.redacted_text}
|
|
134
|
+
elif isinstance(m, str):
|
|
135
|
+
res = self.pii_guard.process(m)
|
|
136
|
+
if res.blocked:
|
|
137
|
+
return None
|
|
138
|
+
m = res.redacted_text
|
|
139
|
+
out.append(m)
|
|
140
|
+
return out
|
|
141
|
+
return messages
|
|
142
|
+
|
|
143
|
+
def search(self, query: str, *, user_id: str | None = None,
|
|
144
|
+
agent_id: str | None = None, run_id: str | None = None,
|
|
145
|
+
limit: int | None = None, timestamp=None,
|
|
146
|
+
branch: str | None = None, **kw) -> dict:
|
|
147
|
+
"""Neuro-symbolic retrieval with full provenance. Mem0-shaped output."""
|
|
148
|
+
user_id = user_id or self.config.default_user_id
|
|
149
|
+
ts = parse_ts(timestamp) if timestamp else None
|
|
150
|
+
result = self.reader.search(query, user_id=user_id, agent_id=agent_id,
|
|
151
|
+
run_id=run_id, k=limit, ts=ts,
|
|
152
|
+
branch=branch)
|
|
153
|
+
hits = result.facts and self.prefetcher.note_hits(
|
|
154
|
+
[f.id for f in result.facts])
|
|
155
|
+
return {
|
|
156
|
+
"results": result.memories(),
|
|
157
|
+
"context_block": result.context_block,
|
|
158
|
+
"relations": [{"source": f.subject, "relationship": f.relation,
|
|
159
|
+
"destination": f.value,
|
|
160
|
+
"valid_from": f.valid_from, "valid_to": f.valid_to}
|
|
161
|
+
for f in result.facts],
|
|
162
|
+
"provenance": result.provenance,
|
|
163
|
+
"intent": result.intent,
|
|
164
|
+
"timing": result.timing,
|
|
165
|
+
"llm_calls": 0,
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
def get_all(self, *, user_id: str | None = None, agent_id: str | None = None,
|
|
169
|
+
run_id: str | None = None, limit: int = 200,
|
|
170
|
+
branch: str | None = None) -> dict:
|
|
171
|
+
user_id = user_id or self.config.default_user_id
|
|
172
|
+
facts = self.store.query_facts(user_id=user_id, agent_id=agent_id,
|
|
173
|
+
run_id=run_id, branch=branch,
|
|
174
|
+
active=True, limit=limit)
|
|
175
|
+
return {"results": [{"id": f.id,
|
|
176
|
+
"memory": f"{f.subject} | {f.relation} | {f.value}",
|
|
177
|
+
"event": "ADD",
|
|
178
|
+
"valid_from": f.valid_from,
|
|
179
|
+
"valid_to": f.valid_to,
|
|
180
|
+
"confidence": f.confidence,
|
|
181
|
+
"memory_type": f.memory_type,
|
|
182
|
+
"hash": f.source_hash} for f in facts]}
|
|
183
|
+
|
|
184
|
+
def get(self, memory_id: str) -> dict | None:
|
|
185
|
+
f = self.store.get_fact(memory_id)
|
|
186
|
+
if not f:
|
|
187
|
+
return None
|
|
188
|
+
chunk = self.store.get_chunk(f.source_id) if f.source_id else None
|
|
189
|
+
return {"id": f.id, "memory": f"{f.subject} | {f.relation} | {f.value}",
|
|
190
|
+
"event": "ADD", "valid_from": f.valid_from,
|
|
191
|
+
"valid_to": f.valid_to, "confidence": f.confidence,
|
|
192
|
+
"hash": f.source_hash, "source": chunk["text"] if chunk else None,
|
|
193
|
+
"verified": bool(chunk and chunk["hash"] == f.source_hash)}
|
|
194
|
+
|
|
195
|
+
def history(self, memory_id: str) -> list[dict]:
|
|
196
|
+
"""Full bi-temporal history of a fact chain (supersessions included)."""
|
|
197
|
+
f = self.store.get_fact(memory_id)
|
|
198
|
+
if not f:
|
|
199
|
+
return []
|
|
200
|
+
chain = [f]
|
|
201
|
+
seen = {f.id}
|
|
202
|
+
frontier = [f.id]
|
|
203
|
+
while frontier:
|
|
204
|
+
nxt = []
|
|
205
|
+
for fid in frontier:
|
|
206
|
+
for e in self.store.edges_of(fid, "CONTRADICTS", "out"):
|
|
207
|
+
other = self.store.get_fact(e["dst"])
|
|
208
|
+
if other and other.id not in seen:
|
|
209
|
+
seen.add(other.id)
|
|
210
|
+
chain.append(other)
|
|
211
|
+
nxt.append(other.id)
|
|
212
|
+
for e in self.store.edges_of(fid, "CONTRADICTS", "in"):
|
|
213
|
+
other = self.store.get_fact(e["src"])
|
|
214
|
+
if other and other.id not in seen:
|
|
215
|
+
seen.add(other.id)
|
|
216
|
+
chain.append(other)
|
|
217
|
+
nxt.append(other.id)
|
|
218
|
+
frontier = nxt
|
|
219
|
+
chain.sort(key=lambda x: (x.valid_from, x.tx_from))
|
|
220
|
+
return [{"id": c.id,
|
|
221
|
+
"memory": f"{c.subject} | {c.relation} | {c.value}",
|
|
222
|
+
"event": "ADD" if c.is_active else ("QUARANTINED" if c.quarantined else "SUPERSEDED"),
|
|
223
|
+
"valid_from": c.valid_from, "valid_to": c.valid_to,
|
|
224
|
+
"recorded_at": c.tx_from} for c in chain]
|
|
225
|
+
|
|
226
|
+
def update(self, memory_id: str, data: str) -> dict:
|
|
227
|
+
"""Rewrite a fact's value (audit-logged, hash re-verified)."""
|
|
228
|
+
f = self.store.get_fact(memory_id)
|
|
229
|
+
if not f:
|
|
230
|
+
raise ContextMError(f"no fact {memory_id}")
|
|
231
|
+
old = f.value
|
|
232
|
+
self.store.update_fact(memory_id, value=data,
|
|
233
|
+
provenance={**f.provenance,
|
|
234
|
+
"manual_update": True,
|
|
235
|
+
"previous_value": old})
|
|
236
|
+
self.palace.add(memory_id, self.palace.vsa.encode_fact(
|
|
237
|
+
self.palace.embedder.embed(f.subject),
|
|
238
|
+
self.palace.embedder.embed(f.relation),
|
|
239
|
+
self.palace.embedder.embed(data)))
|
|
240
|
+
return {"id": memory_id, "event": "UPDATE",
|
|
241
|
+
"previous_value": old, "new_value": data}
|
|
242
|
+
|
|
243
|
+
def delete(self, memory_id: str) -> dict:
|
|
244
|
+
f = self.store.get_fact(memory_id)
|
|
245
|
+
if not f:
|
|
246
|
+
return {"id": memory_id, "event": "NOOP"}
|
|
247
|
+
self.store.update_fact(memory_id, is_active=0,
|
|
248
|
+
provenance={**f.provenance, "deleted": True})
|
|
249
|
+
return {"id": memory_id, "event": "DELETE"}
|
|
250
|
+
|
|
251
|
+
def delete_all(self, *, user_id: str | None = None) -> dict:
|
|
252
|
+
user_id = user_id or self.config.default_user_id
|
|
253
|
+
facts = self.store.query_facts(user_id=user_id, active=True)
|
|
254
|
+
for f in facts:
|
|
255
|
+
self.store.update_fact(f.id, is_active=0,
|
|
256
|
+
provenance={**f.provenance, "deleted": True})
|
|
257
|
+
return {"event": "DELETE_ALL", "count": len(facts)}
|
|
258
|
+
|
|
259
|
+
def users(self) -> list[str]:
|
|
260
|
+
rows = self.store.conn.execute(
|
|
261
|
+
"SELECT DISTINCT user_id FROM facts").fetchall()
|
|
262
|
+
return [r["user_id"] for r in rows]
|
|
263
|
+
|
|
264
|
+
def reset(self) -> None:
|
|
265
|
+
self.store.conn.executescript(
|
|
266
|
+
"DELETE FROM facts; DELETE FROM chunks; DELETE FROM edges; "
|
|
267
|
+
"DELETE FROM vectors; DELETE FROM commits; DELETE FROM branches; "
|
|
268
|
+
"DELETE FROM kv;")
|
|
269
|
+
self.store.conn.commit()
|
|
270
|
+
self.store._ensure_genesis()
|
|
271
|
+
self.palace = MemoryPalace(self.config, self.store)
|
|
272
|
+
self.writer = MemoryWriter(self.config, self.store, self.palace,
|
|
273
|
+
self.extractor)
|
|
274
|
+
self.reader = MemoryReader(self.config, self.store, self.palace,
|
|
275
|
+
self.prefetcher)
|
|
276
|
+
self.git = MemoryGit(self.store, self.palace)
|
|
277
|
+
self.zk = ZKProver(self.store, self.reader)
|
|
278
|
+
|
|
279
|
+
# -------------------------------------------------- Zep temporal API
|
|
280
|
+
def get_between(self, start, end, *, user_id: str | None = None,
|
|
281
|
+
field: str = "valid") -> list[dict]:
|
|
282
|
+
user_id = user_id or self.config.default_user_id
|
|
283
|
+
s = parse_ts(start)
|
|
284
|
+
e = parse_ts(end)
|
|
285
|
+
facts = self.store.temporal_window(
|
|
286
|
+
s.isoformat() if s else None, e.isoformat() if e else None,
|
|
287
|
+
user_id=user_id, field=field, active=False)
|
|
288
|
+
return [{"id": f.id, "fact": f.display(),
|
|
289
|
+
"valid_from": f.valid_from, "valid_to": f.valid_to,
|
|
290
|
+
"is_active": f.is_active} for f in facts]
|
|
291
|
+
|
|
292
|
+
def get_before(self, ts, *, user_id: str | None = None,
|
|
293
|
+
field: str = "valid") -> list[dict]:
|
|
294
|
+
return self.get_between("1970-01-01", ts, user_id=user_id, field=field)
|
|
295
|
+
|
|
296
|
+
def get_after(self, ts, *, user_id: str | None = None,
|
|
297
|
+
field: str = "valid") -> list[dict]:
|
|
298
|
+
return self.get_between(ts, "9999-12-31", user_id=user_id, field=field)
|
|
299
|
+
|
|
300
|
+
# -------------------------------------------------- provenance & trust
|
|
301
|
+
def audit(self, query: str, *, user_id: str | None = None) -> dict:
|
|
302
|
+
"""The 'Why' audit trail: query → VSA match → symbolic dereference →
|
|
303
|
+
source hash → original text, for every returned fact."""
|
|
304
|
+
res = self.search(query, user_id=user_id)
|
|
305
|
+
return {"query": query,
|
|
306
|
+
"verification": res["provenance"]["verification"],
|
|
307
|
+
"chain": res["provenance"]["chain"],
|
|
308
|
+
"llm_calls": 0}
|
|
309
|
+
|
|
310
|
+
def verify_integrity(self, sample: int | None = None) -> dict:
|
|
311
|
+
"""Recompute source hashes + vector hashes (tamper detection)."""
|
|
312
|
+
chunks = self.store.conn.execute(
|
|
313
|
+
"SELECT id, text, hash FROM chunks").fetchall()
|
|
314
|
+
bad_chunks = 0
|
|
315
|
+
for c in chunks:
|
|
316
|
+
if self.store.hasher.hash_text(c["text"]) != c["hash"]:
|
|
317
|
+
bad_chunks += 1
|
|
318
|
+
vec = self.palace.health_check(sample)
|
|
319
|
+
return {"chunks_checked": len(chunks), "corrupt_chunks": bad_chunks,
|
|
320
|
+
"vector_check": {k: v for k, v in vec.items()
|
|
321
|
+
if k != "corrupt_ids"},
|
|
322
|
+
"hash_provider": self.store.hasher.name,
|
|
323
|
+
"ok": bad_chunks == 0 and vec["corrupt"] == 0}
|
|
324
|
+
|
|
325
|
+
# -------------------------------------------------- Memory Git
|
|
326
|
+
def branch(self, name: str, from_commit: str | None = None,
|
|
327
|
+
switch: bool = True) -> str:
|
|
328
|
+
out = self.git.branch(name, from_commit, switch)
|
|
329
|
+
self.reader.invalidate_caches()
|
|
330
|
+
return out
|
|
331
|
+
|
|
332
|
+
def checkout(self, name: str) -> None:
|
|
333
|
+
self.git.checkout(name)
|
|
334
|
+
self.reader.invalidate_caches()
|
|
335
|
+
|
|
336
|
+
def merge(self, name: str, strategy: str = "latest-wins") -> dict:
|
|
337
|
+
out = self.git.merge(name, strategy)
|
|
338
|
+
self.reader.invalidate_caches()
|
|
339
|
+
return out
|
|
340
|
+
|
|
341
|
+
def diff(self, a: str, b: str) -> dict:
|
|
342
|
+
return self.git.diff(a, b)
|
|
343
|
+
|
|
344
|
+
def blame(self, subject: str, relation: str | None = None) -> list[dict]:
|
|
345
|
+
return self.git.blame(subject, relation)
|
|
346
|
+
|
|
347
|
+
def log(self, limit: int = 50) -> list[dict]:
|
|
348
|
+
return self.git.log(limit=limit)
|
|
349
|
+
|
|
350
|
+
# -------------------------------------------------- ZK-lite
|
|
351
|
+
def prove(self, query: str, *, user_id: str | None = None,
|
|
352
|
+
threshold: float = 0.2) -> dict:
|
|
353
|
+
return self.zk.prove(query, user_id=user_id or
|
|
354
|
+
self.config.default_user_id,
|
|
355
|
+
threshold=threshold)
|
|
356
|
+
|
|
357
|
+
def verify_proof(self, proof: dict) -> bool:
|
|
358
|
+
return self.zk.verify(proof)
|
|
359
|
+
|
|
360
|
+
# -------------------------------------------------- self-healing
|
|
361
|
+
def health_check(self, sample: int | None = None) -> dict:
|
|
362
|
+
return self.palace.health_check(sample)
|
|
363
|
+
|
|
364
|
+
def heal(self) -> dict:
|
|
365
|
+
facts = {f.id: f for f in self.store.query_facts(active=None)}
|
|
366
|
+
return self.palace.heal(facts)
|
|
367
|
+
|
|
368
|
+
def corrupt(self, rate: float, seed: int = 0, persist: bool = False) -> int:
|
|
369
|
+
return self.palace.corrupt(rate, seed=seed, persist=persist)
|
|
370
|
+
|
|
371
|
+
# -------------------------------------------------- lifecycle & ops
|
|
372
|
+
def apply_rules(self) -> int:
|
|
373
|
+
"""Deferred Datalog materialization after bulk ingest."""
|
|
374
|
+
self.store.begin_batch()
|
|
375
|
+
derived = self.writer.apply_rules()
|
|
376
|
+
self.reader.invalidate_caches()
|
|
377
|
+
return len(derived)
|
|
378
|
+
|
|
379
|
+
# -------------------------------------------------- async enrichment
|
|
380
|
+
def enrich(self, user_id: str | None = None, *, extractor=None,
|
|
381
|
+
limit: int | None = None, min_confidence: float | None = None,
|
|
382
|
+
dry_run: bool = False) -> dict:
|
|
383
|
+
"""Post-store LLM enrichment fallback (μ=0 stays intact on the
|
|
384
|
+
synchronous ingest path; this is the plan's graceful-degradation
|
|
385
|
+
second pass over zero-signal chunks). Returns an EnrichmentReport
|
|
386
|
+
as a dict — LLM call count and provenance markers are auditable.
|
|
387
|
+
"""
|
|
388
|
+
from cortexm.bridge.enrich import enrich as _enrich
|
|
389
|
+
rep = _enrich(self.writer, user_id, extractor=extractor,
|
|
390
|
+
limit=limit, min_confidence=min_confidence,
|
|
391
|
+
dry_run=dry_run)
|
|
392
|
+
self.reader.invalidate_caches()
|
|
393
|
+
return rep.to_dict()
|
|
394
|
+
|
|
395
|
+
def enrich_async(self, user_id: str | None = None, **kw):
|
|
396
|
+
"""Background-thread variant of enrich(). Returns (thread, holder)."""
|
|
397
|
+
from cortexm.bridge.enrich import enrich_async as _ea
|
|
398
|
+
return _ea(self.writer, user_id, **kw)
|
|
399
|
+
|
|
400
|
+
# -------------------------------------------------- scope sandbox
|
|
401
|
+
def promote(self, fact_ids: list[str], *, reviewed_by: str = "system",
|
|
402
|
+
force: bool = False) -> dict:
|
|
403
|
+
"""Promote agent-scoped facts into the user scope (InjecMEM policy).
|
|
404
|
+
|
|
405
|
+
Gated on confidence + a fresh InjecMEM/MINJA rescan of the source
|
|
406
|
+
chunk; every decision lands in the tamper-evident audit chain.
|
|
407
|
+
"""
|
|
408
|
+
from cortexm.security.sandbox import ScopeSandbox
|
|
409
|
+
sandbox = ScopeSandbox(self.config, self.store, self.audit_log)
|
|
410
|
+
out = sandbox.promote(fact_ids, reviewed_by=reviewed_by, force=force)
|
|
411
|
+
self.reader.invalidate_caches()
|
|
412
|
+
return out
|
|
413
|
+
|
|
414
|
+
def consolidate(self, now=None, **kwargs) -> dict:
|
|
415
|
+
"""Run BOTH consolidation passes:
|
|
416
|
+
|
|
417
|
+
(1) lifecycle.consolidate — Dual-Layer Agentic Memory: promote
|
|
418
|
+
reinforced short-term facts, decay untouched ones, demote
|
|
419
|
+
weak long-term facts. Fast, pure-SQL.
|
|
420
|
+
|
|
421
|
+
(2) trace.consolidate.consolidate — Aeon-inspired "dreaming":
|
|
422
|
+
merge redundant triples (MERGED_WITH edges), retire stale
|
|
423
|
+
facts past valid_to + grace, defrag palace, retrain
|
|
424
|
+
MBTB prefetcher. Slower, idempotent, safe.
|
|
425
|
+
|
|
426
|
+
(3) trace.cognition — HMS-style self-organization:
|
|
427
|
+
PatternScanner + AbstractionEngine + GapDetector +
|
|
428
|
+
HypothesisEngine + AnalogyDetector. Emits HYPOTHESIZED_BY
|
|
429
|
+
edges with confidence < 0.5 — never promoted to active
|
|
430
|
+
retrieval unless explicitly confirmed by user input.
|
|
431
|
+
|
|
432
|
+
Returns a combined report. Either pass may be skipped via
|
|
433
|
+
kwargs lifecycle=False / dreaming=False / cognition=False.
|
|
434
|
+
"""
|
|
435
|
+
out = {"lifecycle": {}, "dreaming": {}}
|
|
436
|
+
if kwargs.get("lifecycle", True):
|
|
437
|
+
out["lifecycle"] = lifecycle.consolidate(self.store, now)
|
|
438
|
+
if kwargs.get("dreaming", True):
|
|
439
|
+
from cortexm.trace.consolidate import consolidate as _dream
|
|
440
|
+
# Respect the Config's fade_enabled / tmt_enabled /
|
|
441
|
+
# cognition_enabled flags by default so `cortexm
|
|
442
|
+
# consolidate` runs the full production pass without
|
|
443
|
+
# requiring CLI flag plumbing. CLI / env can still turn
|
|
444
|
+
# them off via kwargs.
|
|
445
|
+
run_fade = kwargs.get("run_fade",
|
|
446
|
+
getattr(self.config, "fade_enabled", True))
|
|
447
|
+
run_tmt = kwargs.get("run_tmt",
|
|
448
|
+
getattr(self.config, "tmt_enabled", False))
|
|
449
|
+
run_cognition = kwargs.get(
|
|
450
|
+
"run_cognition",
|
|
451
|
+
getattr(self.config, "cognition_enabled", False))
|
|
452
|
+
out["dreaming"] = _dream(self.store, palace=self.palace,
|
|
453
|
+
prefetcher=self.prefetcher,
|
|
454
|
+
user_id=kwargs.get("user_id"),
|
|
455
|
+
dry_run=kwargs.get("dry_run", False),
|
|
456
|
+
run_fade=run_fade,
|
|
457
|
+
run_tmt=run_tmt,
|
|
458
|
+
run_cognition=run_cognition)
|
|
459
|
+
return out
|
|
460
|
+
|
|
461
|
+
def export_schema_report(self, user_id: str | None = None) -> dict:
|
|
462
|
+
return export_schema_report(self.store, user_id)
|
|
463
|
+
|
|
464
|
+
@staticmethod
|
|
465
|
+
def merge_schema_reports(reports: list[dict]) -> dict:
|
|
466
|
+
return merge_schema_reports(reports)
|
|
467
|
+
|
|
468
|
+
# -------------------------------------------------- introspection
|
|
469
|
+
def stats(self) -> dict:
|
|
470
|
+
s = self.store.stats()
|
|
471
|
+
s.update(self.palace.storage_stats())
|
|
472
|
+
s.update(self.reader.slb.stats())
|
|
473
|
+
s.update(self.prefetcher.stats())
|
|
474
|
+
s["counters"] = metrics.counters()
|
|
475
|
+
s["u0_protocol"] = "verified" if metrics.llm_calls() == 0 else "VIOLATED"
|
|
476
|
+
s["hash_provider"] = self.store.hasher.name
|
|
477
|
+
s["vsa_mode"] = self.config.vsa_mode
|
|
478
|
+
s["dims"] = self.config.dims
|
|
479
|
+
return s
|
|
480
|
+
|
|
481
|
+
def storage_stats(self) -> dict:
|
|
482
|
+
return self.palace.storage_stats()
|
|
483
|
+
|
|
484
|
+
# -------------------------------------------------- sidecar blob arena
|
|
485
|
+
def enable_blob_arena(self, path: str | os.PathLike) -> dict:
|
|
486
|
+
"""Opt-in: migrate chunks.text into a sidecar mmap-backed blob
|
|
487
|
+
file (Aeon-inspired). After migration:
|
|
488
|
+
|
|
489
|
+
- chunks.text := first 64 bytes of the source (preview)
|
|
490
|
+
- chunks.blob_offset := byte offset in the arena file
|
|
491
|
+
- chunks.blob_len := payload length
|
|
492
|
+
- chunks.blob_compressed := 0/1
|
|
493
|
+
|
|
494
|
+
Graph queries that only need the preview still work without
|
|
495
|
+
touching the arena. Full text is fetched on demand via
|
|
496
|
+
arena.get_text(offset, len, compressed).
|
|
497
|
+
|
|
498
|
+
Returns a migration report dict.
|
|
499
|
+
"""
|
|
500
|
+
from cortexm.trace.blob_arena import (
|
|
501
|
+
BlobArena, migrate_chunks_to_arena)
|
|
502
|
+
arena = BlobArena(path)
|
|
503
|
+
report = migrate_chunks_to_arena(self.store, arena)
|
|
504
|
+
# keep arena handle on self so it stays alive for the life of
|
|
505
|
+
# the Memory instance; the host can grab it via .blob_arena
|
|
506
|
+
self.blob_arena = arena
|
|
507
|
+
return report
|
|
508
|
+
|
|
509
|
+
def get_chunk_text(self, chunk_id: str) -> str:
|
|
510
|
+
"""Fetch full text for a chunk — from the arena if migrated,
|
|
511
|
+
otherwise from the inline text column."""
|
|
512
|
+
arena = getattr(self, "blob_arena", None)
|
|
513
|
+
if arena is not None:
|
|
514
|
+
from cortexm.trace.blob_arena import get_chunk_text as _g
|
|
515
|
+
return _g(self.store, arena, chunk_id)
|
|
516
|
+
row = self.store.conn.execute(
|
|
517
|
+
"SELECT text FROM chunks WHERE id=?", (chunk_id,)).fetchone()
|
|
518
|
+
return row[0] if row else ""
|
|
519
|
+
|
|
520
|
+
# ------------------------------------- engineered role vectors (NSR)
|
|
521
|
+
def use_engineered_role_vectors(self, *, n_epochs: int = 200,
|
|
522
|
+
lr: float = 0.01,
|
|
523
|
+
save_path: str | None = None,
|
|
524
|
+
verbose: bool = False) -> dict:
|
|
525
|
+
"""NSR-inspired: train a tiny autoencoder on the actual fact
|
|
526
|
+
corpus and use the top-k principal directions as role vectors.
|
|
527
|
+
|
|
528
|
+
arXiv insight: random role vectors (current default) waste
|
|
529
|
+
capacity on directions orthogonal to the data. Engineered ones
|
|
530
|
+
sit on the data's principal axes — higher effective capacity,
|
|
531
|
+
lower cross-talk, better retrieval SNR.
|
|
532
|
+
|
|
533
|
+
After this call, the palace's VSA will use the engineered role
|
|
534
|
+
vectors for bind() / unbind() / probe() operations. New facts
|
|
535
|
+
ingested after this call will be encoded with the engineered
|
|
536
|
+
vectors; previously-encoded facts keep their original (random)
|
|
537
|
+
holograms until the palace is rebuilt.
|
|
538
|
+
|
|
539
|
+
Returns the autoencoder training report.
|
|
540
|
+
|
|
541
|
+
NOTE: this is OPT-IN. The default behavior (random role
|
|
542
|
+
vectors) is unchanged unless this method is called.
|
|
543
|
+
"""
|
|
544
|
+
from cortexm.vsa.role_vectors import EngineeredRoleVectors
|
|
545
|
+
# pull the fact matrix from the palace — we need the actual
|
|
546
|
+
# S/R/V vectors used to encode the existing facts
|
|
547
|
+
facts = self.store.query_facts(active=True)
|
|
548
|
+
if not facts:
|
|
549
|
+
return {"trained": False, "reason": "no_facts_in_store"}
|
|
550
|
+
# build a (n_facts * 3, dims) matrix: S, R, V vectors for each
|
|
551
|
+
# fact, stacked. Each row is a single role-filler vector.
|
|
552
|
+
rows = []
|
|
553
|
+
for f in facts:
|
|
554
|
+
for field, role in (
|
|
555
|
+
(f.subject, "S"), (f.relation, "R"), (f.value, "V")):
|
|
556
|
+
if field:
|
|
557
|
+
rows.append(self.palace.embedder.embed(field))
|
|
558
|
+
if not rows:
|
|
559
|
+
return {"trained": False, "reason": "no_text_to_embed"}
|
|
560
|
+
matrix = np.stack(rows).astype(np.float32)
|
|
561
|
+
# cap the matrix size — 10k samples is plenty for PCA
|
|
562
|
+
if len(matrix) > 10_000:
|
|
563
|
+
import numpy as _np
|
|
564
|
+
rng = _np.random.default_rng(self.config.seed)
|
|
565
|
+
idx = rng.choice(len(matrix), 10_000, replace=False)
|
|
566
|
+
matrix = matrix[idx]
|
|
567
|
+
erv = EngineeredRoleVectors(
|
|
568
|
+
dims=self.config.dims, n_roles=3,
|
|
569
|
+
seed=self.config.seed, n_epochs=n_epochs, lr=lr)
|
|
570
|
+
report = erv.fit(matrix)
|
|
571
|
+
if erv.is_fit:
|
|
572
|
+
self.palace.vsa.use_engineered(erv)
|
|
573
|
+
self._engineered_role_vectors = erv
|
|
574
|
+
if save_path:
|
|
575
|
+
erv.save(save_path)
|
|
576
|
+
if verbose:
|
|
577
|
+
print(f"[engineered-role-vectors] {report}")
|
|
578
|
+
return report
|
|
579
|
+
|
|
580
|
+
def save_engineered_role_vectors(self, path: str) -> None:
|
|
581
|
+
erv = getattr(self, "_engineered_role_vectors", None)
|
|
582
|
+
if erv is None:
|
|
583
|
+
raise RuntimeError("no engineered role vectors to save — "
|
|
584
|
+
"call use_engineered_role_vectors() first")
|
|
585
|
+
erv.save(path)
|
|
586
|
+
|
|
587
|
+
def load_engineered_role_vectors(self, path: str) -> dict:
|
|
588
|
+
"""Load a previously-saved .npz of engineered role vectors
|
|
589
|
+
and swap them in as the active role vectors."""
|
|
590
|
+
from cortexm.vsa.role_vectors import EngineeredRoleVectors
|
|
591
|
+
erv = EngineeredRoleVectors(dims=self.config.dims,
|
|
592
|
+
seed=self.config.seed)
|
|
593
|
+
erv.load(path)
|
|
594
|
+
if erv.is_fit:
|
|
595
|
+
self.palace.vsa.use_engineered(erv)
|
|
596
|
+
self._engineered_role_vectors = erv
|
|
597
|
+
return {"loaded": True, "path": path}
|
|
598
|
+
return {"loaded": False, "reason": "file_empty_or_corrupt"}
|
|
599
|
+
|
|
600
|
+
def close(self) -> None:
|
|
601
|
+
self.palace.close()
|
|
602
|
+
self.store.close()
|
|
603
|
+
|
|
604
|
+
def _reopen(self) -> None:
|
|
605
|
+
"""Rebind every component to a freshly-opened store (post-restore)."""
|
|
606
|
+
from cortexm.security.pii import PIIGuard, PIIVault
|
|
607
|
+
from cortexm.security.rbac import APIKeyStore
|
|
608
|
+
from cortexm.enterprise.audit import AuditLog
|
|
609
|
+
from cortexm.enterprise.governance import Governance
|
|
610
|
+
self.store = TraceStore(self.config.db_path,
|
|
611
|
+
HashProvider(self.config.hash_provider),
|
|
612
|
+
wal_sync=getattr(self.config, "wal_sync", "normal"))
|
|
613
|
+
self.palace = MemoryPalace(self.config, self.store)
|
|
614
|
+
self.writer = MemoryWriter(self.config, self.store, self.palace,
|
|
615
|
+
self.extractor)
|
|
616
|
+
self.reader = MemoryReader(self.config, self.store, self.palace,
|
|
617
|
+
self.prefetcher)
|
|
618
|
+
self.git = MemoryGit(self.store, self.palace)
|
|
619
|
+
self.zk = ZKProver(self.store, self.reader)
|
|
620
|
+
self.pii_vault = PIIVault(self.store, self.cipher)
|
|
621
|
+
self.pii_guard = PIIGuard(self.config.pii_mode, self.pii_vault)
|
|
622
|
+
self.keys = APIKeyStore(self.store)
|
|
623
|
+
self.audit_log = AuditLog(self.store, enabled=self.config.audit_enabled)
|
|
624
|
+
self.governance = Governance(self)
|
|
625
|
+
# the governance object captured the OLD store/palace in __init__;
|
|
626
|
+
# rebind to the fresh ones
|
|
627
|
+
self.governance.store = self.store
|
|
628
|
+
self.governance.palace = self.palace
|
|
629
|
+
self.governance.audit = self.audit_log
|
|
630
|
+
|
|
631
|
+
def __enter__(self) -> "Memory":
|
|
632
|
+
return self
|
|
633
|
+
|
|
634
|
+
def __exit__(self, *exc) -> None:
|
|
635
|
+
self.close()
|
|
File without changes
|