cortexm 0.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- context_m.py +17 -0
- cortexm/__init__.py +45 -0
- cortexm/accel.py +403 -0
- cortexm/api/__init__.py +0 -0
- cortexm/api/chaos.py +118 -0
- cortexm/api/memory.py +635 -0
- cortexm/bench/__init__.py +0 -0
- cortexm/bench/abilities.py +311 -0
- cortexm/bench/baselines.py +89 -0
- cortexm/bench/beam_loader.py +317 -0
- cortexm/bench/generator.py +376 -0
- cortexm/bench/harness.py +211 -0
- cortexm/bench/messy.py +218 -0
- cortexm/bench/micro.py +251 -0
- cortexm/bench/ood.py +443 -0
- cortexm/bench/run.py +137 -0
- cortexm/bridge/__init__.py +0 -0
- cortexm/bridge/dates.py +178 -0
- cortexm/bridge/decoders.py +204 -0
- cortexm/bridge/enrich.py +255 -0
- cortexm/bridge/extractor.py +316 -0
- cortexm/bridge/fallback.py +332 -0
- cortexm/bridge/onnx_runtime.py +158 -0
- cortexm/bridge/patterns.py +760 -0
- cortexm/bridge/ppr.py +104 -0
- cortexm/bridge/prefilter.py +188 -0
- cortexm/bridge/query_extract.py +420 -0
- cortexm/bridge/reader.py +1174 -0
- cortexm/bridge/rerank.py +204 -0
- cortexm/bridge/writer.py +492 -0
- cortexm/cli.py +295 -0
- cortexm/cognition/__init__.py +53 -0
- cortexm/cognition/abstraction.py +192 -0
- cortexm/cognition/analogy.py +159 -0
- cortexm/cognition/engine.py +204 -0
- cortexm/cognition/gaps.py +365 -0
- cortexm/cognition/scanner.py +204 -0
- cortexm/config.py +375 -0
- cortexm/cortexm.py +8 -0
- cortexm/enterprise/__init__.py +0 -0
- cortexm/enterprise/audit.py +178 -0
- cortexm/enterprise/governance.py +239 -0
- cortexm/errors.py +35 -0
- cortexm/features/__init__.py +0 -0
- cortexm/features/git.py +204 -0
- cortexm/features/prefetch.py +88 -0
- cortexm/features/zk.py +105 -0
- cortexm/federation/__init__.py +39 -0
- cortexm/federation/crdt.py +275 -0
- cortexm/federation/fabric.py +109 -0
- cortexm/federation/hlc.py +80 -0
- cortexm/federation/node.py +145 -0
- cortexm/federation/schema_report.py +73 -0
- cortexm/federation/transport.py +164 -0
- cortexm/index/__init__.py +19 -0
- cortexm/index/nsg.py +386 -0
- cortexm/mcp/__init__.py +0 -0
- cortexm/mcp/server.py +985 -0
- cortexm/metrics.py +62 -0
- cortexm/migrate/__init__.py +0 -0
- cortexm/migrate/importers.py +192 -0
- cortexm/provenance/__init__.py +78 -0
- cortexm/provenance/agent.py +214 -0
- cortexm/provenance/cose.py +201 -0
- cortexm/provenance/scitt.py +258 -0
- cortexm/provenance/vc.py +250 -0
- cortexm/security/__init__.py +0 -0
- cortexm/security/crypto.py +162 -0
- cortexm/security/hashes.py +140 -0
- cortexm/security/injection.py +149 -0
- cortexm/security/mind.py +154 -0
- cortexm/security/pii.py +265 -0
- cortexm/security/rbac.py +169 -0
- cortexm/security/sandbox.py +131 -0
- cortexm/security/zk_hamming.py +142 -0
- cortexm/security/zk_sql.py +485 -0
- cortexm/server/__init__.py +0 -0
- cortexm/server/metrics.py +88 -0
- cortexm/server/rest.py +936 -0
- cortexm/server/sparql.py +984 -0
- cortexm/text/__init__.py +0 -0
- cortexm/text/dissim.py +252 -0
- cortexm/text/embedder.py +155 -0
- cortexm/text/fuzzy.py +218 -0
- cortexm/text/idiolect.py +253 -0
- cortexm/text/labse.py +374 -0
- cortexm/text/tokenizer.py +79 -0
- cortexm/trace/__init__.py +0 -0
- cortexm/trace/blob_arena.py +277 -0
- cortexm/trace/consolidate.py +337 -0
- cortexm/trace/contradictions.py +69 -0
- cortexm/trace/dedup.py +114 -0
- cortexm/trace/edges.py +214 -0
- cortexm/trace/fact.py +121 -0
- cortexm/trace/fade.py +245 -0
- cortexm/trace/lifecycle.py +112 -0
- cortexm/trace/rebuild.py +173 -0
- cortexm/trace/rules.py +171 -0
- cortexm/trace/store.py +680 -0
- cortexm/trace/structural.py +183 -0
- cortexm/trace/tmt.py +335 -0
- cortexm/util.py +148 -0
- cortexm/vsa/__init__.py +0 -0
- cortexm/vsa/attribution.py +149 -0
- cortexm/vsa/cleanup.py +161 -0
- cortexm/vsa/codecs.py +397 -0
- cortexm/vsa/hologram_overlay.py +139 -0
- cortexm/vsa/index.py +163 -0
- cortexm/vsa/ops.py +149 -0
- cortexm/vsa/palace.py +446 -0
- cortexm/vsa/role_vectors.py +236 -0
- cortexm/vsa/slb.py +78 -0
- cortexm/vsa/tlsh_trie.py +137 -0
- cortexm/vsa/working_memory.py +249 -0
- cortexm-0.3.0.dist-info/METADATA +482 -0
- cortexm-0.3.0.dist-info/RECORD +120 -0
- cortexm-0.3.0.dist-info/WHEEL +5 -0
- cortexm-0.3.0.dist-info/entry_points.txt +2 -0
- cortexm-0.3.0.dist-info/licenses/LICENSE +190 -0
- cortexm-0.3.0.dist-info/top_level.txt +2 -0
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
"""Federation — multi-node memory replication and schema aggregation.
|
|
2
|
+
|
|
3
|
+
Two distinct capabilities (do not conflate them):
|
|
4
|
+
|
|
5
|
+
1. CRDT REPLICATION (this package's core): conflict-free replicated
|
|
6
|
+
bi-temporal memory state. Nodes exchange HMAC-signed digest/delta
|
|
7
|
+
envelopes over any transport; union merge + HLC-stamped OR-set
|
|
8
|
+
resolution guarantees deterministic convergence with no coordinator,
|
|
9
|
+
no locks, and no lost retraction semantics.
|
|
10
|
+
|
|
11
|
+
2. SCHEMA AGGREGATION (``schema_report``): the privacy-preserving
|
|
12
|
+
Semantic Flywheel — opt-in nodes contribute relation histograms only.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
from cortexm.federation.crdt import (
|
|
16
|
+
DEFAULT_BUCKETS,
|
|
17
|
+
FederationState,
|
|
18
|
+
FactVersion,
|
|
19
|
+
fact_key,
|
|
20
|
+
)
|
|
21
|
+
from cortexm.federation.fabric import (
|
|
22
|
+
apply_to_store,
|
|
23
|
+
export_to_crdt,
|
|
24
|
+
node_from_store,
|
|
25
|
+
)
|
|
26
|
+
from cortexm.federation.hlc import HLC, parse_stamp
|
|
27
|
+
from cortexm.federation.node import FederationError, FederationNode
|
|
28
|
+
from cortexm.federation.schema_report import (
|
|
29
|
+
export_schema_report,
|
|
30
|
+
merge_schema_reports,
|
|
31
|
+
)
|
|
32
|
+
from cortexm.federation.transport import FileTransport, InMemoryMesh
|
|
33
|
+
|
|
34
|
+
__all__ = [
|
|
35
|
+
"DEFAULT_BUCKETS", "FederationState", "FactVersion", "fact_key",
|
|
36
|
+
"HLC", "parse_stamp", "FederationNode", "FederationError",
|
|
37
|
+
"InMemoryMesh", "FileTransport", "export_to_crdt", "apply_to_store",
|
|
38
|
+
"node_from_store", "export_schema_report", "merge_schema_reports",
|
|
39
|
+
]
|
|
@@ -0,0 +1,275 @@
|
|
|
1
|
+
"""Bi-temporal fact CRDT — conflict-free replicated memory state.
|
|
2
|
+
|
|
3
|
+
DATA MODEL
|
|
4
|
+
----------
|
|
5
|
+
The symbolic layer is Subject-Relation-Value triples with bi-temporal
|
|
6
|
+
timestamps (valid time + transaction time). Federation replicates it as a
|
|
7
|
+
keyed multi-version register:
|
|
8
|
+
|
|
9
|
+
* SINGLE_VALUED relations (works_at, lives_in, ...) key on
|
|
10
|
+
``(user_id, subject, relation)`` — every historical value is a VERSION in
|
|
11
|
+
the register; the bi-temporal history is the version set itself.
|
|
12
|
+
* MULTI_VALUED relations (prefers, uses, ...) key on
|
|
13
|
+
``(user_id, subject, relation, value)`` — each distinct value is its own
|
|
14
|
+
register (an OR-set member).
|
|
15
|
+
|
|
16
|
+
MERGE SEMANTICS (per key)
|
|
17
|
+
-------------------------
|
|
18
|
+
Versions are IMMUTABLE and uniquely stamped by an HLC. Merging two
|
|
19
|
+
replicas is the per-stamp UNION of version sets — commutative,
|
|
20
|
+
associative, idempotent. Resolution for reads is deterministic:
|
|
21
|
+
|
|
22
|
+
* the winning (live) version is the highest-stamped non-tombstone version
|
|
23
|
+
whose stamp is greater than the highest tombstone stamp for that key
|
|
24
|
+
(write-after-retract wins, retract-after-write wins — classic OR-set
|
|
25
|
+
semantics under a causally-consistent total order);
|
|
26
|
+
* a tombstone causally after a write retracts it;
|
|
27
|
+
* HLC total order means every replica picks the same winner without
|
|
28
|
+
coordination.
|
|
29
|
+
|
|
30
|
+
GDPR / PURGE
|
|
31
|
+
------------
|
|
32
|
+
A tombstone with ``purge=True`` is a poison pill: it retracts the key and
|
|
33
|
+
authorises payload GC after the retention window (see ``gc_tombstones``).
|
|
34
|
+
Payloads are dropped; stamps remain until every replica has seen the
|
|
35
|
+
tombstone (digest convergence), so late deltas cannot resurrect purged
|
|
36
|
+
data.
|
|
37
|
+
"""
|
|
38
|
+
|
|
39
|
+
from __future__ import annotations
|
|
40
|
+
|
|
41
|
+
import hashlib
|
|
42
|
+
import json
|
|
43
|
+
from dataclasses import dataclass, field
|
|
44
|
+
|
|
45
|
+
from cortexm.federation.hlc import HLC, parse_stamp
|
|
46
|
+
|
|
47
|
+
DEFAULT_BUCKETS = 64
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def fact_key(user_id: str, subject: str, relation: str,
|
|
51
|
+
value: str | None = None, single_valued: bool = True) -> str:
|
|
52
|
+
"""Stable CRDT key. SINGLE_VALUED keys omit the value so successive
|
|
53
|
+
values become versions of one register; MULTI_VALUED keys include it."""
|
|
54
|
+
parts = [user_id or "default", subject.lower(), relation.lower()]
|
|
55
|
+
if not single_valued and value is not None:
|
|
56
|
+
parts.append(value.lower())
|
|
57
|
+
return "\x1f".join(parts)
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
@dataclass
|
|
61
|
+
class FactVersion:
|
|
62
|
+
stamp: str # HLC string — globally unique version id
|
|
63
|
+
payload: dict # serialisable fact fields
|
|
64
|
+
tombstone: bool = False
|
|
65
|
+
purge: bool = False # poison-pill tombstone (GDPR)
|
|
66
|
+
|
|
67
|
+
def canonical(self) -> str:
|
|
68
|
+
return json.dumps({"s": self.stamp, "p": self.payload,
|
|
69
|
+
"t": int(self.tombstone), "g": int(self.purge)},
|
|
70
|
+
sort_keys=True, separators=(",", ":"))
|
|
71
|
+
|
|
72
|
+
def digest(self) -> str:
|
|
73
|
+
return hashlib.sha256(self.canonical().encode()).hexdigest()[:16]
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
@dataclass
|
|
77
|
+
class BucketStat:
|
|
78
|
+
n: int = 0
|
|
79
|
+
max_stamp: str = ""
|
|
80
|
+
xor: int = 0
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
@dataclass
|
|
84
|
+
class FederationState:
|
|
85
|
+
"""The CRDT: key -> stamp -> version. Pure data; all operations are
|
|
86
|
+
functions so merge logic is trivially testable."""
|
|
87
|
+
node_id: str
|
|
88
|
+
buckets: int = DEFAULT_BUCKETS
|
|
89
|
+
_versions: dict[str, dict[str, FactVersion]] = field(default_factory=dict)
|
|
90
|
+
|
|
91
|
+
# ---------------------------------------------------------------- write
|
|
92
|
+
def put(self, key: str, payload: dict, clock: HLC,
|
|
93
|
+
tombstone: bool = False, purge: bool = False,
|
|
94
|
+
stamp: str | None = None) -> FactVersion:
|
|
95
|
+
st = stamp or clock.tick()
|
|
96
|
+
v = FactVersion(stamp=st, payload=payload, tombstone=tombstone,
|
|
97
|
+
purge=purge)
|
|
98
|
+
self._versions.setdefault(key, {})[st] = v
|
|
99
|
+
return v
|
|
100
|
+
|
|
101
|
+
# ---------------------------------------------------------------- merge
|
|
102
|
+
def merge_versions(self, key: str, versions: dict[str, dict]) -> int:
|
|
103
|
+
"""Union-merge remote versions into a key. Returns #applied."""
|
|
104
|
+
slot = self._versions.setdefault(key, {})
|
|
105
|
+
applied = 0
|
|
106
|
+
for stamp, raw in versions.items():
|
|
107
|
+
if stamp not in slot:
|
|
108
|
+
slot[stamp] = FactVersion(
|
|
109
|
+
stamp=stamp,
|
|
110
|
+
payload=raw.get("payload", {}),
|
|
111
|
+
tombstone=bool(raw.get("tombstone", False)),
|
|
112
|
+
purge=bool(raw.get("purge", False)))
|
|
113
|
+
applied += 1
|
|
114
|
+
return applied
|
|
115
|
+
|
|
116
|
+
def apply_delta(self, delta: dict) -> int:
|
|
117
|
+
"""Idempotent: re-applying the same delta changes nothing."""
|
|
118
|
+
applied = 0
|
|
119
|
+
for key, versions in delta.get("versions", {}).items():
|
|
120
|
+
applied += self.merge_versions(key, versions)
|
|
121
|
+
return applied
|
|
122
|
+
|
|
123
|
+
# ---------------------------------------------------------------- read
|
|
124
|
+
def keys(self) -> list[str]:
|
|
125
|
+
return list(self._versions)
|
|
126
|
+
|
|
127
|
+
def all_versions(self, key: str) -> list[FactVersion]:
|
|
128
|
+
return sorted(self._versions.get(key, {}).values(),
|
|
129
|
+
key=lambda v: v.stamp)
|
|
130
|
+
|
|
131
|
+
def live_versions(self, key: str) -> list[FactVersion]:
|
|
132
|
+
"""OR-set resolution: drop versions causally covered by a tombstone.
|
|
133
|
+
|
|
134
|
+
A tombstone covers every version with a strictly smaller stamp. A
|
|
135
|
+
purge tombstone additionally covers versions with LARGER stamps
|
|
136
|
+
(poison pill — nothing may resurrect a purged key except a write
|
|
137
|
+
that causally follows the purge, i.e. stamps issued after the
|
|
138
|
+
purge was *received* are still allowed via the purge's own stamp
|
|
139
|
+
being smaller... we keep it strict: purge kills everything with a
|
|
140
|
+
different stamp than itself).
|
|
141
|
+
"""
|
|
142
|
+
vs = self._versions.get(key, {})
|
|
143
|
+
if not vs:
|
|
144
|
+
return []
|
|
145
|
+
tomb_stamps = [v.stamp for v in vs.values() if v.tombstone]
|
|
146
|
+
purge_stamps = [v.stamp for v in vs.values() if v.purge]
|
|
147
|
+
max_tomb = max(tomb_stamps) if tomb_stamps else ""
|
|
148
|
+
out = []
|
|
149
|
+
for v in vs.values():
|
|
150
|
+
if v.tombstone:
|
|
151
|
+
continue
|
|
152
|
+
if v.stamp < max_tomb:
|
|
153
|
+
continue # retracted
|
|
154
|
+
if purge_stamps and v.stamp not in purge_stamps:
|
|
155
|
+
continue # purged
|
|
156
|
+
out.append(v)
|
|
157
|
+
return sorted(out, key=lambda v: v.stamp)
|
|
158
|
+
|
|
159
|
+
def winner(self, key: str) -> FactVersion | None:
|
|
160
|
+
live = self.live_versions(key)
|
|
161
|
+
return live[-1] if live else None
|
|
162
|
+
|
|
163
|
+
def value_history(self, key: str) -> list[FactVersion]:
|
|
164
|
+
"""All live versions ordered by valid_from — the bi-temporal
|
|
165
|
+
timeline for a SINGLE_VALUED key."""
|
|
166
|
+
live = self.live_versions(key)
|
|
167
|
+
|
|
168
|
+
def vf(v):
|
|
169
|
+
return v.payload.get("valid_from") or ""
|
|
170
|
+
|
|
171
|
+
return sorted(live, key=vf)
|
|
172
|
+
|
|
173
|
+
# ------------------------------------------------------------- digests
|
|
174
|
+
def bucket_of(self, key: str) -> int:
|
|
175
|
+
h = int(hashlib.blake2b(key.encode(), digest_size=8).hexdigest(), 16)
|
|
176
|
+
return h % self.buckets
|
|
177
|
+
|
|
178
|
+
def digest(self) -> dict:
|
|
179
|
+
"""Per-bucket (count, max_stamp, xor-fold) vector. Two replicas
|
|
180
|
+
with identical digests are (hash-collision aside) identical."""
|
|
181
|
+
stats: dict[int, BucketStat] = {}
|
|
182
|
+
for key, versions in self._versions.items():
|
|
183
|
+
b = self.bucket_of(key)
|
|
184
|
+
st = stats.setdefault(b, BucketStat())
|
|
185
|
+
for v in versions.values():
|
|
186
|
+
st.n += 1
|
|
187
|
+
if v.stamp > st.max_stamp:
|
|
188
|
+
st.max_stamp = v.stamp
|
|
189
|
+
st.xor ^= int(v.digest(), 16)
|
|
190
|
+
return {str(b): [st.n, st.max_stamp, f"{st.xor:016x}"]
|
|
191
|
+
for b, st in sorted(stats.items())}
|
|
192
|
+
|
|
193
|
+
def delta_for(self, their_digest: dict) -> dict:
|
|
194
|
+
"""Ship every version of every bucket where the digests differ."""
|
|
195
|
+
mine = self.digest()
|
|
196
|
+
diverged = []
|
|
197
|
+
for b, stat in mine.items():
|
|
198
|
+
if their_digest.get(b) != stat:
|
|
199
|
+
diverged.append(int(b))
|
|
200
|
+
# buckets they have that I don't (still ship mine — union merge)
|
|
201
|
+
for b in their_digest:
|
|
202
|
+
if b not in mine:
|
|
203
|
+
diverged.append(int(b))
|
|
204
|
+
versions: dict[str, dict[str, dict]] = {}
|
|
205
|
+
for key, vs in self._versions.items():
|
|
206
|
+
if self.bucket_of(key) in diverged:
|
|
207
|
+
versions[key] = {st: {"payload": v.payload,
|
|
208
|
+
"tombstone": v.tombstone,
|
|
209
|
+
"purge": v.purge}
|
|
210
|
+
for st, v in vs.items()}
|
|
211
|
+
return {"buckets": sorted(set(diverged)), "versions": versions}
|
|
212
|
+
|
|
213
|
+
# ------------------------------------------------------------- gc / size
|
|
214
|
+
def gc_tombstones(self, older_than_stamp: str) -> dict:
|
|
215
|
+
"""Drop tombstoned payloads past the retention watermark.
|
|
216
|
+
|
|
217
|
+
Safe only once every replica has seen the tombstone (digest
|
|
218
|
+
convergence); the caller enforces that. Stamps of purged keys are
|
|
219
|
+
retained as empty tombstones so late deltas merge to the same
|
|
220
|
+
(retracted) resolution instead of resurrecting data.
|
|
221
|
+
"""
|
|
222
|
+
removed = {"versions": 0, "payloads": 0}
|
|
223
|
+
for key, vs in list(self._versions.items()):
|
|
224
|
+
for st, v in list(vs.items()):
|
|
225
|
+
if v.tombstone and st < older_than_stamp:
|
|
226
|
+
if v.payload:
|
|
227
|
+
v.payload = {}
|
|
228
|
+
removed["payloads"] += 1
|
|
229
|
+
removed["versions"] += 1
|
|
230
|
+
return removed
|
|
231
|
+
|
|
232
|
+
def stats(self) -> dict:
|
|
233
|
+
n_versions = sum(len(v) for v in self._versions.values())
|
|
234
|
+
n_keys = len(self._versions)
|
|
235
|
+
n_tomb = sum(1 for vs in self._versions.values()
|
|
236
|
+
for v in vs.values() if v.tombstone)
|
|
237
|
+
return {"keys": n_keys, "versions": n_versions,
|
|
238
|
+
"tombstones": n_tomb}
|
|
239
|
+
|
|
240
|
+
# ---------------------------------------------------------- canonicalise
|
|
241
|
+
def canonical_bytes(self) -> bytes:
|
|
242
|
+
"""Byte-exact canonical form — two converged replicas produce
|
|
243
|
+
IDENTICAL bytes. This is the convergence oracle used by tests."""
|
|
244
|
+
out = {}
|
|
245
|
+
for key in sorted(self._versions):
|
|
246
|
+
vs = self._versions[key]
|
|
247
|
+
out[key] = {st: vs[st].canonical()
|
|
248
|
+
for st in sorted(vs)}
|
|
249
|
+
return json.dumps(out, sort_keys=True,
|
|
250
|
+
separators=(",", ":")).encode()
|
|
251
|
+
|
|
252
|
+
# ------------------------------------------------------------ snapshot
|
|
253
|
+
def snapshot(self) -> dict:
|
|
254
|
+
return {"node_id": self.node_id, "buckets": self.buckets,
|
|
255
|
+
"versions": {k: {st: {"payload": v.payload,
|
|
256
|
+
"tombstone": v.tombstone,
|
|
257
|
+
"purge": v.purge}
|
|
258
|
+
for st, v in vs.items()}
|
|
259
|
+
for k, vs in self._versions.items()}}
|
|
260
|
+
|
|
261
|
+
@classmethod
|
|
262
|
+
def from_snapshot(cls, d: dict) -> "FederationState":
|
|
263
|
+
st = cls(node_id=d["node_id"], buckets=d.get("buckets",
|
|
264
|
+
DEFAULT_BUCKETS))
|
|
265
|
+
for key, vs in d.get("versions", {}).items():
|
|
266
|
+
st._versions[key] = {stamp: FactVersion(
|
|
267
|
+
stamp=stamp, payload=raw.get("payload", {}),
|
|
268
|
+
tombstone=bool(raw.get("tombstone")),
|
|
269
|
+
purge=bool(raw.get("purge"))) for stamp, raw in vs.items()}
|
|
270
|
+
return st
|
|
271
|
+
|
|
272
|
+
|
|
273
|
+
def stamp_sort_key(stamp: str):
|
|
274
|
+
wall, count, node = parse_stamp(stamp)
|
|
275
|
+
return (wall, count, node)
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
"""Bridge the local TraceStore (symbolic layer) into the federated CRDT.
|
|
2
|
+
|
|
3
|
+
Direction 1 — ``export_to_crdt``: every stored fact (active AND historical)
|
|
4
|
+
becomes an immutable CRDT version under its key. SINGLE_VALUED relations
|
|
5
|
+
collapse into one versioned register per (user, subject, relation);
|
|
6
|
+
MULTI_VALUED facts get one register per distinct value.
|
|
7
|
+
|
|
8
|
+
Direction 2 — ``apply_to_store``: merge a node's converged state back into
|
|
9
|
+
a fabric: unknown fact ids are inserted, known ids are updated field-wise
|
|
10
|
+
(stamps guarantee newest-wins deterministically). The fabric's own truth
|
|
11
|
+
maintenance (SUPERSEDE edges) is NOT replayed — the CRDT versions already
|
|
12
|
+
carry the bi-temporal windows, so the store receives resolved history.
|
|
13
|
+
|
|
14
|
+
Caveat (honest scope): retrieval-side caches (VSA palace projections) must
|
|
15
|
+
be rebuilt after applying remote deltas — ``apply_to_store`` returns the
|
|
16
|
+
number of store writes so callers can decide when to reproject.
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
from __future__ import annotations
|
|
20
|
+
|
|
21
|
+
from cortexm.federation.crdt import (
|
|
22
|
+
FederationState,
|
|
23
|
+
fact_key,
|
|
24
|
+
)
|
|
25
|
+
from cortexm.federation.hlc import HLC
|
|
26
|
+
from cortexm.federation.node import FederationNode
|
|
27
|
+
from cortexm.trace.fact import Fact, SINGLE_VALUED
|
|
28
|
+
|
|
29
|
+
_FED_FIELDS = (
|
|
30
|
+
"subject", "relation", "value", "valid_from", "valid_to",
|
|
31
|
+
"confidence", "user_id", "agent_id", "run_id", "memory_type",
|
|
32
|
+
"source_hash", "source_id", "provenance",
|
|
33
|
+
)
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def _fact_payload(f: Fact) -> dict:
|
|
37
|
+
return {k: getattr(f, k) for k in _FED_FIELDS}
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def export_to_crdt(store, node: FederationNode,
|
|
41
|
+
user_id: str | None = None) -> int:
|
|
42
|
+
"""Push the store's facts into the node's CRDT state."""
|
|
43
|
+
facts = store.query_facts(user_id=user_id) # active + historical
|
|
44
|
+
n = 0
|
|
45
|
+
for f in facts:
|
|
46
|
+
single = f.relation in SINGLE_VALUED
|
|
47
|
+
key = fact_key(f.user_id, f.subject, f.relation,
|
|
48
|
+
value=f.value, single_valued=single)
|
|
49
|
+
payload = _fact_payload(f)
|
|
50
|
+
payload["fact_id"] = f.id
|
|
51
|
+
if not f.is_active:
|
|
52
|
+
# inactive fact -> tombstone version carrying its own payload
|
|
53
|
+
# so history queries can still see WHAT was retracted
|
|
54
|
+
node.state.put(key, payload, node.clock, tombstone=True)
|
|
55
|
+
else:
|
|
56
|
+
node.state.put(key, payload, node.clock)
|
|
57
|
+
n += 1
|
|
58
|
+
return n
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def apply_to_store(node: FederationNode, store) -> dict:
|
|
62
|
+
"""Pull the node's converged CRDT state into a store."""
|
|
63
|
+
inserted = updated = retracted = 0
|
|
64
|
+
for key in node.state.keys():
|
|
65
|
+
live = node.state.value_history(key)
|
|
66
|
+
for v in live:
|
|
67
|
+
payload = v.payload
|
|
68
|
+
fact_id = payload.get("fact_id")
|
|
69
|
+
required = ("subject", "relation", "value", "valid_from")
|
|
70
|
+
if not fact_id or any(k not in payload for k in required):
|
|
71
|
+
continue # incomplete version — not fabric-shaped
|
|
72
|
+
existing = store.get_fact(fact_id)
|
|
73
|
+
if existing is None:
|
|
74
|
+
f = Fact(id=fact_id, **{k: payload[k] for k in _FED_FIELDS
|
|
75
|
+
if k in payload})
|
|
76
|
+
store.insert_fact(f)
|
|
77
|
+
inserted += 1
|
|
78
|
+
else:
|
|
79
|
+
dirty = {k: payload[k] for k in _FED_FIELDS
|
|
80
|
+
if k in payload and getattr(existing, k) !=
|
|
81
|
+
payload[k]}
|
|
82
|
+
if dirty:
|
|
83
|
+
store.update_fact(fact_id, **dirty)
|
|
84
|
+
updated += 1
|
|
85
|
+
# retracted keys: mirror tombstones into the store
|
|
86
|
+
all_v = node.state.all_versions(key)
|
|
87
|
+
tombstoned_ids = {v.payload.get("fact_id") for v in all_v
|
|
88
|
+
if v.tombstone and v.payload}
|
|
89
|
+
for fid in tombstoned_ids:
|
|
90
|
+
if fid and store.get_fact(fid) is not None:
|
|
91
|
+
f = store.get_fact(fid)
|
|
92
|
+
if f.is_active:
|
|
93
|
+
store.update_fact(fid, is_active=False)
|
|
94
|
+
retracted += 1
|
|
95
|
+
return {"inserted": inserted, "updated": updated,
|
|
96
|
+
"retracted": retracted}
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def node_from_store(node_id: str, store, members: list[str] | None = None,
|
|
100
|
+
federation_key: str = "default-fed-key",
|
|
101
|
+
user_id: str | None = None) -> FederationNode:
|
|
102
|
+
node = FederationNode(node_id, members=members,
|
|
103
|
+
federation_key=federation_key)
|
|
104
|
+
export_to_crdt(store, node, user_id=user_id)
|
|
105
|
+
return node
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
__all__ = ["export_to_crdt", "apply_to_store", "node_from_store",
|
|
109
|
+
"FederationState", "FederationNode", "HLC", "fact_key"]
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
"""Hybrid Logical Clock — causality-consistent total order for federation.
|
|
2
|
+
|
|
3
|
+
Wall-clock milliseconds give rough ordering; a counter breaks same-ms ties
|
|
4
|
+
and absorbs clock skew via the receive rule (max of local and remote);
|
|
5
|
+
node_id makes the order total and deterministic on every replica.
|
|
6
|
+
|
|
7
|
+
String form ``wall.counter.node`` sorts lexicographically EXACTLY like the
|
|
8
|
+
tuple (wall is zero-padded), so CRDT stamps can be compared as plain
|
|
9
|
+
strings everywhere downstream.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import threading
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def _pad(wall_ms: int) -> str:
|
|
18
|
+
return f"{wall_ms:019d}"
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class HLC:
|
|
22
|
+
"""One clock per node. Thread-safe."""
|
|
23
|
+
|
|
24
|
+
__slots__ = ("node_id", "_wall", "_count", "_lock")
|
|
25
|
+
|
|
26
|
+
def __init__(self, node_id: str, wall_ms: int = 0, count: int = 0) -> None:
|
|
27
|
+
self.node_id = node_id
|
|
28
|
+
self._wall = wall_ms
|
|
29
|
+
self._count = count
|
|
30
|
+
self._lock = threading.Lock()
|
|
31
|
+
|
|
32
|
+
# -- local event ------------------------------------------------------
|
|
33
|
+
def tick(self, now_ms: int | None = None) -> str:
|
|
34
|
+
import time
|
|
35
|
+
now = int(time.time() * 1000) if now_ms is None else now_ms
|
|
36
|
+
with self._lock:
|
|
37
|
+
if now > self._wall:
|
|
38
|
+
self._wall, self._count = now, 0
|
|
39
|
+
else:
|
|
40
|
+
self._count += 1
|
|
41
|
+
return self._stamp()
|
|
42
|
+
|
|
43
|
+
# -- remote event -----------------------------------------------------
|
|
44
|
+
def receive(self, remote_stamp: str) -> str:
|
|
45
|
+
wall, count, _node = parse_stamp(remote_stamp)
|
|
46
|
+
import time
|
|
47
|
+
now = int(time.time() * 1000)
|
|
48
|
+
with self._lock:
|
|
49
|
+
if now > self._wall and now > wall:
|
|
50
|
+
self._wall, self._count = now, 0
|
|
51
|
+
elif wall > self._wall:
|
|
52
|
+
self._wall, self._count = wall, count + 1
|
|
53
|
+
else:
|
|
54
|
+
self._count = max(self._count, count) + 1
|
|
55
|
+
return self._stamp()
|
|
56
|
+
|
|
57
|
+
def now(self) -> str:
|
|
58
|
+
return self.tick()
|
|
59
|
+
|
|
60
|
+
def peek(self) -> str:
|
|
61
|
+
with self._lock:
|
|
62
|
+
return self._stamp()
|
|
63
|
+
|
|
64
|
+
def _stamp(self) -> str:
|
|
65
|
+
return f"{_pad(self._wall)}.{self._count:06d}.{self.node_id}"
|
|
66
|
+
|
|
67
|
+
# -- persistence ------------------------------------------------------
|
|
68
|
+
def state(self) -> dict:
|
|
69
|
+
with self._lock:
|
|
70
|
+
return {"node_id": self.node_id, "wall": self._wall,
|
|
71
|
+
"count": self._count}
|
|
72
|
+
|
|
73
|
+
@classmethod
|
|
74
|
+
def from_state(cls, d: dict) -> "HLC":
|
|
75
|
+
return cls(d["node_id"], d["wall"], d["count"])
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def parse_stamp(stamp: str) -> tuple[int, int, str]:
|
|
79
|
+
wall_s, count_s, node = stamp.split(".", 2)
|
|
80
|
+
return int(wall_s), int(count_s), node
|
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
"""FederationNode — identity, delta signing/verification, anti-entropy sync.
|
|
2
|
+
|
|
3
|
+
Threat model (documented honestly):
|
|
4
|
+
* members share a symmetric federation key (HMAC-SHA256); every delta is
|
|
5
|
+
signed and the sender must be a known member id;
|
|
6
|
+
* signatures authenticate origin + integrity, NOT freshness — replaying an
|
|
7
|
+
old delta is harmless (merge is idempotent) but replay of a *valid*
|
|
8
|
+
signed delta is possible; monotonic HLC stamps bound how far a replayed
|
|
9
|
+
delta can move state;
|
|
10
|
+
* this is transport-agnostic: deltas are JSON envelopes handed to whatever
|
|
11
|
+
transport is configured (in-memory mesh for tests, files for offline
|
|
12
|
+
federation, HTTP/TCP in production).
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
import hashlib
|
|
18
|
+
import hmac
|
|
19
|
+
import json
|
|
20
|
+
|
|
21
|
+
from cortexm.federation.crdt import FederationState
|
|
22
|
+
from cortexm.federation.hlc import HLC
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def _canonical_env(env: dict) -> bytes:
|
|
26
|
+
return json.dumps(env, sort_keys=True, separators=(",", ":")).encode()
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def sign_envelope(env: dict, key: str) -> str:
|
|
30
|
+
return hmac.new(key.encode(), _canonical_env(env),
|
|
31
|
+
hashlib.sha256).hexdigest()
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
class FederationError(Exception):
|
|
35
|
+
pass
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
class FederationNode:
|
|
39
|
+
"""One replica of the federated memory state."""
|
|
40
|
+
|
|
41
|
+
def __init__(self, node_id: str, members: list[str] | None = None,
|
|
42
|
+
federation_key: str = "default-fed-key",
|
|
43
|
+
buckets: int = 64,
|
|
44
|
+
state: FederationState | None = None) -> None:
|
|
45
|
+
self.node_id = node_id
|
|
46
|
+
self.members = set(members or [node_id])
|
|
47
|
+
self.members.add(node_id)
|
|
48
|
+
self.federation_key = federation_key
|
|
49
|
+
self.clock = HLC(node_id)
|
|
50
|
+
self.state = state or FederationState(node_id, buckets)
|
|
51
|
+
self.bytes_sent = 0
|
|
52
|
+
self.bytes_received = 0
|
|
53
|
+
self.syncs = 0
|
|
54
|
+
|
|
55
|
+
# ------------------------------------------------------------ writes
|
|
56
|
+
def put_fact(self, key: str, payload: dict) -> str:
|
|
57
|
+
return self.state.put(key, payload, self.clock)
|
|
58
|
+
|
|
59
|
+
def retract(self, key: str, purge: bool = False) -> str:
|
|
60
|
+
return self.state.put(key, {}, self.clock,
|
|
61
|
+
tombstone=True, purge=purge)
|
|
62
|
+
|
|
63
|
+
# ------------------------------------------------------------ digests
|
|
64
|
+
def digest_envelope(self) -> dict:
|
|
65
|
+
env = {"type": "digest", "from": self.node_id,
|
|
66
|
+
"clock": self.clock.now(),
|
|
67
|
+
"digest": self.state.digest()}
|
|
68
|
+
env["sig"] = sign_envelope({k: v for k, v in env.items()},
|
|
69
|
+
self.federation_key)
|
|
70
|
+
return env
|
|
71
|
+
|
|
72
|
+
# ------------------------------------------------------------ deltas
|
|
73
|
+
def delta_envelope_for(self, their_digest_env: dict) -> dict:
|
|
74
|
+
self._verify(their_digest_env, expected_type="digest")
|
|
75
|
+
# absorb sender's clock — keeps our stamps causally after theirs
|
|
76
|
+
self.clock.receive(their_digest_env["clock"])
|
|
77
|
+
their_digest = their_digest_env["digest"]
|
|
78
|
+
delta = self.state.delta_for(their_digest)
|
|
79
|
+
env = {"type": "delta", "from": self.node_id,
|
|
80
|
+
"to": their_digest_env["from"],
|
|
81
|
+
"clock": self.clock.now(),
|
|
82
|
+
"delta": delta}
|
|
83
|
+
env["sig"] = sign_envelope({k: v for k, v in env.items()},
|
|
84
|
+
self.federation_key)
|
|
85
|
+
return env
|
|
86
|
+
|
|
87
|
+
def apply_delta_envelope(self, env: dict) -> int:
|
|
88
|
+
self._verify(env, expected_type="delta")
|
|
89
|
+
if env.get("to") not in (None, self.node_id):
|
|
90
|
+
raise FederationError(f"delta addressed to {env.get('to')}")
|
|
91
|
+
self.clock.receive(env["clock"])
|
|
92
|
+
n = self.state.apply_delta(env["delta"])
|
|
93
|
+
return n
|
|
94
|
+
|
|
95
|
+
def _verify(self, env: dict, expected_type: str) -> None:
|
|
96
|
+
if env.get("type") != expected_type:
|
|
97
|
+
raise FederationError(f"expected {expected_type}, got "
|
|
98
|
+
f"{env.get('type')}")
|
|
99
|
+
sender = env.get("from")
|
|
100
|
+
if sender not in self.members:
|
|
101
|
+
raise FederationError(f"unknown sender: {sender}")
|
|
102
|
+
sig = env.pop("sig", None)
|
|
103
|
+
if sig is None:
|
|
104
|
+
raise FederationError("unsigned envelope rejected")
|
|
105
|
+
expect = sign_envelope(env, self.federation_key)
|
|
106
|
+
if not hmac.compare_digest(sig, expect):
|
|
107
|
+
raise FederationError(f"bad signature from {sender}")
|
|
108
|
+
env["sig"] = sig # restore
|
|
109
|
+
|
|
110
|
+
# ------------------------------------------------------------ sync
|
|
111
|
+
def sync_with(self, other: "FederationNode") -> dict:
|
|
112
|
+
"""Two-way anti-entropy: digest exchange then both-direction deltas.
|
|
113
|
+
|
|
114
|
+
Order (A=self, B=other):
|
|
115
|
+
A --digest--> B ; B --delta(for A)--> A ; A applies
|
|
116
|
+
B --digest--> A ; A --delta(for B)--> B ; B applies
|
|
117
|
+
After both half-rounds the states converge (union merge + OR-set
|
|
118
|
+
resolution is commutative). Returns a small accounting dict.
|
|
119
|
+
"""
|
|
120
|
+
a_bytes = b_bytes = 0
|
|
121
|
+
|
|
122
|
+
d1 = self.digest_envelope()
|
|
123
|
+
delta1 = other.delta_envelope_for(d1)
|
|
124
|
+
a_bytes += len(json.dumps(d1))
|
|
125
|
+
b_bytes += len(json.dumps(delta1))
|
|
126
|
+
self.bytes_received += len(json.dumps(delta1))
|
|
127
|
+
other.bytes_sent += len(json.dumps(delta1))
|
|
128
|
+
applied_self = self.apply_delta_envelope(delta1)
|
|
129
|
+
|
|
130
|
+
d2 = other.digest_envelope()
|
|
131
|
+
delta2 = self.delta_envelope_for(d2)
|
|
132
|
+
b_bytes += len(json.dumps(d2))
|
|
133
|
+
a_bytes += len(json.dumps(delta2))
|
|
134
|
+
other.bytes_received += len(json.dumps(delta2))
|
|
135
|
+
self.bytes_sent += len(json.dumps(delta2))
|
|
136
|
+
applied_other = other.apply_delta_envelope(delta2)
|
|
137
|
+
|
|
138
|
+
self.syncs += 1
|
|
139
|
+
other.syncs += 1
|
|
140
|
+
return {"a_sent_bytes": a_bytes, "b_sent_bytes": b_bytes,
|
|
141
|
+
"a_applied": applied_self, "b_applied": applied_other}
|
|
142
|
+
|
|
143
|
+
def converged_with(self, other: "FederationNode") -> bool:
|
|
144
|
+
return (self.state.canonical_bytes() ==
|
|
145
|
+
other.state.canonical_bytes())
|