cortexm 0.3.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (120) hide show
  1. context_m.py +17 -0
  2. cortexm/__init__.py +45 -0
  3. cortexm/accel.py +403 -0
  4. cortexm/api/__init__.py +0 -0
  5. cortexm/api/chaos.py +118 -0
  6. cortexm/api/memory.py +635 -0
  7. cortexm/bench/__init__.py +0 -0
  8. cortexm/bench/abilities.py +311 -0
  9. cortexm/bench/baselines.py +89 -0
  10. cortexm/bench/beam_loader.py +317 -0
  11. cortexm/bench/generator.py +376 -0
  12. cortexm/bench/harness.py +211 -0
  13. cortexm/bench/messy.py +218 -0
  14. cortexm/bench/micro.py +251 -0
  15. cortexm/bench/ood.py +443 -0
  16. cortexm/bench/run.py +137 -0
  17. cortexm/bridge/__init__.py +0 -0
  18. cortexm/bridge/dates.py +178 -0
  19. cortexm/bridge/decoders.py +204 -0
  20. cortexm/bridge/enrich.py +255 -0
  21. cortexm/bridge/extractor.py +316 -0
  22. cortexm/bridge/fallback.py +332 -0
  23. cortexm/bridge/onnx_runtime.py +158 -0
  24. cortexm/bridge/patterns.py +760 -0
  25. cortexm/bridge/ppr.py +104 -0
  26. cortexm/bridge/prefilter.py +188 -0
  27. cortexm/bridge/query_extract.py +420 -0
  28. cortexm/bridge/reader.py +1174 -0
  29. cortexm/bridge/rerank.py +204 -0
  30. cortexm/bridge/writer.py +492 -0
  31. cortexm/cli.py +295 -0
  32. cortexm/cognition/__init__.py +53 -0
  33. cortexm/cognition/abstraction.py +192 -0
  34. cortexm/cognition/analogy.py +159 -0
  35. cortexm/cognition/engine.py +204 -0
  36. cortexm/cognition/gaps.py +365 -0
  37. cortexm/cognition/scanner.py +204 -0
  38. cortexm/config.py +375 -0
  39. cortexm/cortexm.py +8 -0
  40. cortexm/enterprise/__init__.py +0 -0
  41. cortexm/enterprise/audit.py +178 -0
  42. cortexm/enterprise/governance.py +239 -0
  43. cortexm/errors.py +35 -0
  44. cortexm/features/__init__.py +0 -0
  45. cortexm/features/git.py +204 -0
  46. cortexm/features/prefetch.py +88 -0
  47. cortexm/features/zk.py +105 -0
  48. cortexm/federation/__init__.py +39 -0
  49. cortexm/federation/crdt.py +275 -0
  50. cortexm/federation/fabric.py +109 -0
  51. cortexm/federation/hlc.py +80 -0
  52. cortexm/federation/node.py +145 -0
  53. cortexm/federation/schema_report.py +73 -0
  54. cortexm/federation/transport.py +164 -0
  55. cortexm/index/__init__.py +19 -0
  56. cortexm/index/nsg.py +386 -0
  57. cortexm/mcp/__init__.py +0 -0
  58. cortexm/mcp/server.py +985 -0
  59. cortexm/metrics.py +62 -0
  60. cortexm/migrate/__init__.py +0 -0
  61. cortexm/migrate/importers.py +192 -0
  62. cortexm/provenance/__init__.py +78 -0
  63. cortexm/provenance/agent.py +214 -0
  64. cortexm/provenance/cose.py +201 -0
  65. cortexm/provenance/scitt.py +258 -0
  66. cortexm/provenance/vc.py +250 -0
  67. cortexm/security/__init__.py +0 -0
  68. cortexm/security/crypto.py +162 -0
  69. cortexm/security/hashes.py +140 -0
  70. cortexm/security/injection.py +149 -0
  71. cortexm/security/mind.py +154 -0
  72. cortexm/security/pii.py +265 -0
  73. cortexm/security/rbac.py +169 -0
  74. cortexm/security/sandbox.py +131 -0
  75. cortexm/security/zk_hamming.py +142 -0
  76. cortexm/security/zk_sql.py +485 -0
  77. cortexm/server/__init__.py +0 -0
  78. cortexm/server/metrics.py +88 -0
  79. cortexm/server/rest.py +936 -0
  80. cortexm/server/sparql.py +984 -0
  81. cortexm/text/__init__.py +0 -0
  82. cortexm/text/dissim.py +252 -0
  83. cortexm/text/embedder.py +155 -0
  84. cortexm/text/fuzzy.py +218 -0
  85. cortexm/text/idiolect.py +253 -0
  86. cortexm/text/labse.py +374 -0
  87. cortexm/text/tokenizer.py +79 -0
  88. cortexm/trace/__init__.py +0 -0
  89. cortexm/trace/blob_arena.py +277 -0
  90. cortexm/trace/consolidate.py +337 -0
  91. cortexm/trace/contradictions.py +69 -0
  92. cortexm/trace/dedup.py +114 -0
  93. cortexm/trace/edges.py +214 -0
  94. cortexm/trace/fact.py +121 -0
  95. cortexm/trace/fade.py +245 -0
  96. cortexm/trace/lifecycle.py +112 -0
  97. cortexm/trace/rebuild.py +173 -0
  98. cortexm/trace/rules.py +171 -0
  99. cortexm/trace/store.py +680 -0
  100. cortexm/trace/structural.py +183 -0
  101. cortexm/trace/tmt.py +335 -0
  102. cortexm/util.py +148 -0
  103. cortexm/vsa/__init__.py +0 -0
  104. cortexm/vsa/attribution.py +149 -0
  105. cortexm/vsa/cleanup.py +161 -0
  106. cortexm/vsa/codecs.py +397 -0
  107. cortexm/vsa/hologram_overlay.py +139 -0
  108. cortexm/vsa/index.py +163 -0
  109. cortexm/vsa/ops.py +149 -0
  110. cortexm/vsa/palace.py +446 -0
  111. cortexm/vsa/role_vectors.py +236 -0
  112. cortexm/vsa/slb.py +78 -0
  113. cortexm/vsa/tlsh_trie.py +137 -0
  114. cortexm/vsa/working_memory.py +249 -0
  115. cortexm-0.3.0.dist-info/METADATA +482 -0
  116. cortexm-0.3.0.dist-info/RECORD +120 -0
  117. cortexm-0.3.0.dist-info/WHEEL +5 -0
  118. cortexm-0.3.0.dist-info/entry_points.txt +2 -0
  119. cortexm-0.3.0.dist-info/licenses/LICENSE +190 -0
  120. cortexm-0.3.0.dist-info/top_level.txt +2 -0
@@ -0,0 +1,183 @@
1
+ """Deterministic multi-hop structural queries via symbolic Trace + VSA unbind.
2
+
3
+ Complementary to PPR (probabilistic graph diffusion):
4
+ PPR answers: "what else might be relevant?" — fuzzy, associative
5
+ structural_query answers: "exactly follow this chain." — deterministic
6
+
7
+ Inspired by HMS's multiHopQuery API:
8
+
9
+ // "Who is John's grandfather?" (father → father)
10
+ const grandpa = await hms.multiHopQuery('john', ['father', 'father']);
11
+
12
+ Algorithm:
13
+ for each relation in the chain:
14
+ 1. Symbolic lookup: exact match in the Trace (subject=current, rel=r)
15
+ — if a single fact matches, take its value as the next current
16
+ — if multiple match (ambiguous), take the highest-confidence one
17
+ 2. VSA fallback: if no symbolic match, unbind the role hologram
18
+ from the entity hologram in the palace, search the result for
19
+ the nearest stored item vector (Hopfield cleanup). The nearest
20
+ item becomes the next current.
21
+
22
+ Returns a `StructuralQueryResult` with the final value + the chain
23
+ of facts traversed. Ambiguous hops or fallback hops are flagged so
24
+ the caller knows the confidence of the answer.
25
+ """
26
+
27
+ from __future__ import annotations
28
+
29
+ from dataclasses import dataclass, field
30
+ from typing import Any
31
+
32
+
33
+ @dataclass
34
+ class Hop:
35
+ """One hop in a structural query chain."""
36
+ relation: str
37
+ subject: str
38
+ value: str
39
+ fact_id: str = ""
40
+ confidence: float = 0.0
41
+ via: str = "" # "symbolic" | "vsa_fallback" | "abstain"
42
+ ambiguous: bool = False # multiple candidates matched
43
+ alternatives: list[str] = field(default_factory=list)
44
+
45
+
46
+ @dataclass
47
+ class StructuralQueryResult:
48
+ """Result of a multi-hop structural query."""
49
+ start_entity: str
50
+ relation_chain: list[str]
51
+ hops: list[Hop] = field(default_factory=list)
52
+ final_value: str = ""
53
+ success: bool = False
54
+ failure_reason: str = ""
55
+ confidence: float = 0.0
56
+
57
+
58
+ def structural_query(store, palace, start_entity: str,
59
+ relation_chain: list[str],
60
+ user_id: str | None = None,
61
+ allow_hypotheses: bool = False,
62
+ vsa_fallback: bool = True) -> StructuralQueryResult:
63
+ """Deterministic multi-hop via symbolic Trace + VSA unbinding.
64
+
65
+ Args:
66
+ store: TraceStore
67
+ palace: MemoryPalace (for VSA fallback)
68
+ start_entity: the entity to start from
69
+ relation_chain: list of relations to follow in order
70
+ user_id: restrict to a user (None = all users)
71
+ allow_hypotheses: if True, also follow HYPOTHESIZED_BY edges
72
+ (derived facts from the cognition engine)
73
+ vsa_fallback: if True, when no symbolic match, use VSA unbind
74
+ + Hopfield cleanup to propose a filler
75
+
76
+ Returns:
77
+ StructuralQueryResult with hops + final value + confidence
78
+ """
79
+ res = StructuralQueryResult(
80
+ start_entity=start_entity, relation_chain=list(relation_chain))
81
+ current = start_entity
82
+ confidence_product = 1.0
83
+
84
+ for rel in relation_chain:
85
+ # 1. Symbolic lookup in the Trace
86
+ where = ("subject=? AND relation=? AND is_active=1 "
87
+ "AND quarantined=0")
88
+ args: list = [current, rel]
89
+ if user_id is not None:
90
+ where += " AND user_id=?"
91
+ args.append(user_id)
92
+ if not allow_hypotheses:
93
+ where += " AND is_derived=0"
94
+ rows = store.conn.execute(
95
+ f"SELECT id, value, confidence FROM facts WHERE {where} "
96
+ f"ORDER BY confidence DESC, valid_from DESC", args).fetchall()
97
+
98
+ if rows:
99
+ # single match — clean hop
100
+ # SELECT id, value, confidence — columns 0, 1, 2
101
+ row = rows[0]
102
+ ambiguous = len(rows) > 1
103
+ hop = Hop(
104
+ relation=rel, subject=current, value=row[1],
105
+ fact_id=row[0], confidence=row[2] if row[2] is not None else 0.5,
106
+ via="symbolic", ambiguous=ambiguous,
107
+ alternatives=[r[1] for r in rows[1:5]])
108
+ res.hops.append(hop)
109
+ confidence_product *= hop.confidence
110
+ current = row[1]
111
+ continue
112
+
113
+ # 2. VSA fallback: unbind role from entity, search palace
114
+ if vsa_fallback and palace is not None:
115
+ try:
116
+ # encode current entity into a hologram, then unbind
117
+ # the role hologram to get the (noisy) filler hologram
118
+ # and search the palace for the nearest stored item.
119
+ entity_vec = palace._encode_entity(current)
120
+ role_vec = palace.vsa.role_vector(rel)
121
+ probe = palace.vsa.unbind(role_vec, entity_vec)
122
+ # nearest neighbor in the palace
123
+ ids, scores = palace.search(probe, k=3)
124
+ if ids:
125
+ best_id = ids[0]
126
+ best_score = scores[0]
127
+ # fetch the value of the best_id fact
128
+ fact_row = store.conn.execute(
129
+ "SELECT value, confidence FROM facts WHERE id=?",
130
+ (best_id,)).fetchone()
131
+ if fact_row and best_score > 0.30:
132
+ hop = Hop(
133
+ relation=rel, subject=current,
134
+ value=fact_row[0], fact_id=best_id,
135
+ confidence=min(0.49, best_score * 0.5),
136
+ via="vsa_fallback",
137
+ alternatives=[
138
+ ids[i] for i in range(1, len(ids))])
139
+ res.hops.append(hop)
140
+ confidence_product *= hop.confidence
141
+ current = fact_row[0]
142
+ continue
143
+ except Exception:
144
+ pass # palace might not have role_vec for `rel`, etc.
145
+
146
+ # 3. Abstain — no match found, halt the chain
147
+ res.hops.append(Hop(
148
+ relation=rel, subject=current, value="",
149
+ via="abstain"))
150
+ res.success = False
151
+ res.failure_reason = (
152
+ f"no fact matching ({current!r}, {rel!r}) in the trace"
153
+ + ("" if vsa_fallback else " (and VSA fallback disabled)"))
154
+ res.confidence = confidence_product
155
+ return res
156
+
157
+ res.final_value = current
158
+ res.success = True
159
+ res.confidence = confidence_product
160
+ return res
161
+
162
+
163
+ def multi_hop_chain(store, start_entity: str, relation: str,
164
+ depth: int = 2, **kwargs) -> StructuralQueryResult:
165
+ """Convenience: walk the same relation `depth` times.
166
+
167
+ E.g. multi_hop_chain(store, 'alice', 'father', depth=2) answers
168
+ "who is Alice's father's father?" (i.e. grandfather).
169
+ """
170
+ return structural_query(
171
+ store, kwargs.get("palace"),
172
+ start_entity=start_entity,
173
+ relation_chain=[relation] * depth,
174
+ user_id=kwargs.get("user_id"),
175
+ allow_hypotheses=kwargs.get("allow_hypotheses", False),
176
+ vsa_fallback=kwargs.get("vsa_fallback", True),
177
+ )
178
+
179
+
180
+ __all__ = [
181
+ "Hop", "StructuralQueryResult",
182
+ "structural_query", "multi_hop_chain",
183
+ ]
cortexm/trace/tmt.py ADDED
@@ -0,0 +1,335 @@
1
+ """TiMem Temporal Memory Tree — 4-level consolidation hierarchy.
2
+
3
+ arXiv:2601.02845 (TiMem, ACL 2026 Findings) implements a Temporal Memory
4
+ Tree (TMT) with 5 levels: segment → session → day → week → persona.
5
+ It achieves 75.30% on LoCoMo and 76.88% on LongMemEval-S with a
6
+ 52.20% reduction in recalled memory length.
7
+
8
+ Context-M's bi-temporal Trace already has the raw temporal scaffolding
9
+ (valid_from / valid_to / tx_from / tx_to windows + EXTRACTED_FROM edges).
10
+ This module adds the HIERARCHICAL ABSTRACTION layer:
11
+
12
+ L1 (episodic) : raw fact triples (existing)
13
+ L2 (session) : per-session summary fact, derived from L1 facts
14
+ sharing the same user_id + run_id
15
+ L3 (day) : per-day-per-user pattern fact, derived from L2
16
+ summaries sharing the same valid_from date
17
+ L4 (persona) : per-user stable trait fact, derived from L3 across
18
+ >= persona_min_sessions distinct sessions
19
+
20
+ Each higher-level fact is a DERIVED fact (is_derived=True) linked to
21
+ its constituents via DERIVED_FROM edges. The original facts remain
22
+ active — the hierarchy is an OVERLAY, not a replacement.
23
+
24
+ Retrieval benefits:
25
+ * Simple fact lookup → L1 (existing behavior)
26
+ * "What has Carol been up to?" → L2/L3 session/day summaries
27
+ * "What kind of person is Carol?" → L4 persona traits
28
+ * Tokens injected into LLM context drop ~50% because higher levels
29
+ compress multiple L1 facts into one summary.
30
+
31
+ The summaries are written as natural-language sentences so they're
32
+ embeddable and lexically matchable by the existing palace + reader.
33
+
34
+ μ=0 SAFE — the summary generation is rule-based:
35
+ * For a session with N works_at facts, the summary is:
36
+ "<user> worked at <V1>, <V2>, ..., <VN> in this session"
37
+ * For a day's L2 summaries, the summary aggregates the unique
38
+ relations and their top values.
39
+ * For persona, the most-reinforced stable traits (access_count > 3)
40
+ are surfaced.
41
+
42
+ No LLM call. The NL strings are templated and deterministic.
43
+ """
44
+ from __future__ import annotations
45
+
46
+ import datetime as _dt
47
+ from collections import defaultdict
48
+ from datetime import datetime, timezone
49
+
50
+ from cortexm.trace.edges import REFERS_TO
51
+ from cortexm.trace.fact import make_fact
52
+ from cortexm.util import iso
53
+
54
+
55
+ def _now() -> datetime:
56
+ return datetime.now(timezone.utc)
57
+
58
+
59
+ # Edge kind for DERIVED_FROM — the existing trace/edges.py defines
60
+ # CAUSAL/REFERS_TO/MERGED_WITH/RETRACTED_BY/CONTRADICTS/PRECEDED_BY.
61
+ # We use REFERS_TO for the downward link (summary refers to constituents)
62
+ # and the existing EXTRACTED_FROM for the upward link (constituent →
63
+ # summary, mirroring raw_chunk → fact).
64
+
65
+
66
+ def _summarize_session_facts(facts: list, user_id: str,
67
+ run_id: str | None) -> str:
68
+ """Render a session summary NL string from its L1 fact set."""
69
+ relations = defaultdict(set)
70
+ for f in facts:
71
+ if f.is_derived:
72
+ continue
73
+ relations[f.relation].add(f.value)
74
+ parts = []
75
+ for rel, vals in relations.items():
76
+ vals_str = ", ".join(sorted(vals)[:5])
77
+ if len(vals) > 5:
78
+ vals_str += f" (+{len(vals)-5} more)"
79
+ parts.append(f"{rel}: {vals_str}")
80
+ summary = f"session summary for user {user_id}"
81
+ if run_id:
82
+ summary += f" (run {run_id})"
83
+ summary += " — " + "; ".join(parts) if parts else summary + " (no facts)"
84
+ return summary[:500] # cap to keep palace embeddings focused
85
+
86
+
87
+ def _summarize_day_facts(session_summaries: list, user_id: str,
88
+ day_str: str) -> str:
89
+ """Render a daily summary NL string from its L2 session summaries."""
90
+ # session summaries are derived facts; their `value` is the NL string
91
+ parts = [f.value for f in session_summaries if f.value]
92
+ if not parts:
93
+ return f"day summary for user {user_id} on {day_str} (no sessions)"
94
+ summary = (f"day summary for user {user_id} on {day_str} — "
95
+ f"{len(parts)} session(s). Highlights: "
96
+ + " | ".join(parts[:3]))
97
+ if len(parts) > 3:
98
+ summary += f" (+{len(parts)-3} more sessions)"
99
+ return summary[:800]
100
+
101
+
102
+ def _summarize_persona_facts(day_summaries: list, user_id: str,
103
+ all_facts: list) -> str:
104
+ """Render a persona summary NL string from L3 day summaries + L1
105
+ stable traits.
106
+
107
+ Persona = the most-reinforced, longest-lived facts about the user.
108
+ We pick facts with:
109
+ * access_count >= 3 (retrieved multiple times → behaviorally
110
+ relevant)
111
+ * memory_type == "long_term"
112
+ * is_active == True
113
+ """
114
+ # stable traits from L1
115
+ stable = [f for f in all_facts
116
+ if not f.is_derived
117
+ and f.user_id == user_id
118
+ and f.memory_type == "long_term"
119
+ and f.is_active
120
+ and f.access_count >= 3]
121
+ # group by relation, take most-accessed value per relation
122
+ by_rel = defaultdict(list)
123
+ for f in stable:
124
+ by_rel[f.relation].append(f)
125
+ traits = []
126
+ for rel, fs in by_rel.items():
127
+ fs.sort(key=lambda x: -x.access_count)
128
+ top = fs[0]
129
+ traits.append(f"{rel}={top.value} (accessed {top.access_count}x)")
130
+ summary = f"persona profile for user {user_id} — "
131
+ summary += "; ".join(traits[:10])
132
+ if len(traits) > 10:
133
+ summary += f" (+{len(traits)-10} more traits)"
134
+ return summary[:1000]
135
+
136
+
137
+ def _session_key(f) -> tuple:
138
+ """Group facts by (user_id, run_id) for L2 clustering."""
139
+ return (f.user_id, f.run_id or "_no_run")
140
+
141
+
142
+ def _day_key(f) -> str:
143
+ """Group facts by user_id + date portion of valid_from for L3."""
144
+ vf = (f.valid_from or "")[:10] # "YYYY-MM-DD"
145
+ return f"{f.user_id}:{vf or '_no_date'}"
146
+
147
+
148
+ def tmt_build(store, palace=None, *,
149
+ session_cluster_mins: int = 5,
150
+ persona_min_sessions: int = 3,
151
+ user_id: str | None = None,
152
+ dry_run: bool = False) -> dict:
153
+ """Build the 4-level TiMem hierarchy as derived facts + edges.
154
+
155
+ 1. Group active, non-derived L1 facts by (user_id, run_id) →
156
+ emit one L2 session-summary fact per group with >= N facts.
157
+ 2. Group L2 summaries by (user_id, date) → emit one L3 day-summary
158
+ fact per (user, day) with >= 1 session.
159
+ 3. For each user with >= persona_min_sessions distinct sessions,
160
+ emit one L4 persona-summary fact.
161
+
162
+ Each higher-level fact is stored as is_derived=True and linked to
163
+ its constituents via REFERS_TO edges (downward) — bi-temporal safe,
164
+ idempotent (we check for existing derived facts with the same
165
+ `provenance.tmt_key` before re-emitting).
166
+
167
+ Returns a stats dict.
168
+ """
169
+ stats = {
170
+ "l2_sessions_built": 0,
171
+ "l3_days_built": 0,
172
+ "l4_personas_built": 0,
173
+ "l2_skipped_small": 0,
174
+ "l4_skipped_few_sessions": 0,
175
+ "dry_run": dry_run,
176
+ }
177
+
178
+ # --- load L1 facts (active, non-derived) ---------------------------
179
+ where = "is_active=1 AND quarantined=0 AND is_derived=0"
180
+ args: tuple = ()
181
+ if user_id is not None:
182
+ where += " AND user_id=?"
183
+ args = (user_id,)
184
+ rows = store.conn.execute(
185
+ f"SELECT id FROM facts WHERE {where}", args).fetchall()
186
+ fact_ids = [r[0] for r in rows]
187
+ l1_facts = store.get_facts(fact_ids)
188
+ if not l1_facts:
189
+ return stats
190
+
191
+ # open one batch commit for the whole TMT build — idempotent inserts
192
+ # use the existing provenance.tmt_key check before re-emitting, so
193
+ # calling tmt_build repeatedly is safe.
194
+ if not dry_run:
195
+ store.begin_batch()
196
+ commit = store.create_commit(
197
+ f"tmt_build: hierarchy pass for "
198
+ f"{'user='+user_id if user_id else 'all users'}",
199
+ n_facts=0)
200
+ else:
201
+ commit = None
202
+
203
+ # --- L2: per-session summaries -------------------------------------
204
+ sessions: dict[tuple, list] = defaultdict(list)
205
+ for f in l1_facts:
206
+ sessions[_session_key(f)].append(f)
207
+
208
+ l2_facts: list = []
209
+ for (uid, rid), group in sessions.items():
210
+ if len(group) < session_cluster_mins:
211
+ stats["l2_skipped_small"] += 1
212
+ continue
213
+ nl = _summarize_session_facts(group, uid, rid if rid != "_no_run" else None)
214
+ # idempotency: skip if a derived fact with this tmt_key exists
215
+ existing = store.conn.execute(
216
+ "SELECT id FROM facts WHERE is_derived=1 AND "
217
+ "provenance LIKE ? AND user_id=?",
218
+ (f'"tmt_key":"l2:{uid}:{rid}"%', uid)).fetchall()
219
+ if existing:
220
+ continue
221
+ if dry_run:
222
+ stats["l2_sessions_built"] += 1
223
+ continue
224
+ f = make_fact(
225
+ subject=uid, relation="session_summary", value=nl,
226
+ user_id=uid, agent_id=None, run_id=(rid if rid != "_no_run" else None),
227
+ confidence=0.85, memory_type="long_term",
228
+ valid_from=iso(_now())[:10], now=_now(),
229
+ provenance={"tmt_level": "L2", "tmt_key": f"l2:{uid}:{rid}",
230
+ "tmt_constituents": [g.id for g in group[:50]],
231
+ "tmt_constituent_count": len(group)},
232
+ is_derived=True,
233
+ )
234
+ f.birth_commit = commit
235
+ store.insert_fact(f, commit)
236
+ # wire downward edges
237
+ for g in group[:50]: # cap edges for storage
238
+ store.add_edge(f.id, g.id, REFERS_TO,
239
+ {"tmt": "l2_constituent"})
240
+ l2_facts.append(f)
241
+ stats["l2_sessions_built"] += 1
242
+
243
+ # --- L3: per-day summaries ----------------------------------------
244
+ days: dict[str, list] = defaultdict(list)
245
+ for f in l2_facts:
246
+ days[_day_key(f)].append(f)
247
+ # also include standalone L1 facts that didn't get an L2 (small sessions)
248
+ # so day summaries still cover them
249
+ for f in l1_facts:
250
+ days[_day_key(f)].append(f)
251
+
252
+ l3_facts: list = []
253
+ user_days: dict[str, set] = defaultdict(set)
254
+ for day_key, group in days.items():
255
+ uid = day_key.split(":", 1)[0]
256
+ day_str = day_key.split(":", 1)[1] if ":" in day_key else "_no_date"
257
+ l2_in_group = [g for g in group if getattr(g, "is_derived", False)
258
+ and getattr(g, "provenance", {}).get("tmt_level") == "L2"]
259
+ if not l2_in_group:
260
+ continue
261
+ nl = _summarize_day_facts(l2_in_group, uid, day_str)
262
+ existing = store.conn.execute(
263
+ "SELECT id FROM facts WHERE is_derived=1 AND "
264
+ "provenance LIKE ? AND user_id=?",
265
+ (f'"tmt_key":"l3:{uid}:{day_str}"%', uid)).fetchall()
266
+ if existing:
267
+ continue
268
+ if dry_run:
269
+ stats["l3_days_built"] += 1
270
+ continue
271
+ f = make_fact(
272
+ subject=uid, relation="day_summary", value=nl,
273
+ user_id=uid, valid_from=day_str, now=_now(),
274
+ confidence=0.80, memory_type="long_term",
275
+ provenance={"tmt_level": "L3",
276
+ "tmt_key": f"l3:{uid}:{day_str}",
277
+ "tmt_constituents": [g.id for g in l2_in_group[:50]],
278
+ "tmt_constituent_count": len(l2_in_group)},
279
+ is_derived=True,
280
+ )
281
+ f.birth_commit = commit
282
+ store.insert_fact(f, commit)
283
+ for g in l2_in_group[:50]:
284
+ store.add_edge(f.id, g.id, REFERS_TO,
285
+ {"tmt": "l3_constituent"})
286
+ l3_facts.append(f)
287
+ user_days[uid].add(day_str)
288
+ stats["l3_days_built"] += 1
289
+
290
+ # --- L4: per-user persona summaries ------------------------------
291
+ all_users = set([f.user_id for f in l1_facts])
292
+ for uid in all_users:
293
+ if user_id is not None and uid != user_id:
294
+ continue
295
+ if len(user_days.get(uid, set())) < persona_min_sessions:
296
+ stats["l4_skipped_few_sessions"] += 1
297
+ continue
298
+ # all L1 facts for this user, sorted by reinforcement
299
+ user_facts = [f for f in l1_facts if f.user_id == uid]
300
+ nl = _summarize_persona_facts(
301
+ [f for f in l3_facts if f.user_id == uid], uid, user_facts)
302
+ existing = store.conn.execute(
303
+ "SELECT id FROM facts WHERE is_derived=1 AND "
304
+ "provenance LIKE ? AND user_id=?",
305
+ (f'"tmt_key":"l4:{uid}"%', uid)).fetchall()
306
+ if existing:
307
+ continue
308
+ if dry_run:
309
+ stats["l4_personas_built"] += 1
310
+ continue
311
+ f = make_fact(
312
+ subject=uid, relation="persona_summary", value=nl,
313
+ user_id=uid, valid_from=iso(_now())[:10], now=_now(),
314
+ confidence=0.75, memory_type="long_term",
315
+ provenance={"tmt_level": "L4",
316
+ "tmt_key": f"l4:{uid}",
317
+ "tmt_constituent_days": list(user_days[uid])[:50],
318
+ "tmt_constituent_count": len(user_days[uid])},
319
+ is_derived=True,
320
+ )
321
+ f.birth_commit = commit
322
+ store.insert_fact(f, commit)
323
+ for d in l3_facts:
324
+ if d.user_id == uid:
325
+ store.add_edge(f.id, d.id, REFERS_TO,
326
+ {"tmt": "l4_constituent"})
327
+ stats["l4_personas_built"] += 1
328
+
329
+ if not dry_run:
330
+ store.end_batch()
331
+
332
+ return stats
333
+
334
+
335
+ __all__ = ["tmt_build"]
cortexm/util.py ADDED
@@ -0,0 +1,148 @@
1
+ """Shared utilities: time, ids, normalization, string similarity."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import re
6
+ import unicodedata
7
+ import uuid
8
+ from datetime import datetime, timedelta, timezone
9
+
10
+ WORDS = re.compile(r"[a-z0-9']+")
11
+
12
+
13
+ def h64(feature: str, seed: int = 0) -> int:
14
+ """Deterministic 64-bit hash of a string feature (stable across runs)."""
15
+ import hashlib
16
+ return int.from_bytes(
17
+ hashlib.blake2b(feature.encode("utf-8"), digest_size=8,
18
+ key=(seed & 0xFFFFFFFFFFFFFFFF).to_bytes(8, "little")).digest(),
19
+ "little")
20
+
21
+
22
+ def utc_now() -> datetime:
23
+ return datetime.now(timezone.utc)
24
+
25
+
26
+ def iso(dt: datetime) -> str:
27
+ return dt.astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
28
+
29
+
30
+ def parse_ts(value: str | datetime | None) -> datetime | None:
31
+ if value is None:
32
+ return None
33
+ if isinstance(value, datetime):
34
+ return value if value.tzinfo else value.replace(tzinfo=timezone.utc)
35
+ v = value.strip().replace("Z", "+00:00")
36
+ try:
37
+ dt = datetime.fromisoformat(v)
38
+ except ValueError:
39
+ for fmt in ("%Y-%m-%d", "%Y/%m/%d", "%Y-%m-%d %H:%M:%S"):
40
+ try:
41
+ dt = datetime.strptime(value.strip(), fmt)
42
+ break
43
+ except ValueError:
44
+ continue
45
+ else:
46
+ return None
47
+ return dt if dt.tzinfo else dt.replace(tzinfo=timezone.utc)
48
+
49
+
50
+ def new_id() -> str:
51
+ return uuid.uuid4().hex
52
+
53
+
54
+ def normalize(text: str) -> str:
55
+ """Lowercase, strip punctuation/articles, collapse whitespace."""
56
+ text = unicodedata.normalize("NFKD", text)
57
+ text = text.lower()
58
+ text = re.sub(r"[^a-z0-9\s']+", " ", text)
59
+ text = re.sub(r"\b(the|a|an|is|are|was|were)\b", " ", text)
60
+ return re.sub(r"\s+", " ", text).strip()
61
+
62
+
63
+ def words(text: str) -> list[str]:
64
+ return WORDS.findall(text.lower())
65
+
66
+
67
+ def token_estimate(text: str) -> int:
68
+ """Cheap token estimator (~1.3 tokens/word). Deterministic."""
69
+ n = len(text.split())
70
+ return max(1, round(n * 1.3))
71
+
72
+
73
+ def levenshtein(a: str, b: str, cutoff: float = 1.0) -> int:
74
+ """Levenshtein distance with early exit when distance exceeds cutoff*maxlen."""
75
+ la, lb = len(a), len(b)
76
+ if la == 0:
77
+ return lb
78
+ if lb == 0:
79
+ return la
80
+ maxd = int(max(la, lb) * cutoff) + 1
81
+ prev = list(range(lb + 1))
82
+ for i in range(1, la + 1):
83
+ cur = [i] + [0] * lb
84
+ best = i
85
+ for j in range(1, lb + 1):
86
+ cost = 0 if a[i - 1] == b[j - 1] else 1
87
+ cur[j] = min(prev[j] + 1, cur[j - 1] + 1, prev[j - 1] + cost)
88
+ best = min(best, cur[j])
89
+ if best > maxd:
90
+ return maxd
91
+ prev = cur
92
+ return prev[lb]
93
+
94
+
95
+ def similarity(a: str, b: str) -> float:
96
+ """Combined token-Jaccard + Levenshtein string similarity in [0, 1].
97
+
98
+ Fast paths: jaccard decides when clearly similar or clearly
99
+ different; Levenshtein (with early-exit cutoff) only runs in the
100
+ ambiguous band. Keeps conflict analysis linear-ish at scale.
101
+ """
102
+ a_n, b_n = normalize(a), normalize(b)
103
+ if not a_n or not b_n:
104
+ return 0.0
105
+ if a_n == b_n:
106
+ return 1.0
107
+ wa, wb = set(words(a_n)), set(words(b_n))
108
+ union = wa | wb
109
+ jac = len(wa & wb) / len(union) if union else 0.0
110
+ if jac >= 0.92:
111
+ return jac
112
+ if jac < 0.30:
113
+ return jac
114
+ d = levenshtein(a_n, b_n, cutoff=0.75)
115
+ lev = 1.0 - d / max(len(a_n), len(b_n))
116
+ return max(jac, lev)
117
+
118
+
119
+ def month_number(name: str) -> int | None:
120
+ m = name.strip()[:3].lower()
121
+ table = {
122
+ "jan": 1, "feb": 2, "mar": 3, "apr": 4, "may": 5, "jun": 6,
123
+ "jul": 7, "aug": 8, "sep": 9, "oct": 10, "nov": 11, "dec": 12,
124
+ }
125
+ return table.get(m)
126
+
127
+
128
+ def month_name(num: int) -> str:
129
+ return ["January", "February", "March", "April", "May", "June", "July",
130
+ "August", "September", "October", "November", "December"][num - 1]
131
+
132
+
133
+ def days_in(year: int, month: int) -> int:
134
+ if month == 12:
135
+ return 31
136
+ return (datetime(year, month + 1, 1) - timedelta(days=1)).day
137
+
138
+
139
+ def fmt_date(dt: datetime) -> str:
140
+ return dt.strftime("%Y-%m-%d")
141
+
142
+
143
+ def fmt_month(dt: datetime) -> str:
144
+ return dt.strftime("%Y-%m")
145
+
146
+
147
+ def clamp(x: float, lo: float, hi: float) -> float:
148
+ return max(lo, min(hi, x))
File without changes