cortexm 0.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- context_m.py +17 -0
- cortexm/__init__.py +45 -0
- cortexm/accel.py +403 -0
- cortexm/api/__init__.py +0 -0
- cortexm/api/chaos.py +118 -0
- cortexm/api/memory.py +635 -0
- cortexm/bench/__init__.py +0 -0
- cortexm/bench/abilities.py +311 -0
- cortexm/bench/baselines.py +89 -0
- cortexm/bench/beam_loader.py +317 -0
- cortexm/bench/generator.py +376 -0
- cortexm/bench/harness.py +211 -0
- cortexm/bench/messy.py +218 -0
- cortexm/bench/micro.py +251 -0
- cortexm/bench/ood.py +443 -0
- cortexm/bench/run.py +137 -0
- cortexm/bridge/__init__.py +0 -0
- cortexm/bridge/dates.py +178 -0
- cortexm/bridge/decoders.py +204 -0
- cortexm/bridge/enrich.py +255 -0
- cortexm/bridge/extractor.py +316 -0
- cortexm/bridge/fallback.py +332 -0
- cortexm/bridge/onnx_runtime.py +158 -0
- cortexm/bridge/patterns.py +760 -0
- cortexm/bridge/ppr.py +104 -0
- cortexm/bridge/prefilter.py +188 -0
- cortexm/bridge/query_extract.py +420 -0
- cortexm/bridge/reader.py +1174 -0
- cortexm/bridge/rerank.py +204 -0
- cortexm/bridge/writer.py +492 -0
- cortexm/cli.py +295 -0
- cortexm/cognition/__init__.py +53 -0
- cortexm/cognition/abstraction.py +192 -0
- cortexm/cognition/analogy.py +159 -0
- cortexm/cognition/engine.py +204 -0
- cortexm/cognition/gaps.py +365 -0
- cortexm/cognition/scanner.py +204 -0
- cortexm/config.py +375 -0
- cortexm/cortexm.py +8 -0
- cortexm/enterprise/__init__.py +0 -0
- cortexm/enterprise/audit.py +178 -0
- cortexm/enterprise/governance.py +239 -0
- cortexm/errors.py +35 -0
- cortexm/features/__init__.py +0 -0
- cortexm/features/git.py +204 -0
- cortexm/features/prefetch.py +88 -0
- cortexm/features/zk.py +105 -0
- cortexm/federation/__init__.py +39 -0
- cortexm/federation/crdt.py +275 -0
- cortexm/federation/fabric.py +109 -0
- cortexm/federation/hlc.py +80 -0
- cortexm/federation/node.py +145 -0
- cortexm/federation/schema_report.py +73 -0
- cortexm/federation/transport.py +164 -0
- cortexm/index/__init__.py +19 -0
- cortexm/index/nsg.py +386 -0
- cortexm/mcp/__init__.py +0 -0
- cortexm/mcp/server.py +985 -0
- cortexm/metrics.py +62 -0
- cortexm/migrate/__init__.py +0 -0
- cortexm/migrate/importers.py +192 -0
- cortexm/provenance/__init__.py +78 -0
- cortexm/provenance/agent.py +214 -0
- cortexm/provenance/cose.py +201 -0
- cortexm/provenance/scitt.py +258 -0
- cortexm/provenance/vc.py +250 -0
- cortexm/security/__init__.py +0 -0
- cortexm/security/crypto.py +162 -0
- cortexm/security/hashes.py +140 -0
- cortexm/security/injection.py +149 -0
- cortexm/security/mind.py +154 -0
- cortexm/security/pii.py +265 -0
- cortexm/security/rbac.py +169 -0
- cortexm/security/sandbox.py +131 -0
- cortexm/security/zk_hamming.py +142 -0
- cortexm/security/zk_sql.py +485 -0
- cortexm/server/__init__.py +0 -0
- cortexm/server/metrics.py +88 -0
- cortexm/server/rest.py +936 -0
- cortexm/server/sparql.py +984 -0
- cortexm/text/__init__.py +0 -0
- cortexm/text/dissim.py +252 -0
- cortexm/text/embedder.py +155 -0
- cortexm/text/fuzzy.py +218 -0
- cortexm/text/idiolect.py +253 -0
- cortexm/text/labse.py +374 -0
- cortexm/text/tokenizer.py +79 -0
- cortexm/trace/__init__.py +0 -0
- cortexm/trace/blob_arena.py +277 -0
- cortexm/trace/consolidate.py +337 -0
- cortexm/trace/contradictions.py +69 -0
- cortexm/trace/dedup.py +114 -0
- cortexm/trace/edges.py +214 -0
- cortexm/trace/fact.py +121 -0
- cortexm/trace/fade.py +245 -0
- cortexm/trace/lifecycle.py +112 -0
- cortexm/trace/rebuild.py +173 -0
- cortexm/trace/rules.py +171 -0
- cortexm/trace/store.py +680 -0
- cortexm/trace/structural.py +183 -0
- cortexm/trace/tmt.py +335 -0
- cortexm/util.py +148 -0
- cortexm/vsa/__init__.py +0 -0
- cortexm/vsa/attribution.py +149 -0
- cortexm/vsa/cleanup.py +161 -0
- cortexm/vsa/codecs.py +397 -0
- cortexm/vsa/hologram_overlay.py +139 -0
- cortexm/vsa/index.py +163 -0
- cortexm/vsa/ops.py +149 -0
- cortexm/vsa/palace.py +446 -0
- cortexm/vsa/role_vectors.py +236 -0
- cortexm/vsa/slb.py +78 -0
- cortexm/vsa/tlsh_trie.py +137 -0
- cortexm/vsa/working_memory.py +249 -0
- cortexm-0.3.0.dist-info/METADATA +482 -0
- cortexm-0.3.0.dist-info/RECORD +120 -0
- cortexm-0.3.0.dist-info/WHEEL +5 -0
- cortexm-0.3.0.dist-info/entry_points.txt +2 -0
- cortexm-0.3.0.dist-info/licenses/LICENSE +190 -0
- cortexm-0.3.0.dist-info/top_level.txt +2 -0
|
@@ -0,0 +1,183 @@
|
|
|
1
|
+
"""Deterministic multi-hop structural queries via symbolic Trace + VSA unbind.
|
|
2
|
+
|
|
3
|
+
Complementary to PPR (probabilistic graph diffusion):
|
|
4
|
+
PPR answers: "what else might be relevant?" — fuzzy, associative
|
|
5
|
+
structural_query answers: "exactly follow this chain." — deterministic
|
|
6
|
+
|
|
7
|
+
Inspired by HMS's multiHopQuery API:
|
|
8
|
+
|
|
9
|
+
// "Who is John's grandfather?" (father → father)
|
|
10
|
+
const grandpa = await hms.multiHopQuery('john', ['father', 'father']);
|
|
11
|
+
|
|
12
|
+
Algorithm:
|
|
13
|
+
for each relation in the chain:
|
|
14
|
+
1. Symbolic lookup: exact match in the Trace (subject=current, rel=r)
|
|
15
|
+
— if a single fact matches, take its value as the next current
|
|
16
|
+
— if multiple match (ambiguous), take the highest-confidence one
|
|
17
|
+
2. VSA fallback: if no symbolic match, unbind the role hologram
|
|
18
|
+
from the entity hologram in the palace, search the result for
|
|
19
|
+
the nearest stored item vector (Hopfield cleanup). The nearest
|
|
20
|
+
item becomes the next current.
|
|
21
|
+
|
|
22
|
+
Returns a `StructuralQueryResult` with the final value + the chain
|
|
23
|
+
of facts traversed. Ambiguous hops or fallback hops are flagged so
|
|
24
|
+
the caller knows the confidence of the answer.
|
|
25
|
+
"""
|
|
26
|
+
|
|
27
|
+
from __future__ import annotations
|
|
28
|
+
|
|
29
|
+
from dataclasses import dataclass, field
|
|
30
|
+
from typing import Any
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
@dataclass
|
|
34
|
+
class Hop:
|
|
35
|
+
"""One hop in a structural query chain."""
|
|
36
|
+
relation: str
|
|
37
|
+
subject: str
|
|
38
|
+
value: str
|
|
39
|
+
fact_id: str = ""
|
|
40
|
+
confidence: float = 0.0
|
|
41
|
+
via: str = "" # "symbolic" | "vsa_fallback" | "abstain"
|
|
42
|
+
ambiguous: bool = False # multiple candidates matched
|
|
43
|
+
alternatives: list[str] = field(default_factory=list)
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
@dataclass
|
|
47
|
+
class StructuralQueryResult:
|
|
48
|
+
"""Result of a multi-hop structural query."""
|
|
49
|
+
start_entity: str
|
|
50
|
+
relation_chain: list[str]
|
|
51
|
+
hops: list[Hop] = field(default_factory=list)
|
|
52
|
+
final_value: str = ""
|
|
53
|
+
success: bool = False
|
|
54
|
+
failure_reason: str = ""
|
|
55
|
+
confidence: float = 0.0
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def structural_query(store, palace, start_entity: str,
|
|
59
|
+
relation_chain: list[str],
|
|
60
|
+
user_id: str | None = None,
|
|
61
|
+
allow_hypotheses: bool = False,
|
|
62
|
+
vsa_fallback: bool = True) -> StructuralQueryResult:
|
|
63
|
+
"""Deterministic multi-hop via symbolic Trace + VSA unbinding.
|
|
64
|
+
|
|
65
|
+
Args:
|
|
66
|
+
store: TraceStore
|
|
67
|
+
palace: MemoryPalace (for VSA fallback)
|
|
68
|
+
start_entity: the entity to start from
|
|
69
|
+
relation_chain: list of relations to follow in order
|
|
70
|
+
user_id: restrict to a user (None = all users)
|
|
71
|
+
allow_hypotheses: if True, also follow HYPOTHESIZED_BY edges
|
|
72
|
+
(derived facts from the cognition engine)
|
|
73
|
+
vsa_fallback: if True, when no symbolic match, use VSA unbind
|
|
74
|
+
+ Hopfield cleanup to propose a filler
|
|
75
|
+
|
|
76
|
+
Returns:
|
|
77
|
+
StructuralQueryResult with hops + final value + confidence
|
|
78
|
+
"""
|
|
79
|
+
res = StructuralQueryResult(
|
|
80
|
+
start_entity=start_entity, relation_chain=list(relation_chain))
|
|
81
|
+
current = start_entity
|
|
82
|
+
confidence_product = 1.0
|
|
83
|
+
|
|
84
|
+
for rel in relation_chain:
|
|
85
|
+
# 1. Symbolic lookup in the Trace
|
|
86
|
+
where = ("subject=? AND relation=? AND is_active=1 "
|
|
87
|
+
"AND quarantined=0")
|
|
88
|
+
args: list = [current, rel]
|
|
89
|
+
if user_id is not None:
|
|
90
|
+
where += " AND user_id=?"
|
|
91
|
+
args.append(user_id)
|
|
92
|
+
if not allow_hypotheses:
|
|
93
|
+
where += " AND is_derived=0"
|
|
94
|
+
rows = store.conn.execute(
|
|
95
|
+
f"SELECT id, value, confidence FROM facts WHERE {where} "
|
|
96
|
+
f"ORDER BY confidence DESC, valid_from DESC", args).fetchall()
|
|
97
|
+
|
|
98
|
+
if rows:
|
|
99
|
+
# single match — clean hop
|
|
100
|
+
# SELECT id, value, confidence — columns 0, 1, 2
|
|
101
|
+
row = rows[0]
|
|
102
|
+
ambiguous = len(rows) > 1
|
|
103
|
+
hop = Hop(
|
|
104
|
+
relation=rel, subject=current, value=row[1],
|
|
105
|
+
fact_id=row[0], confidence=row[2] if row[2] is not None else 0.5,
|
|
106
|
+
via="symbolic", ambiguous=ambiguous,
|
|
107
|
+
alternatives=[r[1] for r in rows[1:5]])
|
|
108
|
+
res.hops.append(hop)
|
|
109
|
+
confidence_product *= hop.confidence
|
|
110
|
+
current = row[1]
|
|
111
|
+
continue
|
|
112
|
+
|
|
113
|
+
# 2. VSA fallback: unbind role from entity, search palace
|
|
114
|
+
if vsa_fallback and palace is not None:
|
|
115
|
+
try:
|
|
116
|
+
# encode current entity into a hologram, then unbind
|
|
117
|
+
# the role hologram to get the (noisy) filler hologram
|
|
118
|
+
# and search the palace for the nearest stored item.
|
|
119
|
+
entity_vec = palace._encode_entity(current)
|
|
120
|
+
role_vec = palace.vsa.role_vector(rel)
|
|
121
|
+
probe = palace.vsa.unbind(role_vec, entity_vec)
|
|
122
|
+
# nearest neighbor in the palace
|
|
123
|
+
ids, scores = palace.search(probe, k=3)
|
|
124
|
+
if ids:
|
|
125
|
+
best_id = ids[0]
|
|
126
|
+
best_score = scores[0]
|
|
127
|
+
# fetch the value of the best_id fact
|
|
128
|
+
fact_row = store.conn.execute(
|
|
129
|
+
"SELECT value, confidence FROM facts WHERE id=?",
|
|
130
|
+
(best_id,)).fetchone()
|
|
131
|
+
if fact_row and best_score > 0.30:
|
|
132
|
+
hop = Hop(
|
|
133
|
+
relation=rel, subject=current,
|
|
134
|
+
value=fact_row[0], fact_id=best_id,
|
|
135
|
+
confidence=min(0.49, best_score * 0.5),
|
|
136
|
+
via="vsa_fallback",
|
|
137
|
+
alternatives=[
|
|
138
|
+
ids[i] for i in range(1, len(ids))])
|
|
139
|
+
res.hops.append(hop)
|
|
140
|
+
confidence_product *= hop.confidence
|
|
141
|
+
current = fact_row[0]
|
|
142
|
+
continue
|
|
143
|
+
except Exception:
|
|
144
|
+
pass # palace might not have role_vec for `rel`, etc.
|
|
145
|
+
|
|
146
|
+
# 3. Abstain — no match found, halt the chain
|
|
147
|
+
res.hops.append(Hop(
|
|
148
|
+
relation=rel, subject=current, value="",
|
|
149
|
+
via="abstain"))
|
|
150
|
+
res.success = False
|
|
151
|
+
res.failure_reason = (
|
|
152
|
+
f"no fact matching ({current!r}, {rel!r}) in the trace"
|
|
153
|
+
+ ("" if vsa_fallback else " (and VSA fallback disabled)"))
|
|
154
|
+
res.confidence = confidence_product
|
|
155
|
+
return res
|
|
156
|
+
|
|
157
|
+
res.final_value = current
|
|
158
|
+
res.success = True
|
|
159
|
+
res.confidence = confidence_product
|
|
160
|
+
return res
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def multi_hop_chain(store, start_entity: str, relation: str,
|
|
164
|
+
depth: int = 2, **kwargs) -> StructuralQueryResult:
|
|
165
|
+
"""Convenience: walk the same relation `depth` times.
|
|
166
|
+
|
|
167
|
+
E.g. multi_hop_chain(store, 'alice', 'father', depth=2) answers
|
|
168
|
+
"who is Alice's father's father?" (i.e. grandfather).
|
|
169
|
+
"""
|
|
170
|
+
return structural_query(
|
|
171
|
+
store, kwargs.get("palace"),
|
|
172
|
+
start_entity=start_entity,
|
|
173
|
+
relation_chain=[relation] * depth,
|
|
174
|
+
user_id=kwargs.get("user_id"),
|
|
175
|
+
allow_hypotheses=kwargs.get("allow_hypotheses", False),
|
|
176
|
+
vsa_fallback=kwargs.get("vsa_fallback", True),
|
|
177
|
+
)
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
__all__ = [
|
|
181
|
+
"Hop", "StructuralQueryResult",
|
|
182
|
+
"structural_query", "multi_hop_chain",
|
|
183
|
+
]
|
cortexm/trace/tmt.py
ADDED
|
@@ -0,0 +1,335 @@
|
|
|
1
|
+
"""TiMem Temporal Memory Tree — 4-level consolidation hierarchy.
|
|
2
|
+
|
|
3
|
+
arXiv:2601.02845 (TiMem, ACL 2026 Findings) implements a Temporal Memory
|
|
4
|
+
Tree (TMT) with 5 levels: segment → session → day → week → persona.
|
|
5
|
+
It achieves 75.30% on LoCoMo and 76.88% on LongMemEval-S with a
|
|
6
|
+
52.20% reduction in recalled memory length.
|
|
7
|
+
|
|
8
|
+
Context-M's bi-temporal Trace already has the raw temporal scaffolding
|
|
9
|
+
(valid_from / valid_to / tx_from / tx_to windows + EXTRACTED_FROM edges).
|
|
10
|
+
This module adds the HIERARCHICAL ABSTRACTION layer:
|
|
11
|
+
|
|
12
|
+
L1 (episodic) : raw fact triples (existing)
|
|
13
|
+
L2 (session) : per-session summary fact, derived from L1 facts
|
|
14
|
+
sharing the same user_id + run_id
|
|
15
|
+
L3 (day) : per-day-per-user pattern fact, derived from L2
|
|
16
|
+
summaries sharing the same valid_from date
|
|
17
|
+
L4 (persona) : per-user stable trait fact, derived from L3 across
|
|
18
|
+
>= persona_min_sessions distinct sessions
|
|
19
|
+
|
|
20
|
+
Each higher-level fact is a DERIVED fact (is_derived=True) linked to
|
|
21
|
+
its constituents via DERIVED_FROM edges. The original facts remain
|
|
22
|
+
active — the hierarchy is an OVERLAY, not a replacement.
|
|
23
|
+
|
|
24
|
+
Retrieval benefits:
|
|
25
|
+
* Simple fact lookup → L1 (existing behavior)
|
|
26
|
+
* "What has Carol been up to?" → L2/L3 session/day summaries
|
|
27
|
+
* "What kind of person is Carol?" → L4 persona traits
|
|
28
|
+
* Tokens injected into LLM context drop ~50% because higher levels
|
|
29
|
+
compress multiple L1 facts into one summary.
|
|
30
|
+
|
|
31
|
+
The summaries are written as natural-language sentences so they're
|
|
32
|
+
embeddable and lexically matchable by the existing palace + reader.
|
|
33
|
+
|
|
34
|
+
μ=0 SAFE — the summary generation is rule-based:
|
|
35
|
+
* For a session with N works_at facts, the summary is:
|
|
36
|
+
"<user> worked at <V1>, <V2>, ..., <VN> in this session"
|
|
37
|
+
* For a day's L2 summaries, the summary aggregates the unique
|
|
38
|
+
relations and their top values.
|
|
39
|
+
* For persona, the most-reinforced stable traits (access_count > 3)
|
|
40
|
+
are surfaced.
|
|
41
|
+
|
|
42
|
+
No LLM call. The NL strings are templated and deterministic.
|
|
43
|
+
"""
|
|
44
|
+
from __future__ import annotations
|
|
45
|
+
|
|
46
|
+
import datetime as _dt
|
|
47
|
+
from collections import defaultdict
|
|
48
|
+
from datetime import datetime, timezone
|
|
49
|
+
|
|
50
|
+
from cortexm.trace.edges import REFERS_TO
|
|
51
|
+
from cortexm.trace.fact import make_fact
|
|
52
|
+
from cortexm.util import iso
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def _now() -> datetime:
|
|
56
|
+
return datetime.now(timezone.utc)
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
# Edge kind for DERIVED_FROM — the existing trace/edges.py defines
|
|
60
|
+
# CAUSAL/REFERS_TO/MERGED_WITH/RETRACTED_BY/CONTRADICTS/PRECEDED_BY.
|
|
61
|
+
# We use REFERS_TO for the downward link (summary refers to constituents)
|
|
62
|
+
# and the existing EXTRACTED_FROM for the upward link (constituent →
|
|
63
|
+
# summary, mirroring raw_chunk → fact).
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def _summarize_session_facts(facts: list, user_id: str,
|
|
67
|
+
run_id: str | None) -> str:
|
|
68
|
+
"""Render a session summary NL string from its L1 fact set."""
|
|
69
|
+
relations = defaultdict(set)
|
|
70
|
+
for f in facts:
|
|
71
|
+
if f.is_derived:
|
|
72
|
+
continue
|
|
73
|
+
relations[f.relation].add(f.value)
|
|
74
|
+
parts = []
|
|
75
|
+
for rel, vals in relations.items():
|
|
76
|
+
vals_str = ", ".join(sorted(vals)[:5])
|
|
77
|
+
if len(vals) > 5:
|
|
78
|
+
vals_str += f" (+{len(vals)-5} more)"
|
|
79
|
+
parts.append(f"{rel}: {vals_str}")
|
|
80
|
+
summary = f"session summary for user {user_id}"
|
|
81
|
+
if run_id:
|
|
82
|
+
summary += f" (run {run_id})"
|
|
83
|
+
summary += " — " + "; ".join(parts) if parts else summary + " (no facts)"
|
|
84
|
+
return summary[:500] # cap to keep palace embeddings focused
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def _summarize_day_facts(session_summaries: list, user_id: str,
|
|
88
|
+
day_str: str) -> str:
|
|
89
|
+
"""Render a daily summary NL string from its L2 session summaries."""
|
|
90
|
+
# session summaries are derived facts; their `value` is the NL string
|
|
91
|
+
parts = [f.value for f in session_summaries if f.value]
|
|
92
|
+
if not parts:
|
|
93
|
+
return f"day summary for user {user_id} on {day_str} (no sessions)"
|
|
94
|
+
summary = (f"day summary for user {user_id} on {day_str} — "
|
|
95
|
+
f"{len(parts)} session(s). Highlights: "
|
|
96
|
+
+ " | ".join(parts[:3]))
|
|
97
|
+
if len(parts) > 3:
|
|
98
|
+
summary += f" (+{len(parts)-3} more sessions)"
|
|
99
|
+
return summary[:800]
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def _summarize_persona_facts(day_summaries: list, user_id: str,
|
|
103
|
+
all_facts: list) -> str:
|
|
104
|
+
"""Render a persona summary NL string from L3 day summaries + L1
|
|
105
|
+
stable traits.
|
|
106
|
+
|
|
107
|
+
Persona = the most-reinforced, longest-lived facts about the user.
|
|
108
|
+
We pick facts with:
|
|
109
|
+
* access_count >= 3 (retrieved multiple times → behaviorally
|
|
110
|
+
relevant)
|
|
111
|
+
* memory_type == "long_term"
|
|
112
|
+
* is_active == True
|
|
113
|
+
"""
|
|
114
|
+
# stable traits from L1
|
|
115
|
+
stable = [f for f in all_facts
|
|
116
|
+
if not f.is_derived
|
|
117
|
+
and f.user_id == user_id
|
|
118
|
+
and f.memory_type == "long_term"
|
|
119
|
+
and f.is_active
|
|
120
|
+
and f.access_count >= 3]
|
|
121
|
+
# group by relation, take most-accessed value per relation
|
|
122
|
+
by_rel = defaultdict(list)
|
|
123
|
+
for f in stable:
|
|
124
|
+
by_rel[f.relation].append(f)
|
|
125
|
+
traits = []
|
|
126
|
+
for rel, fs in by_rel.items():
|
|
127
|
+
fs.sort(key=lambda x: -x.access_count)
|
|
128
|
+
top = fs[0]
|
|
129
|
+
traits.append(f"{rel}={top.value} (accessed {top.access_count}x)")
|
|
130
|
+
summary = f"persona profile for user {user_id} — "
|
|
131
|
+
summary += "; ".join(traits[:10])
|
|
132
|
+
if len(traits) > 10:
|
|
133
|
+
summary += f" (+{len(traits)-10} more traits)"
|
|
134
|
+
return summary[:1000]
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def _session_key(f) -> tuple:
|
|
138
|
+
"""Group facts by (user_id, run_id) for L2 clustering."""
|
|
139
|
+
return (f.user_id, f.run_id or "_no_run")
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def _day_key(f) -> str:
|
|
143
|
+
"""Group facts by user_id + date portion of valid_from for L3."""
|
|
144
|
+
vf = (f.valid_from or "")[:10] # "YYYY-MM-DD"
|
|
145
|
+
return f"{f.user_id}:{vf or '_no_date'}"
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def tmt_build(store, palace=None, *,
|
|
149
|
+
session_cluster_mins: int = 5,
|
|
150
|
+
persona_min_sessions: int = 3,
|
|
151
|
+
user_id: str | None = None,
|
|
152
|
+
dry_run: bool = False) -> dict:
|
|
153
|
+
"""Build the 4-level TiMem hierarchy as derived facts + edges.
|
|
154
|
+
|
|
155
|
+
1. Group active, non-derived L1 facts by (user_id, run_id) →
|
|
156
|
+
emit one L2 session-summary fact per group with >= N facts.
|
|
157
|
+
2. Group L2 summaries by (user_id, date) → emit one L3 day-summary
|
|
158
|
+
fact per (user, day) with >= 1 session.
|
|
159
|
+
3. For each user with >= persona_min_sessions distinct sessions,
|
|
160
|
+
emit one L4 persona-summary fact.
|
|
161
|
+
|
|
162
|
+
Each higher-level fact is stored as is_derived=True and linked to
|
|
163
|
+
its constituents via REFERS_TO edges (downward) — bi-temporal safe,
|
|
164
|
+
idempotent (we check for existing derived facts with the same
|
|
165
|
+
`provenance.tmt_key` before re-emitting).
|
|
166
|
+
|
|
167
|
+
Returns a stats dict.
|
|
168
|
+
"""
|
|
169
|
+
stats = {
|
|
170
|
+
"l2_sessions_built": 0,
|
|
171
|
+
"l3_days_built": 0,
|
|
172
|
+
"l4_personas_built": 0,
|
|
173
|
+
"l2_skipped_small": 0,
|
|
174
|
+
"l4_skipped_few_sessions": 0,
|
|
175
|
+
"dry_run": dry_run,
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
# --- load L1 facts (active, non-derived) ---------------------------
|
|
179
|
+
where = "is_active=1 AND quarantined=0 AND is_derived=0"
|
|
180
|
+
args: tuple = ()
|
|
181
|
+
if user_id is not None:
|
|
182
|
+
where += " AND user_id=?"
|
|
183
|
+
args = (user_id,)
|
|
184
|
+
rows = store.conn.execute(
|
|
185
|
+
f"SELECT id FROM facts WHERE {where}", args).fetchall()
|
|
186
|
+
fact_ids = [r[0] for r in rows]
|
|
187
|
+
l1_facts = store.get_facts(fact_ids)
|
|
188
|
+
if not l1_facts:
|
|
189
|
+
return stats
|
|
190
|
+
|
|
191
|
+
# open one batch commit for the whole TMT build — idempotent inserts
|
|
192
|
+
# use the existing provenance.tmt_key check before re-emitting, so
|
|
193
|
+
# calling tmt_build repeatedly is safe.
|
|
194
|
+
if not dry_run:
|
|
195
|
+
store.begin_batch()
|
|
196
|
+
commit = store.create_commit(
|
|
197
|
+
f"tmt_build: hierarchy pass for "
|
|
198
|
+
f"{'user='+user_id if user_id else 'all users'}",
|
|
199
|
+
n_facts=0)
|
|
200
|
+
else:
|
|
201
|
+
commit = None
|
|
202
|
+
|
|
203
|
+
# --- L2: per-session summaries -------------------------------------
|
|
204
|
+
sessions: dict[tuple, list] = defaultdict(list)
|
|
205
|
+
for f in l1_facts:
|
|
206
|
+
sessions[_session_key(f)].append(f)
|
|
207
|
+
|
|
208
|
+
l2_facts: list = []
|
|
209
|
+
for (uid, rid), group in sessions.items():
|
|
210
|
+
if len(group) < session_cluster_mins:
|
|
211
|
+
stats["l2_skipped_small"] += 1
|
|
212
|
+
continue
|
|
213
|
+
nl = _summarize_session_facts(group, uid, rid if rid != "_no_run" else None)
|
|
214
|
+
# idempotency: skip if a derived fact with this tmt_key exists
|
|
215
|
+
existing = store.conn.execute(
|
|
216
|
+
"SELECT id FROM facts WHERE is_derived=1 AND "
|
|
217
|
+
"provenance LIKE ? AND user_id=?",
|
|
218
|
+
(f'"tmt_key":"l2:{uid}:{rid}"%', uid)).fetchall()
|
|
219
|
+
if existing:
|
|
220
|
+
continue
|
|
221
|
+
if dry_run:
|
|
222
|
+
stats["l2_sessions_built"] += 1
|
|
223
|
+
continue
|
|
224
|
+
f = make_fact(
|
|
225
|
+
subject=uid, relation="session_summary", value=nl,
|
|
226
|
+
user_id=uid, agent_id=None, run_id=(rid if rid != "_no_run" else None),
|
|
227
|
+
confidence=0.85, memory_type="long_term",
|
|
228
|
+
valid_from=iso(_now())[:10], now=_now(),
|
|
229
|
+
provenance={"tmt_level": "L2", "tmt_key": f"l2:{uid}:{rid}",
|
|
230
|
+
"tmt_constituents": [g.id for g in group[:50]],
|
|
231
|
+
"tmt_constituent_count": len(group)},
|
|
232
|
+
is_derived=True,
|
|
233
|
+
)
|
|
234
|
+
f.birth_commit = commit
|
|
235
|
+
store.insert_fact(f, commit)
|
|
236
|
+
# wire downward edges
|
|
237
|
+
for g in group[:50]: # cap edges for storage
|
|
238
|
+
store.add_edge(f.id, g.id, REFERS_TO,
|
|
239
|
+
{"tmt": "l2_constituent"})
|
|
240
|
+
l2_facts.append(f)
|
|
241
|
+
stats["l2_sessions_built"] += 1
|
|
242
|
+
|
|
243
|
+
# --- L3: per-day summaries ----------------------------------------
|
|
244
|
+
days: dict[str, list] = defaultdict(list)
|
|
245
|
+
for f in l2_facts:
|
|
246
|
+
days[_day_key(f)].append(f)
|
|
247
|
+
# also include standalone L1 facts that didn't get an L2 (small sessions)
|
|
248
|
+
# so day summaries still cover them
|
|
249
|
+
for f in l1_facts:
|
|
250
|
+
days[_day_key(f)].append(f)
|
|
251
|
+
|
|
252
|
+
l3_facts: list = []
|
|
253
|
+
user_days: dict[str, set] = defaultdict(set)
|
|
254
|
+
for day_key, group in days.items():
|
|
255
|
+
uid = day_key.split(":", 1)[0]
|
|
256
|
+
day_str = day_key.split(":", 1)[1] if ":" in day_key else "_no_date"
|
|
257
|
+
l2_in_group = [g for g in group if getattr(g, "is_derived", False)
|
|
258
|
+
and getattr(g, "provenance", {}).get("tmt_level") == "L2"]
|
|
259
|
+
if not l2_in_group:
|
|
260
|
+
continue
|
|
261
|
+
nl = _summarize_day_facts(l2_in_group, uid, day_str)
|
|
262
|
+
existing = store.conn.execute(
|
|
263
|
+
"SELECT id FROM facts WHERE is_derived=1 AND "
|
|
264
|
+
"provenance LIKE ? AND user_id=?",
|
|
265
|
+
(f'"tmt_key":"l3:{uid}:{day_str}"%', uid)).fetchall()
|
|
266
|
+
if existing:
|
|
267
|
+
continue
|
|
268
|
+
if dry_run:
|
|
269
|
+
stats["l3_days_built"] += 1
|
|
270
|
+
continue
|
|
271
|
+
f = make_fact(
|
|
272
|
+
subject=uid, relation="day_summary", value=nl,
|
|
273
|
+
user_id=uid, valid_from=day_str, now=_now(),
|
|
274
|
+
confidence=0.80, memory_type="long_term",
|
|
275
|
+
provenance={"tmt_level": "L3",
|
|
276
|
+
"tmt_key": f"l3:{uid}:{day_str}",
|
|
277
|
+
"tmt_constituents": [g.id for g in l2_in_group[:50]],
|
|
278
|
+
"tmt_constituent_count": len(l2_in_group)},
|
|
279
|
+
is_derived=True,
|
|
280
|
+
)
|
|
281
|
+
f.birth_commit = commit
|
|
282
|
+
store.insert_fact(f, commit)
|
|
283
|
+
for g in l2_in_group[:50]:
|
|
284
|
+
store.add_edge(f.id, g.id, REFERS_TO,
|
|
285
|
+
{"tmt": "l3_constituent"})
|
|
286
|
+
l3_facts.append(f)
|
|
287
|
+
user_days[uid].add(day_str)
|
|
288
|
+
stats["l3_days_built"] += 1
|
|
289
|
+
|
|
290
|
+
# --- L4: per-user persona summaries ------------------------------
|
|
291
|
+
all_users = set([f.user_id for f in l1_facts])
|
|
292
|
+
for uid in all_users:
|
|
293
|
+
if user_id is not None and uid != user_id:
|
|
294
|
+
continue
|
|
295
|
+
if len(user_days.get(uid, set())) < persona_min_sessions:
|
|
296
|
+
stats["l4_skipped_few_sessions"] += 1
|
|
297
|
+
continue
|
|
298
|
+
# all L1 facts for this user, sorted by reinforcement
|
|
299
|
+
user_facts = [f for f in l1_facts if f.user_id == uid]
|
|
300
|
+
nl = _summarize_persona_facts(
|
|
301
|
+
[f for f in l3_facts if f.user_id == uid], uid, user_facts)
|
|
302
|
+
existing = store.conn.execute(
|
|
303
|
+
"SELECT id FROM facts WHERE is_derived=1 AND "
|
|
304
|
+
"provenance LIKE ? AND user_id=?",
|
|
305
|
+
(f'"tmt_key":"l4:{uid}"%', uid)).fetchall()
|
|
306
|
+
if existing:
|
|
307
|
+
continue
|
|
308
|
+
if dry_run:
|
|
309
|
+
stats["l4_personas_built"] += 1
|
|
310
|
+
continue
|
|
311
|
+
f = make_fact(
|
|
312
|
+
subject=uid, relation="persona_summary", value=nl,
|
|
313
|
+
user_id=uid, valid_from=iso(_now())[:10], now=_now(),
|
|
314
|
+
confidence=0.75, memory_type="long_term",
|
|
315
|
+
provenance={"tmt_level": "L4",
|
|
316
|
+
"tmt_key": f"l4:{uid}",
|
|
317
|
+
"tmt_constituent_days": list(user_days[uid])[:50],
|
|
318
|
+
"tmt_constituent_count": len(user_days[uid])},
|
|
319
|
+
is_derived=True,
|
|
320
|
+
)
|
|
321
|
+
f.birth_commit = commit
|
|
322
|
+
store.insert_fact(f, commit)
|
|
323
|
+
for d in l3_facts:
|
|
324
|
+
if d.user_id == uid:
|
|
325
|
+
store.add_edge(f.id, d.id, REFERS_TO,
|
|
326
|
+
{"tmt": "l4_constituent"})
|
|
327
|
+
stats["l4_personas_built"] += 1
|
|
328
|
+
|
|
329
|
+
if not dry_run:
|
|
330
|
+
store.end_batch()
|
|
331
|
+
|
|
332
|
+
return stats
|
|
333
|
+
|
|
334
|
+
|
|
335
|
+
__all__ = ["tmt_build"]
|
cortexm/util.py
ADDED
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
"""Shared utilities: time, ids, normalization, string similarity."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import re
|
|
6
|
+
import unicodedata
|
|
7
|
+
import uuid
|
|
8
|
+
from datetime import datetime, timedelta, timezone
|
|
9
|
+
|
|
10
|
+
WORDS = re.compile(r"[a-z0-9']+")
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def h64(feature: str, seed: int = 0) -> int:
|
|
14
|
+
"""Deterministic 64-bit hash of a string feature (stable across runs)."""
|
|
15
|
+
import hashlib
|
|
16
|
+
return int.from_bytes(
|
|
17
|
+
hashlib.blake2b(feature.encode("utf-8"), digest_size=8,
|
|
18
|
+
key=(seed & 0xFFFFFFFFFFFFFFFF).to_bytes(8, "little")).digest(),
|
|
19
|
+
"little")
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def utc_now() -> datetime:
|
|
23
|
+
return datetime.now(timezone.utc)
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def iso(dt: datetime) -> str:
|
|
27
|
+
return dt.astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def parse_ts(value: str | datetime | None) -> datetime | None:
|
|
31
|
+
if value is None:
|
|
32
|
+
return None
|
|
33
|
+
if isinstance(value, datetime):
|
|
34
|
+
return value if value.tzinfo else value.replace(tzinfo=timezone.utc)
|
|
35
|
+
v = value.strip().replace("Z", "+00:00")
|
|
36
|
+
try:
|
|
37
|
+
dt = datetime.fromisoformat(v)
|
|
38
|
+
except ValueError:
|
|
39
|
+
for fmt in ("%Y-%m-%d", "%Y/%m/%d", "%Y-%m-%d %H:%M:%S"):
|
|
40
|
+
try:
|
|
41
|
+
dt = datetime.strptime(value.strip(), fmt)
|
|
42
|
+
break
|
|
43
|
+
except ValueError:
|
|
44
|
+
continue
|
|
45
|
+
else:
|
|
46
|
+
return None
|
|
47
|
+
return dt if dt.tzinfo else dt.replace(tzinfo=timezone.utc)
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def new_id() -> str:
|
|
51
|
+
return uuid.uuid4().hex
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def normalize(text: str) -> str:
|
|
55
|
+
"""Lowercase, strip punctuation/articles, collapse whitespace."""
|
|
56
|
+
text = unicodedata.normalize("NFKD", text)
|
|
57
|
+
text = text.lower()
|
|
58
|
+
text = re.sub(r"[^a-z0-9\s']+", " ", text)
|
|
59
|
+
text = re.sub(r"\b(the|a|an|is|are|was|were)\b", " ", text)
|
|
60
|
+
return re.sub(r"\s+", " ", text).strip()
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def words(text: str) -> list[str]:
|
|
64
|
+
return WORDS.findall(text.lower())
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def token_estimate(text: str) -> int:
|
|
68
|
+
"""Cheap token estimator (~1.3 tokens/word). Deterministic."""
|
|
69
|
+
n = len(text.split())
|
|
70
|
+
return max(1, round(n * 1.3))
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def levenshtein(a: str, b: str, cutoff: float = 1.0) -> int:
|
|
74
|
+
"""Levenshtein distance with early exit when distance exceeds cutoff*maxlen."""
|
|
75
|
+
la, lb = len(a), len(b)
|
|
76
|
+
if la == 0:
|
|
77
|
+
return lb
|
|
78
|
+
if lb == 0:
|
|
79
|
+
return la
|
|
80
|
+
maxd = int(max(la, lb) * cutoff) + 1
|
|
81
|
+
prev = list(range(lb + 1))
|
|
82
|
+
for i in range(1, la + 1):
|
|
83
|
+
cur = [i] + [0] * lb
|
|
84
|
+
best = i
|
|
85
|
+
for j in range(1, lb + 1):
|
|
86
|
+
cost = 0 if a[i - 1] == b[j - 1] else 1
|
|
87
|
+
cur[j] = min(prev[j] + 1, cur[j - 1] + 1, prev[j - 1] + cost)
|
|
88
|
+
best = min(best, cur[j])
|
|
89
|
+
if best > maxd:
|
|
90
|
+
return maxd
|
|
91
|
+
prev = cur
|
|
92
|
+
return prev[lb]
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def similarity(a: str, b: str) -> float:
|
|
96
|
+
"""Combined token-Jaccard + Levenshtein string similarity in [0, 1].
|
|
97
|
+
|
|
98
|
+
Fast paths: jaccard decides when clearly similar or clearly
|
|
99
|
+
different; Levenshtein (with early-exit cutoff) only runs in the
|
|
100
|
+
ambiguous band. Keeps conflict analysis linear-ish at scale.
|
|
101
|
+
"""
|
|
102
|
+
a_n, b_n = normalize(a), normalize(b)
|
|
103
|
+
if not a_n or not b_n:
|
|
104
|
+
return 0.0
|
|
105
|
+
if a_n == b_n:
|
|
106
|
+
return 1.0
|
|
107
|
+
wa, wb = set(words(a_n)), set(words(b_n))
|
|
108
|
+
union = wa | wb
|
|
109
|
+
jac = len(wa & wb) / len(union) if union else 0.0
|
|
110
|
+
if jac >= 0.92:
|
|
111
|
+
return jac
|
|
112
|
+
if jac < 0.30:
|
|
113
|
+
return jac
|
|
114
|
+
d = levenshtein(a_n, b_n, cutoff=0.75)
|
|
115
|
+
lev = 1.0 - d / max(len(a_n), len(b_n))
|
|
116
|
+
return max(jac, lev)
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def month_number(name: str) -> int | None:
|
|
120
|
+
m = name.strip()[:3].lower()
|
|
121
|
+
table = {
|
|
122
|
+
"jan": 1, "feb": 2, "mar": 3, "apr": 4, "may": 5, "jun": 6,
|
|
123
|
+
"jul": 7, "aug": 8, "sep": 9, "oct": 10, "nov": 11, "dec": 12,
|
|
124
|
+
}
|
|
125
|
+
return table.get(m)
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def month_name(num: int) -> str:
|
|
129
|
+
return ["January", "February", "March", "April", "May", "June", "July",
|
|
130
|
+
"August", "September", "October", "November", "December"][num - 1]
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def days_in(year: int, month: int) -> int:
|
|
134
|
+
if month == 12:
|
|
135
|
+
return 31
|
|
136
|
+
return (datetime(year, month + 1, 1) - timedelta(days=1)).day
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def fmt_date(dt: datetime) -> str:
|
|
140
|
+
return dt.strftime("%Y-%m-%d")
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def fmt_month(dt: datetime) -> str:
|
|
144
|
+
return dt.strftime("%Y-%m")
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
def clamp(x: float, lo: float, hi: float) -> float:
|
|
148
|
+
return max(lo, min(hi, x))
|
cortexm/vsa/__init__.py
ADDED
|
File without changes
|