graph-mind-memory 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- associative_memory.py +507 -0
- automatic_capture.py +288 -0
- automatic_capture_cli.py +72 -0
- brain_log.py +501 -0
- coding_memory.py +438 -0
- concept_plan_benchmark_v071.py +248 -0
- conversation_memory.py +476 -0
- development_memory.py +450 -0
- development_paths.py +40 -0
- embedding_warmup.py +171 -0
- entity_timeline.py +130 -0
- executable_memory.py +312 -0
- executable_plan_v2_diagnostic.py +272 -0
- graph_mind_mcp_server.py +864 -0
- graph_mind_memory-0.1.0.dist-info/METADATA +252 -0
- graph_mind_memory-0.1.0.dist-info/RECORD +43 -0
- graph_mind_memory-0.1.0.dist-info/WHEEL +5 -0
- graph_mind_memory-0.1.0.dist-info/entry_points.txt +4 -0
- graph_mind_memory-0.1.0.dist-info/licenses/LICENSE +661 -0
- graph_mind_memory-0.1.0.dist-info/top_level.txt +37 -0
- grounded_concept_bridge_v072.py +511 -0
- ingestion_gate.py +113 -0
- ingestion_synonyms.py +165 -0
- install.py +198 -0
- llm_concept_planner.py +170 -0
- local_brain.py +354 -0
- longmemeval_adapter.py +120 -0
- longmemeval_ingestion_calibration.py +366 -0
- longmemeval_ingestion_preflight.py +188 -0
- longmemeval_ingestion_v7.py +356 -0
- memory_algebra.py +155 -0
- memory_contract.py +154 -0
- memory_normalization.py +13 -0
- memory_vocabulary.py +34 -0
- semantic_grounding_v073.py +501 -0
- semantic_llm_extractor.py +215 -0
- semantic_memory.py +155 -0
- semantic_recall.py +100 -0
- state_relation_contract.py +158 -0
- temporal_resolver.py +272 -0
- temporal_state.py +210 -0
- universal_personal_memory.py +234 -0
- vector_cache.py +194 -0
associative_memory.py
ADDED
|
@@ -0,0 +1,507 @@
|
|
|
1
|
+
"""Bounded sparse associative recall over Graph-MIND's evidence vault."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
from development_paths import use_wal
|
|
5
|
+
|
|
6
|
+
from datetime import datetime
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
import hashlib
|
|
9
|
+
import json
|
|
10
|
+
import math
|
|
11
|
+
import re
|
|
12
|
+
import sqlite3
|
|
13
|
+
import unicodedata
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
INDEX_VERSION = "engram-associative-layer-v0.6"
|
|
17
|
+
STOP_TERMS = frozenset({
|
|
18
|
+
"a", "an", "and", "are", "as", "at", "be", "by", "for", "from", "in",
|
|
19
|
+
"is", "it", "of", "on", "or", "that", "the", "this", "to", "was", "were",
|
|
20
|
+
"with", "것", "그", "내", "나", "및", "수", "이", "저", "좀", "한", "할",
|
|
21
|
+
})
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def _canonical(value):
|
|
25
|
+
return json.dumps(value, ensure_ascii=False, sort_keys=True,
|
|
26
|
+
separators=(",", ":"), allow_nan=False)
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def _text(value, error):
|
|
30
|
+
if not isinstance(value, str) or not value.strip():
|
|
31
|
+
raise ValueError(error)
|
|
32
|
+
return value.strip()
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _terms(value):
|
|
36
|
+
normalized = unicodedata.normalize("NFKC", value).casefold()
|
|
37
|
+
found = re.findall(r"[^\W_]{2,}", normalized, re.UNICODE)
|
|
38
|
+
return list(dict.fromkeys(term for term in found if term not in STOP_TERMS))
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def _unit(value, error):
|
|
42
|
+
if isinstance(value, bool) or not isinstance(value, (int, float)):
|
|
43
|
+
raise ValueError(error)
|
|
44
|
+
value = float(value)
|
|
45
|
+
if not 0.0 <= value <= 1.0:
|
|
46
|
+
raise ValueError(error)
|
|
47
|
+
return value
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
class AssociativeMemoryIndex:
|
|
51
|
+
"""Associative index; evidence remains owned by the source memory tables."""
|
|
52
|
+
|
|
53
|
+
def __init__(self, path):
|
|
54
|
+
self.path = Path(path)
|
|
55
|
+
self.path.parent.mkdir(parents=True, exist_ok=True)
|
|
56
|
+
# Claude, Codex and the capture service each open this file from their own process.
|
|
57
|
+
# WAL lets them read while one writes; the timeout makes a writer wait its turn
|
|
58
|
+
# instead of failing with "database is locked".
|
|
59
|
+
self.db = sqlite3.connect(str(self.path), timeout=30)
|
|
60
|
+
use_wal(self.db)
|
|
61
|
+
self.db.row_factory = sqlite3.Row
|
|
62
|
+
self.closed = False
|
|
63
|
+
self.db.execute("PRAGMA foreign_keys=ON")
|
|
64
|
+
self.db.executescript("""
|
|
65
|
+
CREATE TABLE IF NOT EXISTS associative_meta(
|
|
66
|
+
key TEXT PRIMARY KEY, value TEXT NOT NULL);
|
|
67
|
+
CREATE TABLE IF NOT EXISTS associative_engrams(
|
|
68
|
+
revision INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
69
|
+
engram_id TEXT NOT NULL UNIQUE,
|
|
70
|
+
source_kind TEXT NOT NULL,
|
|
71
|
+
source_id TEXT NOT NULL,
|
|
72
|
+
scope TEXT NOT NULL,
|
|
73
|
+
happened_at TEXT NOT NULL,
|
|
74
|
+
strength REAL NOT NULL,
|
|
75
|
+
access_count INTEGER NOT NULL DEFAULT 0,
|
|
76
|
+
last_activated_at TEXT,
|
|
77
|
+
signature_json TEXT NOT NULL,
|
|
78
|
+
fingerprint TEXT NOT NULL);
|
|
79
|
+
CREATE INDEX IF NOT EXISTS associative_scope_revision
|
|
80
|
+
ON associative_engrams(scope,revision DESC);
|
|
81
|
+
CREATE TABLE IF NOT EXISTS associative_terms(
|
|
82
|
+
term TEXT NOT NULL,
|
|
83
|
+
engram_id TEXT NOT NULL,
|
|
84
|
+
PRIMARY KEY(term,engram_id),
|
|
85
|
+
FOREIGN KEY(engram_id) REFERENCES associative_engrams(engram_id));
|
|
86
|
+
CREATE INDEX IF NOT EXISTS associative_term_engram
|
|
87
|
+
ON associative_terms(engram_id,term);
|
|
88
|
+
CREATE TABLE IF NOT EXISTS associative_edges(
|
|
89
|
+
source_engram_id TEXT NOT NULL,
|
|
90
|
+
target_engram_id TEXT NOT NULL,
|
|
91
|
+
relation TEXT NOT NULL,
|
|
92
|
+
weight REAL NOT NULL,
|
|
93
|
+
evidence_count INTEGER NOT NULL,
|
|
94
|
+
reinforced_at TEXT NOT NULL,
|
|
95
|
+
PRIMARY KEY(source_engram_id,target_engram_id,relation),
|
|
96
|
+
FOREIGN KEY(source_engram_id) REFERENCES associative_engrams(engram_id),
|
|
97
|
+
FOREIGN KEY(target_engram_id) REFERENCES associative_engrams(engram_id));
|
|
98
|
+
CREATE INDEX IF NOT EXISTS associative_edge_source
|
|
99
|
+
ON associative_edges(source_engram_id,weight DESC);
|
|
100
|
+
CREATE TABLE IF NOT EXISTS associative_pending_edges(
|
|
101
|
+
source_engram_id TEXT NOT NULL,
|
|
102
|
+
target_engram_id TEXT NOT NULL,
|
|
103
|
+
relation TEXT NOT NULL,
|
|
104
|
+
weight REAL NOT NULL,
|
|
105
|
+
reinforced_at TEXT NOT NULL,
|
|
106
|
+
PRIMARY KEY(source_engram_id,target_engram_id,relation));
|
|
107
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS associative_engrams_fts USING fts5(
|
|
108
|
+
engram_id UNINDEXED, text, tokenize='unicode61');
|
|
109
|
+
""")
|
|
110
|
+
with self.db:
|
|
111
|
+
self.db.execute("INSERT OR IGNORE INTO associative_meta VALUES('version',?)",
|
|
112
|
+
(INDEX_VERSION,))
|
|
113
|
+
version = self.db.execute(
|
|
114
|
+
"SELECT value FROM associative_meta WHERE key='version'").fetchone()[0]
|
|
115
|
+
if version != INDEX_VERSION:
|
|
116
|
+
self.close()
|
|
117
|
+
raise ValueError("ASSOCIATIVE_INDEX_VERSION_CHANGED")
|
|
118
|
+
|
|
119
|
+
def __enter__(self):
|
|
120
|
+
return self
|
|
121
|
+
|
|
122
|
+
def __exit__(self, *args):
|
|
123
|
+
self.close()
|
|
124
|
+
|
|
125
|
+
def close(self):
|
|
126
|
+
if not self.closed:
|
|
127
|
+
self.closed = True
|
|
128
|
+
self.db.close()
|
|
129
|
+
|
|
130
|
+
def _link(self, source, target, relation, weight, when):
|
|
131
|
+
if source == target:
|
|
132
|
+
return
|
|
133
|
+
self.db.execute("""INSERT INTO associative_edges(
|
|
134
|
+
source_engram_id,target_engram_id,relation,weight,evidence_count,reinforced_at)
|
|
135
|
+
VALUES(?,?,?,?,1,?)
|
|
136
|
+
ON CONFLICT(source_engram_id,target_engram_id,relation) DO UPDATE SET
|
|
137
|
+
weight=MAX(weight,excluded.weight),
|
|
138
|
+
evidence_count=evidence_count+1,
|
|
139
|
+
reinforced_at=excluded.reinforced_at""",
|
|
140
|
+
(source, target, relation, weight, when))
|
|
141
|
+
|
|
142
|
+
def _resolve_pending(self):
|
|
143
|
+
rows = self.db.execute("""SELECT p.* FROM associative_pending_edges p
|
|
144
|
+
JOIN associative_engrams source ON source.engram_id=p.source_engram_id
|
|
145
|
+
JOIN associative_engrams target ON target.engram_id=p.target_engram_id""").fetchall()
|
|
146
|
+
for row in rows:
|
|
147
|
+
self._link(row["source_engram_id"], row["target_engram_id"],
|
|
148
|
+
row["relation"], row["weight"], row["reinforced_at"])
|
|
149
|
+
self._link(row["target_engram_id"], row["source_engram_id"],
|
|
150
|
+
row["relation"], row["weight"], row["reinforced_at"])
|
|
151
|
+
self.db.execute("""DELETE FROM associative_pending_edges
|
|
152
|
+
WHERE source_engram_id=? AND target_engram_id=? AND relation=?""",
|
|
153
|
+
(row["source_engram_id"], row["target_engram_id"], row["relation"]))
|
|
154
|
+
|
|
155
|
+
def add_engram(self, *, engram_id, source_kind, source_id, scope, happened_at,
|
|
156
|
+
text, cues=None, strength=0.5, related_engram_ids=None,
|
|
157
|
+
max_automatic_links=12):
|
|
158
|
+
normalized = {
|
|
159
|
+
"engram_id": _text(engram_id, "ENGRAM_ID_REQUIRED"),
|
|
160
|
+
"source_kind": _text(source_kind, "ENGRAM_SOURCE_KIND_REQUIRED").upper(),
|
|
161
|
+
"source_id": _text(source_id, "ENGRAM_SOURCE_ID_REQUIRED"),
|
|
162
|
+
"scope": _text(scope, "ENGRAM_SCOPE_REQUIRED"),
|
|
163
|
+
"happened_at": _text(happened_at, "ENGRAM_TIME_REQUIRED"),
|
|
164
|
+
"text": _text(text, "ENGRAM_TEXT_REQUIRED"),
|
|
165
|
+
"strength": _unit(strength, "INVALID_ENGRAM_STRENGTH"),
|
|
166
|
+
}
|
|
167
|
+
cue_values = cues or []
|
|
168
|
+
if not isinstance(cue_values, list) or not all(isinstance(x, str) for x in cue_values):
|
|
169
|
+
raise ValueError("INVALID_ENGRAM_CUES")
|
|
170
|
+
related = related_engram_ids or []
|
|
171
|
+
if not isinstance(related, list) or not all(isinstance(x, str) and x.strip()
|
|
172
|
+
for x in related):
|
|
173
|
+
raise ValueError("INVALID_RELATED_ENGRAM_IDS")
|
|
174
|
+
if type(max_automatic_links) is not int or not 0 <= max_automatic_links <= 50:
|
|
175
|
+
raise ValueError("INVALID_AUTOMATIC_LINK_LIMIT")
|
|
176
|
+
signature = _terms(" ".join([normalized["text"], *cue_values]))
|
|
177
|
+
if not signature:
|
|
178
|
+
raise ValueError("ENGRAM_HAS_NO_SEARCHABLE_SIGNATURE")
|
|
179
|
+
fingerprint_value = {**normalized, "signature": signature,
|
|
180
|
+
"related_engram_ids": sorted(set(related))}
|
|
181
|
+
fingerprint = hashlib.sha256(
|
|
182
|
+
_canonical(fingerprint_value).encode("utf-8")).hexdigest()
|
|
183
|
+
existing = self.db.execute(
|
|
184
|
+
"SELECT revision,fingerprint FROM associative_engrams WHERE engram_id=?",
|
|
185
|
+
(normalized["engram_id"],)).fetchone()
|
|
186
|
+
if existing:
|
|
187
|
+
if existing["fingerprint"] != fingerprint:
|
|
188
|
+
raise ValueError("ENGRAM_ID_COLLISION")
|
|
189
|
+
return {"engram_id": normalized["engram_id"],
|
|
190
|
+
"revision": existing["revision"], "disposition": "ALREADY_INDEXED"}
|
|
191
|
+
|
|
192
|
+
placeholders = ",".join("?" for _ in signature)
|
|
193
|
+
candidates = []
|
|
194
|
+
if max_automatic_links:
|
|
195
|
+
candidates = self.db.execute(f"""SELECT e.engram_id,COUNT(*) AS overlap
|
|
196
|
+
FROM associative_terms t JOIN associative_engrams e
|
|
197
|
+
ON e.engram_id=t.engram_id
|
|
198
|
+
WHERE t.term IN ({placeholders}) AND e.scope=?
|
|
199
|
+
GROUP BY e.engram_id ORDER BY overlap DESC,e.revision DESC LIMIT ?""",
|
|
200
|
+
[*signature, normalized["scope"], max_automatic_links]).fetchall()
|
|
201
|
+
previous = self.db.execute("""SELECT engram_id FROM associative_engrams
|
|
202
|
+
WHERE scope=? ORDER BY happened_at DESC,revision DESC LIMIT 1""",
|
|
203
|
+
(normalized["scope"],)).fetchone()
|
|
204
|
+
now = datetime.now().isoformat(timespec="microseconds")
|
|
205
|
+
with self.db:
|
|
206
|
+
cursor = self.db.execute("""INSERT INTO associative_engrams(
|
|
207
|
+
engram_id,source_kind,source_id,scope,happened_at,strength,signature_json,
|
|
208
|
+
fingerprint) VALUES(?,?,?,?,?,?,?,?)""", (
|
|
209
|
+
normalized["engram_id"], normalized["source_kind"],
|
|
210
|
+
normalized["source_id"], normalized["scope"], normalized["happened_at"],
|
|
211
|
+
normalized["strength"], _canonical(signature), fingerprint))
|
|
212
|
+
self.db.execute("INSERT INTO associative_engrams_fts(engram_id,text) VALUES(?,?)",
|
|
213
|
+
(normalized["engram_id"], normalized["text"]))
|
|
214
|
+
self.db.executemany("INSERT INTO associative_terms(term,engram_id) VALUES(?,?)",
|
|
215
|
+
[(term, normalized["engram_id"]) for term in signature])
|
|
216
|
+
for candidate in candidates:
|
|
217
|
+
weight = min(0.75, 0.2 + 0.1 * candidate["overlap"])
|
|
218
|
+
self._link(normalized["engram_id"], candidate["engram_id"],
|
|
219
|
+
"SHARED_CUE", weight, now)
|
|
220
|
+
self._link(candidate["engram_id"], normalized["engram_id"],
|
|
221
|
+
"SHARED_CUE", weight, now)
|
|
222
|
+
if previous:
|
|
223
|
+
self._link(normalized["engram_id"], previous["engram_id"],
|
|
224
|
+
"TEMPORAL_ADJACENT", 0.3, now)
|
|
225
|
+
self._link(previous["engram_id"], normalized["engram_id"],
|
|
226
|
+
"TEMPORAL_ADJACENT", 0.3, now)
|
|
227
|
+
for target in sorted(set(related)):
|
|
228
|
+
if self.db.execute("SELECT 1 FROM associative_engrams WHERE engram_id=?",
|
|
229
|
+
(target,)).fetchone():
|
|
230
|
+
self._link(normalized["engram_id"], target, "EXPLICIT", 1.0, now)
|
|
231
|
+
self._link(target, normalized["engram_id"], "EXPLICIT", 1.0, now)
|
|
232
|
+
else:
|
|
233
|
+
self.db.execute("""INSERT OR IGNORE INTO associative_pending_edges(
|
|
234
|
+
source_engram_id,target_engram_id,relation,weight,reinforced_at)
|
|
235
|
+
VALUES(?,?,?,?,?)""", (normalized["engram_id"], target,
|
|
236
|
+
"EXPLICIT", 1.0, now))
|
|
237
|
+
self._resolve_pending()
|
|
238
|
+
return {"engram_id": normalized["engram_id"], "revision": cursor.lastrowid,
|
|
239
|
+
"disposition": "INDEXED", "automatic_links": len(candidates),
|
|
240
|
+
"temporal_link": bool(previous)}
|
|
241
|
+
|
|
242
|
+
def _table_exists(self, table):
|
|
243
|
+
return bool(self.db.execute("SELECT 1 FROM sqlite_master WHERE name=?",
|
|
244
|
+
(table,)).fetchone())
|
|
245
|
+
|
|
246
|
+
def sync_sources(self, *, limit=None):
|
|
247
|
+
"""Incrementally index curated memories, conversations and development events."""
|
|
248
|
+
if limit is not None and (type(limit) is not int or not 1 <= limit <= 100000):
|
|
249
|
+
raise ValueError("INVALID_ENGRAM_SYNC_LIMIT")
|
|
250
|
+
remaining = limit
|
|
251
|
+
counts = {"LOCAL_MEMORY": 0, "CONVERSATION": 0, "DEVELOPMENT_EVENT": 0}
|
|
252
|
+
|
|
253
|
+
def allowed():
|
|
254
|
+
return remaining is None or sum(counts.values()) < remaining
|
|
255
|
+
|
|
256
|
+
if self._table_exists("local_brain_memories"):
|
|
257
|
+
rows = self.db.execute("""SELECT m.*,p.importance,p.related_memory_ids_json
|
|
258
|
+
FROM local_brain_memories m LEFT JOIN local_brain_memory_profiles p
|
|
259
|
+
ON p.memory_id=m.memory_id
|
|
260
|
+
WHERE NOT EXISTS(SELECT 1 FROM associative_engrams a
|
|
261
|
+
WHERE a.engram_id='memory:' || m.memory_id)
|
|
262
|
+
ORDER BY m.revision""").fetchall()
|
|
263
|
+
for row in rows:
|
|
264
|
+
if not allowed():
|
|
265
|
+
break
|
|
266
|
+
related = json.loads(row["related_memory_ids_json"]) \
|
|
267
|
+
if row["related_memory_ids_json"] else []
|
|
268
|
+
related = [self._source_to_engram(item) for item in related]
|
|
269
|
+
result = self.add_engram(engram_id="memory:" + row["memory_id"],
|
|
270
|
+
source_kind="LOCAL_MEMORY", source_id=row["memory_id"],
|
|
271
|
+
scope=row["scope"], happened_at=row["effective_at"],
|
|
272
|
+
text=row["title"] + "\n" + row["content"],
|
|
273
|
+
cues=json.loads(row["tags_json"]) + json.loads(row["entities_json"])
|
|
274
|
+
+ [row["memory_type"]],
|
|
275
|
+
strength=float(row["importance"] or 0.5),
|
|
276
|
+
related_engram_ids=related)
|
|
277
|
+
if result["disposition"] == "INDEXED":
|
|
278
|
+
counts["LOCAL_MEMORY"] += 1
|
|
279
|
+
if allowed() and self._table_exists("conversation_turns"):
|
|
280
|
+
rows = self.db.execute("""SELECT t.* FROM conversation_turns t
|
|
281
|
+
WHERE NOT EXISTS(SELECT 1 FROM associative_engrams a
|
|
282
|
+
WHERE a.engram_id='conversation:' || t.turn_id)
|
|
283
|
+
ORDER BY t.revision""").fetchall()
|
|
284
|
+
for row in rows:
|
|
285
|
+
if not allowed():
|
|
286
|
+
break
|
|
287
|
+
result = self.add_engram(engram_id="conversation:" + row["turn_id"],
|
|
288
|
+
source_kind="CONVERSATION", source_id=row["turn_id"],
|
|
289
|
+
scope=row["scope"], happened_at=row["happened_at"],
|
|
290
|
+
text=row["content_redacted"],
|
|
291
|
+
cues=[row["client"], row["role"], row["client_session_id"]],
|
|
292
|
+
strength=0.4)
|
|
293
|
+
if result["disposition"] == "INDEXED":
|
|
294
|
+
counts["CONVERSATION"] += 1
|
|
295
|
+
if allowed() and self._table_exists("development_events"):
|
|
296
|
+
rows = self.db.execute("""SELECT d.* FROM development_events d
|
|
297
|
+
WHERE NOT EXISTS(SELECT 1 FROM associative_engrams a
|
|
298
|
+
WHERE a.engram_id='development:' || d.event_id)
|
|
299
|
+
ORDER BY d.revision""").fetchall()
|
|
300
|
+
for row in rows:
|
|
301
|
+
if not allowed():
|
|
302
|
+
break
|
|
303
|
+
result = self.add_engram(engram_id="development:" + row["event_id"],
|
|
304
|
+
source_kind="DEVELOPMENT_EVENT", source_id=row["event_id"],
|
|
305
|
+
scope=row["project_id"], happened_at=row["effective_at"],
|
|
306
|
+
text=row["event_type"] + " " + row["subject_id"] + " "
|
|
307
|
+
+ row["payload_json"],
|
|
308
|
+
cues=[row["event_type"], row["subject_id"], row["session_id"]],
|
|
309
|
+
strength=0.65)
|
|
310
|
+
if result["disposition"] == "INDEXED":
|
|
311
|
+
counts["DEVELOPMENT_EVENT"] += 1
|
|
312
|
+
return {"status": "ENGRAMS_INDEXED" if sum(counts.values()) else "NO_NEW_ENGRAMS",
|
|
313
|
+
"indexed": counts, "indexed_total": sum(counts.values())}
|
|
314
|
+
|
|
315
|
+
@staticmethod
|
|
316
|
+
def _source_to_engram(source_id):
|
|
317
|
+
if source_id.startswith("conversation-turn:"):
|
|
318
|
+
return "conversation:" + source_id.removeprefix("conversation-turn:")
|
|
319
|
+
if source_id.startswith(("memory:", "conversation:", "development:")):
|
|
320
|
+
return source_id
|
|
321
|
+
return "memory:" + source_id
|
|
322
|
+
|
|
323
|
+
def lexical_search(self, query, *, scopes=None, limit=8):
|
|
324
|
+
"""Return direct FTS matches without graph propagation."""
|
|
325
|
+
query = _text(query, "LEXICAL_QUERY_REQUIRED")
|
|
326
|
+
scope_list = scopes or []
|
|
327
|
+
if not isinstance(scope_list, list) or not all(isinstance(x, str) and x.strip()
|
|
328
|
+
for x in scope_list):
|
|
329
|
+
raise ValueError("INVALID_LEXICAL_SCOPES")
|
|
330
|
+
if type(limit) is not int or not 1 <= limit <= 50:
|
|
331
|
+
raise ValueError("INVALID_LEXICAL_RESULT_LIMIT")
|
|
332
|
+
terms = _terms(query)
|
|
333
|
+
if not terms:
|
|
334
|
+
return {"status": "UNKNOWN", "reason": "NO_SEARCHABLE_QUERY_TERMS",
|
|
335
|
+
"results": []}
|
|
336
|
+
expression = " OR ".join('"' + term.replace('"', '""') + '"'
|
|
337
|
+
for term in terms)
|
|
338
|
+
where, parameters = ["associative_engrams_fts MATCH ?"], [expression]
|
|
339
|
+
if scope_list:
|
|
340
|
+
where.append("e.scope IN (" + ",".join("?" for _ in scope_list) + ")")
|
|
341
|
+
parameters.extend(scope_list)
|
|
342
|
+
parameters.append(limit)
|
|
343
|
+
rows = self.db.execute("""SELECT e.*,associative_engrams_fts.text,
|
|
344
|
+
bm25(associative_engrams_fts) AS rank
|
|
345
|
+
FROM associative_engrams_fts JOIN associative_engrams e
|
|
346
|
+
ON e.engram_id=associative_engrams_fts.engram_id WHERE """
|
|
347
|
+
+ " AND ".join(where) + " ORDER BY rank,e.revision DESC LIMIT ?",
|
|
348
|
+
parameters).fetchall()
|
|
349
|
+
return {"status": "KNOWN" if rows else "UNKNOWN",
|
|
350
|
+
"reason": "LEXICAL_MATCHES_FOUND" if rows else "NO_LEXICAL_MATCH",
|
|
351
|
+
"results": [{"engram_id": row["engram_id"],
|
|
352
|
+
"source_kind": row["source_kind"], "source_id": row["source_id"],
|
|
353
|
+
"scope": row["scope"], "rank": row["rank"],
|
|
354
|
+
"text_excerpt": row["text"][:500]} for row in rows]}
|
|
355
|
+
|
|
356
|
+
def activate(self, query, *, scopes=None, concept_cues=None, seed_limit=12, hops=2,
|
|
357
|
+
fanout=12, max_nodes=40, limit=8, minimum_concept_coverage=0.0,
|
|
358
|
+
coverage_mode="single"):
|
|
359
|
+
query = _text(query, "ASSOCIATIVE_QUERY_REQUIRED")
|
|
360
|
+
cue_list = concept_cues or []
|
|
361
|
+
if not isinstance(cue_list, list) or not all(isinstance(x, str) and x.strip()
|
|
362
|
+
for x in cue_list):
|
|
363
|
+
raise ValueError("INVALID_ASSOCIATIVE_CONCEPT_CUES")
|
|
364
|
+
scope_list = scopes or []
|
|
365
|
+
if not isinstance(scope_list, list) or not all(isinstance(x, str) and x.strip()
|
|
366
|
+
for x in scope_list):
|
|
367
|
+
raise ValueError("INVALID_ASSOCIATIVE_SCOPES")
|
|
368
|
+
bounds = ((seed_limit, 1, 50, "INVALID_SEED_LIMIT"),
|
|
369
|
+
(hops, 0, 4, "INVALID_ASSOCIATIVE_HOPS"),
|
|
370
|
+
(fanout, 1, 30, "INVALID_ASSOCIATIVE_FANOUT"),
|
|
371
|
+
(max_nodes, 1, 100, "INVALID_ACTIVE_NODE_LIMIT"),
|
|
372
|
+
(limit, 1, 30, "INVALID_ASSOCIATIVE_RESULT_LIMIT"))
|
|
373
|
+
for value, lower, upper, error in bounds:
|
|
374
|
+
if type(value) is not int or not lower <= value <= upper:
|
|
375
|
+
raise ValueError(error)
|
|
376
|
+
minimum_concept_coverage = _unit(
|
|
377
|
+
minimum_concept_coverage, "INVALID_MINIMUM_CONCEPT_COVERAGE")
|
|
378
|
+
if coverage_mode not in {"single", "union"}:
|
|
379
|
+
raise ValueError("INVALID_COVERAGE_MODE")
|
|
380
|
+
terms = _terms(" ".join([query, *cue_list]))
|
|
381
|
+
concept_terms = set(_terms(" ".join(cue_list)))
|
|
382
|
+
if not terms:
|
|
383
|
+
return {"status": "UNKNOWN", "reason": "NO_SEARCHABLE_QUERY_TERMS",
|
|
384
|
+
"results": [], "visited_nodes": 0, "traversed_edges": 0}
|
|
385
|
+
expression = " OR ".join('"' + term.replace('"', '""') + '"'
|
|
386
|
+
for term in terms)
|
|
387
|
+
where, parameters = ["associative_engrams_fts MATCH ?"], [expression]
|
|
388
|
+
if scope_list:
|
|
389
|
+
where.append("e.scope IN (" + ",".join("?" for _ in scope_list) + ")")
|
|
390
|
+
parameters.extend(scope_list)
|
|
391
|
+
parameters.append(seed_limit)
|
|
392
|
+
seeds = self.db.execute("""SELECT e.*,associative_engrams_fts.text,
|
|
393
|
+
bm25(associative_engrams_fts) AS rank
|
|
394
|
+
FROM associative_engrams_fts JOIN associative_engrams e
|
|
395
|
+
ON e.engram_id=associative_engrams_fts.engram_id WHERE """ + " AND ".join(where)
|
|
396
|
+
+ " ORDER BY rank,e.revision DESC LIMIT ?", parameters).fetchall()
|
|
397
|
+
if not seeds:
|
|
398
|
+
return {"status": "UNKNOWN", "reason": "NO_ACTIVATION_SEED",
|
|
399
|
+
"results": [], "visited_nodes": 0, "traversed_edges": 0}
|
|
400
|
+
activations, parents, rows = {}, {}, {}
|
|
401
|
+
query_set = set(terms)
|
|
402
|
+
maximum_query_coverage = 0.0
|
|
403
|
+
maximum_concept_coverage = 0.0
|
|
404
|
+
covered_concept_terms = set() # union of matched cue terms across all seeds
|
|
405
|
+
for row in seeds:
|
|
406
|
+
signature = set(json.loads(row["signature_json"]))
|
|
407
|
+
overlap = len(query_set & signature)
|
|
408
|
+
maximum_query_coverage = max(
|
|
409
|
+
maximum_query_coverage, overlap / max(1, len(query_set)))
|
|
410
|
+
if concept_terms:
|
|
411
|
+
maximum_concept_coverage = max(maximum_concept_coverage,
|
|
412
|
+
len(concept_terms & signature) / len(concept_terms))
|
|
413
|
+
covered_concept_terms |= (concept_terms & signature)
|
|
414
|
+
lexical = overlap / math.sqrt(max(1, len(query_set) * len(signature)))
|
|
415
|
+
activation = min(1.0, 0.55 + lexical)
|
|
416
|
+
activations[row["engram_id"]] = activation
|
|
417
|
+
parents[row["engram_id"]] = None
|
|
418
|
+
rows[row["engram_id"]] = row
|
|
419
|
+
union_concept_coverage = (len(covered_concept_terms) / len(concept_terms)
|
|
420
|
+
if concept_terms else 0.0)
|
|
421
|
+
gate_coverage = (union_concept_coverage if coverage_mode == "union"
|
|
422
|
+
else maximum_concept_coverage)
|
|
423
|
+
if (concept_terms and gate_coverage < minimum_concept_coverage):
|
|
424
|
+
return {"status": "UNKNOWN", "reason": "INSUFFICIENT_CONCEPT_COVERAGE",
|
|
425
|
+
"index_version": INDEX_VERSION, "seed_count": len(seeds),
|
|
426
|
+
"concept_cues_used": cue_list, "coverage_mode": coverage_mode,
|
|
427
|
+
"maximum_query_coverage": round(maximum_query_coverage, 6),
|
|
428
|
+
"maximum_concept_coverage": round(maximum_concept_coverage, 6),
|
|
429
|
+
"union_concept_coverage": round(union_concept_coverage, 6),
|
|
430
|
+
"minimum_concept_coverage": minimum_concept_coverage,
|
|
431
|
+
"visited_nodes": len(seeds), "traversed_edges": 0,
|
|
432
|
+
"results": []}
|
|
433
|
+
frontier = set(activations)
|
|
434
|
+
traversed = 0
|
|
435
|
+
allowed_scopes = set(scope_list)
|
|
436
|
+
for hop in range(1, hops + 1):
|
|
437
|
+
contributions = {}
|
|
438
|
+
sources = sorted(frontier, key=lambda item: activations[item], reverse=True)
|
|
439
|
+
placeholders = ",".join("?" for _ in sources)
|
|
440
|
+
edges = self.db.execute(f"""SELECT edge.source_engram_id,
|
|
441
|
+
edge.target_engram_id,edge.relation,edge.weight,target.*
|
|
442
|
+
FROM associative_edges edge JOIN associative_engrams target
|
|
443
|
+
ON target.engram_id=edge.target_engram_id
|
|
444
|
+
WHERE edge.source_engram_id IN ({placeholders})
|
|
445
|
+
ORDER BY edge.source_engram_id,edge.weight DESC""", sources).fetchall()
|
|
446
|
+
per_source = {}
|
|
447
|
+
for edge in edges:
|
|
448
|
+
source = edge["source_engram_id"]
|
|
449
|
+
per_source[source] = per_source.get(source, 0) + 1
|
|
450
|
+
if per_source[source] > fanout:
|
|
451
|
+
continue
|
|
452
|
+
if allowed_scopes and edge["scope"] not in allowed_scopes:
|
|
453
|
+
continue
|
|
454
|
+
traversed += 1
|
|
455
|
+
contribution = activations[source] * edge["weight"] * (0.65 ** hop)
|
|
456
|
+
old = contributions.get(edge["target_engram_id"], 0.0)
|
|
457
|
+
combined = 1.0 - (1.0 - old) * (1.0 - contribution)
|
|
458
|
+
contributions[edge["target_engram_id"]] = combined
|
|
459
|
+
rows[edge["target_engram_id"]] = edge
|
|
460
|
+
if (edge["target_engram_id"] not in parents
|
|
461
|
+
or contribution > activations.get(edge["target_engram_id"], 0.0)):
|
|
462
|
+
parents[edge["target_engram_id"]] = {
|
|
463
|
+
"from": source, "relation": edge["relation"],
|
|
464
|
+
"weight": edge["weight"]}
|
|
465
|
+
for target, value in contributions.items():
|
|
466
|
+
activations[target] = max(activations.get(target, 0.0), value)
|
|
467
|
+
top = sorted(activations, key=activations.get, reverse=True)[:max_nodes]
|
|
468
|
+
activations = {key: activations[key] for key in top}
|
|
469
|
+
frontier = set(contributions) & set(top)
|
|
470
|
+
if not frontier:
|
|
471
|
+
break
|
|
472
|
+
ranked = sorted(activations,
|
|
473
|
+
key=lambda key: activations[key] * (0.5 + rows[key]["strength"]),
|
|
474
|
+
reverse=True)[:limit]
|
|
475
|
+
text_placeholders = ",".join("?" for _ in ranked)
|
|
476
|
+
text_rows = self.db.execute(f"""SELECT engram_id,text
|
|
477
|
+
FROM associative_engrams_fts WHERE engram_id IN ({text_placeholders})""",
|
|
478
|
+
ranked).fetchall()
|
|
479
|
+
excerpts = {row["engram_id"]: row["text"][:500] for row in text_rows}
|
|
480
|
+
results = []
|
|
481
|
+
for engram_id in ranked:
|
|
482
|
+
row = rows[engram_id]
|
|
483
|
+
path, cursor, seen = [], engram_id, set()
|
|
484
|
+
while parents.get(cursor) and cursor not in seen:
|
|
485
|
+
seen.add(cursor)
|
|
486
|
+
step = parents[cursor]
|
|
487
|
+
path.append({"from": step["from"], "to": cursor,
|
|
488
|
+
"relation": step["relation"], "weight": step["weight"]})
|
|
489
|
+
cursor = step["from"]
|
|
490
|
+
path.reverse()
|
|
491
|
+
results.append({"engram_id": engram_id, "source_kind": row["source_kind"],
|
|
492
|
+
"source_id": row["source_id"], "scope": row["scope"],
|
|
493
|
+
"happened_at": row["happened_at"],
|
|
494
|
+
"activation": round(activations[engram_id], 6),
|
|
495
|
+
"strength": row["strength"],
|
|
496
|
+
"text_excerpt": excerpts.get(engram_id, ""),
|
|
497
|
+
"activation_path": path})
|
|
498
|
+
return {"status": "KNOWN", "reason": "ASSOCIATIVE_ACTIVATION_COMPLETED",
|
|
499
|
+
"index_version": INDEX_VERSION, "seed_count": len(seeds),
|
|
500
|
+
"concept_cues_used": cue_list, "coverage_mode": coverage_mode,
|
|
501
|
+
"maximum_query_coverage": round(maximum_query_coverage, 6),
|
|
502
|
+
"maximum_concept_coverage": round(maximum_concept_coverage, 6),
|
|
503
|
+
"union_concept_coverage": round(union_concept_coverage, 6),
|
|
504
|
+
"minimum_concept_coverage": minimum_concept_coverage,
|
|
505
|
+
"visited_nodes": len(activations), "traversed_edges": traversed,
|
|
506
|
+
"bounds": {"hops": hops, "fanout": fanout, "max_nodes": max_nodes},
|
|
507
|
+
"results": results}
|