graph-engine-memory 0.6.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- graph_engine/__init__.py +1 -0
- graph_engine/__main__.py +1275 -0
- graph_engine/access.py +202 -0
- graph_engine/agent_memory.py +103 -0
- graph_engine/brain.py +731 -0
- graph_engine/brain_engine.py +559 -0
- graph_engine/communities.py +503 -0
- graph_engine/consolidation.py +191 -0
- graph_engine/context.py +203 -0
- graph_engine/decay.py +65 -0
- graph_engine/demo.py +165 -0
- graph_engine/dream.py +408 -0
- graph_engine/embedder.py +84 -0
- graph_engine/engine.py +58 -0
- graph_engine/entities.py +376 -0
- graph_engine/evals.py +1206 -0
- graph_engine/feedback.py +155 -0
- graph_engine/gaps.py +209 -0
- graph_engine/graph.py +133 -0
- graph_engine/hygiene.py +213 -0
- graph_engine/inference.py +251 -0
- graph_engine/intent.py +207 -0
- graph_engine/mcp/__init__.py +1 -0
- graph_engine/mcp/agent/instructions.md +34 -0
- graph_engine/mcp/format.py +66 -0
- graph_engine/mcp/server.py +407 -0
- graph_engine/merge.py +220 -0
- graph_engine/model.py +56 -0
- graph_engine/namespaces.py +74 -0
- graph_engine/recall.py +161 -0
- graph_engine/report.py +401 -0
- graph_engine/reranker.py +86 -0
- graph_engine/retrieval.py +296 -0
- graph_engine/review.py +95 -0
- graph_engine/runtime.py +83 -0
- graph_engine/server.py +239 -0
- graph_engine/similarity.py +42 -0
- graph_engine/store.py +63 -0
- graph_engine/suggester.py +63 -0
- graph_engine/temporal.py +216 -0
- graph_engine/tiers.py +135 -0
- graph_engine/web/app.js +578 -0
- graph_engine/web/index.html +169 -0
- graph_engine/web/report.html +39 -0
- graph_engine/web/report.js +50 -0
- graph_engine/web/review.html +104 -0
- graph_engine/web/review.js +211 -0
- graph_engine_memory-0.6.1.dist-info/METADATA +125 -0
- graph_engine_memory-0.6.1.dist-info/RECORD +53 -0
- graph_engine_memory-0.6.1.dist-info/WHEEL +5 -0
- graph_engine_memory-0.6.1.dist-info/entry_points.txt +3 -0
- graph_engine_memory-0.6.1.dist-info/licenses/LICENSE +21 -0
- graph_engine_memory-0.6.1.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,559 @@
|
|
|
1
|
+
"""Growth loop on brain basis: ingest → embed → suggest → commit.
|
|
2
|
+
|
|
3
|
+
Every ingest is a git commit in the private repo — the graph grows as a
|
|
4
|
+
visible commit history.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import os
|
|
10
|
+
import sys
|
|
11
|
+
import threading
|
|
12
|
+
from typing import TYPE_CHECKING
|
|
13
|
+
|
|
14
|
+
if TYPE_CHECKING:
|
|
15
|
+
from .consolidation import ConsolidationConfig
|
|
16
|
+
|
|
17
|
+
from .brain import Brain, Node, Edge
|
|
18
|
+
from .embedder import get_embedder
|
|
19
|
+
from .similarity import cosine
|
|
20
|
+
from .suggester import suggest, is_auto_accept
|
|
21
|
+
from .intent import detect_intent
|
|
22
|
+
from .reranker import get_reranker
|
|
23
|
+
from .review import intent_auto_accept_max
|
|
24
|
+
|
|
25
|
+
DEDUPE_THRESHOLD = 0.92
|
|
26
|
+
# Intent edges may only fire for pairs that are genuinely topically
|
|
27
|
+
# related (same threshold as "extends"). Without this guard, a marker
|
|
28
|
+
# word in the new text ("ersetzt", "supersedes") would subordinate a node
|
|
29
|
+
# to EVERY existing node as intent — in a topically homogeneous brain
|
|
30
|
+
# (shared domain vocabulary) even to almost all of them.
|
|
31
|
+
INTENT_SIM_THRESHOLD = 0.45
|
|
32
|
+
AUTO_ACCEPT_ENV = "IDEAGRAPH_AUTO_ACCEPT" # "1"/"true" → edges accepted without HITL
|
|
33
|
+
# "1"/"true" -> intent edges (supersedes/continues/contradicts) stay
|
|
34
|
+
# pending (HITL review) instead of auto-accepted. Lets the user decide
|
|
35
|
+
# whether automatically detected intents go straight into the graph.
|
|
36
|
+
INTENT_PENDING_ENV = "IDEAGRAPH_INTENT_PENDING"
|
|
37
|
+
|
|
38
|
+
# Serializes all brain write operations per process. The store is a series
|
|
39
|
+
# of whole-file rewrites (read-all → mutate → write-all); without a lock,
|
|
40
|
+
# parallel ingests/resolves interleave and last-writer-wins swallows the
|
|
41
|
+
# other's writes (Audit #1/#15). One lock suffices because all mutations
|
|
42
|
+
# run through the same process (server, CLI, pipeline tools).
|
|
43
|
+
BRAIN_LOCK = threading.RLock()
|
|
44
|
+
|
|
45
|
+
CONSOLIDATE_THRESHOLD_ENV = "IG_CONSOLIDATE_THRESHOLD"
|
|
46
|
+
CONSOLIDATE_MIN_AGE_DAYS_ENV = "IG_CONSOLIDATE_MIN_AGE_DAYS"
|
|
47
|
+
CONSOLIDATE_MIN_COUNT_ENV = "IG_CONSOLIDATE_MIN_COUNT"
|
|
48
|
+
CONSOLIDATE_LIMIT_ENV = "IG_CONSOLIDATE_LIMIT"
|
|
49
|
+
CONSOLIDATE_LLM_CMD_ENV = "IG_CONSOLIDATE_LLM_CMD"
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
GRAPH_HOME_ENV = "IG_GRAPH_HOME"
|
|
53
|
+
GRAPH_ENV = "IG_GRAPH"
|
|
54
|
+
ACCESS_POLICY_ENV = "IG_ACCESS_POLICY"
|
|
55
|
+
PRINCIPAL_ENV = "IG_PRINCIPAL"
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def graph_access_config_from_env() -> dict[str, str]:
|
|
59
|
+
"""Trusted local process configuration for namespaces and access control."""
|
|
60
|
+
return {key: os.environ.get(key, "") for key in
|
|
61
|
+
(GRAPH_HOME_ENV, GRAPH_ENV, ACCESS_POLICY_ENV, PRINCIPAL_ENV)}
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def consolidation_config_from_env(overrides: dict | None = None) -> "ConsolidationConfig":
|
|
65
|
+
"""Read gates at call time; explicit CLI/API values override environment."""
|
|
66
|
+
from .consolidation import ConsolidationConfig
|
|
67
|
+
overrides = overrides or {}
|
|
68
|
+
values = {}
|
|
69
|
+
for key, env_name, cast, default in (
|
|
70
|
+
("threshold", CONSOLIDATE_THRESHOLD_ENV, int, 1),
|
|
71
|
+
("min_age_days", CONSOLIDATE_MIN_AGE_DAYS_ENV, float, 1.0),
|
|
72
|
+
("min_count", CONSOLIDATE_MIN_COUNT_ENV, int, 1),
|
|
73
|
+
("limit", CONSOLIDATE_LIMIT_ENV, int, 50),
|
|
74
|
+
):
|
|
75
|
+
raw = overrides.get(key)
|
|
76
|
+
if raw is None:
|
|
77
|
+
raw = os.environ.get(env_name, default)
|
|
78
|
+
try:
|
|
79
|
+
values[key] = cast(raw)
|
|
80
|
+
except (ValueError, TypeError, OverflowError) as exc:
|
|
81
|
+
raise ValueError(f"{env_name}: invalid {key}: {raw!r}") from exc
|
|
82
|
+
return ConsolidationConfig(**values)
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def auto_accept_from_env() -> bool:
|
|
86
|
+
return os.environ.get(AUTO_ACCEPT_ENV, "").lower() in ("1", "true", "yes")
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def intent_pending_from_env() -> bool:
|
|
90
|
+
return os.environ.get(INTENT_PENDING_ENV, "").lower() in ("1", "true", "yes")
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def _normalize(text: str) -> str:
|
|
94
|
+
"""For the duplicate comparison: lowercase, whitespace flattened."""
|
|
95
|
+
return " ".join(text.lower().split())
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
class BrainEngine:
|
|
99
|
+
def __init__(self, brain: Brain, embedder=None, dedupe_threshold: float = DEDUPE_THRESHOLD,
|
|
100
|
+
reranker=None):
|
|
101
|
+
self.brain = brain
|
|
102
|
+
self.embedder = embedder if embedder is not None else get_embedder()
|
|
103
|
+
self.dedupe_threshold = dedupe_threshold
|
|
104
|
+
# V2#1: optional cross-encoder rerank pass; None/default → no rerank.
|
|
105
|
+
self.reranker = reranker if reranker is not None else get_reranker()
|
|
106
|
+
|
|
107
|
+
def _find_duplicate(self, vec: list[float], exclude_id: str | None = None) -> Node | None:
|
|
108
|
+
"""Closest node above the dedupe threshold — or None. Uses the vector cache."""
|
|
109
|
+
node_ids = {n.id for n in self.brain.read_nodes()
|
|
110
|
+
if n.id != exclude_id and n.ntype != "entity" and n.status != "tombstone"
|
|
111
|
+
and self.brain.can_access("write", n.id)}
|
|
112
|
+
vectors = self.brain.vectors_for(
|
|
113
|
+
node_ids,
|
|
114
|
+
lambda t: self.embedder.embed(_normalize(t)),
|
|
115
|
+
batch_fn=lambda ts: self.embedder.embed_batch([_normalize(t) for t in ts]),
|
|
116
|
+
)
|
|
117
|
+
best: tuple[float, str] | None = None
|
|
118
|
+
for nid, v in vectors.items():
|
|
119
|
+
if len(v) != len(vec):
|
|
120
|
+
continue # Audit #8: foreign dimension = not comparable, skip
|
|
121
|
+
sim = cosine(vec, v)
|
|
122
|
+
if sim >= self.dedupe_threshold and (best is None or sim > best[0]):
|
|
123
|
+
best = (sim, nid)
|
|
124
|
+
if best is None:
|
|
125
|
+
return None
|
|
126
|
+
return next((n for n in self.brain.read_nodes() if n.id == best[1]), None)
|
|
127
|
+
|
|
128
|
+
def consolidate(self, admit_required: bool = False) -> dict:
|
|
129
|
+
"""Dedup-based consolidation (V2#2): dual-buffer promotion.
|
|
130
|
+
|
|
131
|
+
Checks all probation nodes AGAINST EACH OTHER and against active
|
|
132
|
+
(closes the streaming blind spot: nodes ingested in the same commit
|
|
133
|
+
were never compared before). Near-duplicates are MERGED (dedupe,
|
|
134
|
+
never summarize) and the redundant probation node is tombstoned;
|
|
135
|
+
all others are promoted to active.
|
|
136
|
+
|
|
137
|
+
`admit_required` (V2#3 admit rule, opt-in): when True, a node only
|
|
138
|
+
enters the active graph if it has relations (active edges) —
|
|
139
|
+
otherwise it stays in probation. Default False = existing behavior
|
|
140
|
+
(promote all).
|
|
141
|
+
"""
|
|
142
|
+
with BRAIN_LOCK:
|
|
143
|
+
promoted, merged = 0, 0
|
|
144
|
+
for pn in self.brain.read_nodes():
|
|
145
|
+
if pn.status != "probation" or pn.ntype == "entity":
|
|
146
|
+
continue
|
|
147
|
+
pvec = self.brain.vectors_for(
|
|
148
|
+
{pn.id}, lambda t: self.embedder.embed(_normalize(t))
|
|
149
|
+
).get(pn.id)
|
|
150
|
+
dup = self._find_duplicate(pvec, exclude_id=pn.id) if pvec else None
|
|
151
|
+
if dup is not None and dup.status != "tombstone":
|
|
152
|
+
self.brain.merge_node(dup, source=pn.source)
|
|
153
|
+
self.brain.tombstone_node(pn.id)
|
|
154
|
+
merged += 1
|
|
155
|
+
continue
|
|
156
|
+
# V2#3 admit rule (opt-in): without relations (active edges) the
|
|
157
|
+
# node does not enter the active graph — it stays in probation.
|
|
158
|
+
if admit_required and not self._has_relation(pn.id):
|
|
159
|
+
continue
|
|
160
|
+
self.brain.promote_node(pn.id)
|
|
161
|
+
promoted += 1
|
|
162
|
+
if promoted or merged:
|
|
163
|
+
self.brain.rebuild_index()
|
|
164
|
+
return {"promoted": promoted, "merged": merged}
|
|
165
|
+
|
|
166
|
+
def _has_relation(self, node_id: str) -> bool:
|
|
167
|
+
"""V2#3 admit rule: does the node have an active edge (in/out)?"""
|
|
168
|
+
return any(e.source == node_id or e.target == node_id
|
|
169
|
+
for e in self.brain.read_edges() if e.is_current)
|
|
170
|
+
|
|
171
|
+
def demote_forgotten(self, level_fn) -> int:
|
|
172
|
+
"""Graceful degradation (V2#2): active nodes whose level_fn=='tombstone'
|
|
173
|
+
are tombstoned (never hard-deleted). level_fn(node)->str returns the
|
|
174
|
+
decay level (e.g. from decay.decay_level with retrieval counters)."""
|
|
175
|
+
with BRAIN_LOCK:
|
|
176
|
+
count = 0
|
|
177
|
+
for n in self.brain.read_nodes():
|
|
178
|
+
if n.status == "active" and level_fn(n) == "tombstone":
|
|
179
|
+
self.brain.tombstone_node(n.id)
|
|
180
|
+
count += 1
|
|
181
|
+
if count:
|
|
182
|
+
self.brain.rebuild_index()
|
|
183
|
+
return count
|
|
184
|
+
|
|
185
|
+
def ingest(self, text: str, source: str = "human", tags: list[str] | None = None,
|
|
186
|
+
allow_duplicates: bool = False, ntype: str = "semantic",
|
|
187
|
+
auto_accept: bool | None = None,
|
|
188
|
+
relations: list[tuple[str, str]] | None = None,
|
|
189
|
+
edge_origin: str = "manual",
|
|
190
|
+
env: dict[str, str] | None = None) -> tuple[Node, list[Edge], bool]:
|
|
191
|
+
"""Ingest with dedupe. Returns: (node, edges, is_duplicate).
|
|
192
|
+
|
|
193
|
+
On a near-duplicate (cosine >= threshold against the normalized text)
|
|
194
|
+
no new node is created; the existing one is merged instead: the
|
|
195
|
+
source is logged on the node, the commit message says "dup".
|
|
196
|
+
auto_accept (default: env IDEAGRAPH_AUTO_ACCEPT) accepts
|
|
197
|
+
Auto-accept edge suggestions instead of leaving them pending.
|
|
198
|
+
env (Tier-3): per-call env overrides for eval cases. The confidence floor
|
|
199
|
+
(IG_EDGE_CONF_FLOOR, default 0.0 = no filter) drops weak
|
|
200
|
+
Auto-edge suggestions — ROADMAP_CASE `roadmap-confidence-floor`.
|
|
201
|
+
"""
|
|
202
|
+
if auto_accept is None:
|
|
203
|
+
auto_accept = auto_accept_from_env()
|
|
204
|
+
intent_pending = intent_pending_from_env()
|
|
205
|
+
# Intent fan-out dam (ROADMAP_CASE `roadmap-intent-fanout-cap`): the
|
|
206
|
+
# marker heuristic auto-accepted every intent edge it produced, so one
|
|
207
|
+
# marker word could write dozens of false `contradicts` into the graph
|
|
208
|
+
# (live: 39 % of all intent edges ever created were invalidated again).
|
|
209
|
+
# At most `intent_auto_accept_max()` intent edges per source are
|
|
210
|
+
# auto-accepted; the rest are born PENDING (kept, reviewable, not
|
|
211
|
+
# dropped) — see graph_engine/review.py for the policy and its bounds.
|
|
212
|
+
intent_auto_left = intent_auto_accept_max(env)
|
|
213
|
+
text = text.strip()
|
|
214
|
+
if not text:
|
|
215
|
+
raise ValueError("Empty text cannot be ingested.")
|
|
216
|
+
# The complete RMW chain (dedupe → node → embed → edges → vectors →
|
|
217
|
+
# evolve → index → commit) runs under ONE lock: parallel ingests
|
|
218
|
+
# must not duplicate each other's nodes/edges or lose
|
|
219
|
+
# vector-cache writes (Audit #1/#15).
|
|
220
|
+
with BRAIN_LOCK:
|
|
221
|
+
self.brain.ensure_ready() # onboarding: creates a missing brain repo
|
|
222
|
+
self.brain.pull()
|
|
223
|
+
if relations:
|
|
224
|
+
for ref, _kind in relations:
|
|
225
|
+
target = next((n for n in self.brain.read_nodes()
|
|
226
|
+
if n.id == ref or n.text.strip().lower() == ref.strip().lower()), None)
|
|
227
|
+
if target is not None:
|
|
228
|
+
self.brain.authorize("write", target.id)
|
|
229
|
+
vec = self.embedder.embed(_normalize(text))
|
|
230
|
+
if not allow_duplicates:
|
|
231
|
+
dup = self._find_duplicate(vec)
|
|
232
|
+
if dup is not None:
|
|
233
|
+
self.brain.merge_node(dup, source=source)
|
|
234
|
+
self.brain.rebuild_index()
|
|
235
|
+
# Audit #31: a failing commit must not look like data loss —
|
|
236
|
+
# catch, heal the derived index, re-raise with guidance.
|
|
237
|
+
try:
|
|
238
|
+
self.brain.commit_and_push(
|
|
239
|
+
f"ingest dup of {dup.id[:8]}: {_normalize(text)[:50]}…")
|
|
240
|
+
except Exception:
|
|
241
|
+
self._heal_after_failed_commit()
|
|
242
|
+
raise
|
|
243
|
+
return dup, [], True
|
|
244
|
+
node = Node(text=text, source=source, tags=tags, ntype=ntype)
|
|
245
|
+
self.brain.write_node(node)
|
|
246
|
+
# Embedding cache: only new nodes get embedded, the rest comes from vectors.jsonl
|
|
247
|
+
others = {n.id for n in self.brain.read_nodes()
|
|
248
|
+
if n.id != node.id and n.ntype != "entity"
|
|
249
|
+
and self.brain.can_access("write", n.id)}
|
|
250
|
+
candidates = self.brain.vectors_for(
|
|
251
|
+
others,
|
|
252
|
+
lambda t: self.embedder.embed(_normalize(t)),
|
|
253
|
+
batch_fn=lambda ts: self.embedder.embed_batch([_normalize(t) for t in ts]),
|
|
254
|
+
)
|
|
255
|
+
# V2#3 intent edges + admit rule: the new node enters with its relations.
|
|
256
|
+
# Intent edges are pending=False (auto-accepted), so they must
|
|
257
|
+
# additionally prove real topical relatedness (ST cosine
|
|
258
|
+
# >= INTENT_SIM_THRESHOLD), otherwise a single marker word spams all nodes.
|
|
259
|
+
intent_edges: list[Edge] = []
|
|
260
|
+
# Sorted by target id: which edges fall inside the cap must not
|
|
261
|
+
# depend on the filesystem's node read order (determinism).
|
|
262
|
+
for ex in sorted(self.brain.read_nodes(), key=lambda n: n.id):
|
|
263
|
+
if (ex.id == node.id or ex.ntype == "entity"
|
|
264
|
+
or not self.brain.can_access("write", ex.id)):
|
|
265
|
+
continue
|
|
266
|
+
intent = detect_intent(node.text, ex.text)
|
|
267
|
+
if not intent:
|
|
268
|
+
continue
|
|
269
|
+
ex_vec = candidates.get(ex.id)
|
|
270
|
+
if ex_vec is None or cosine(vec, ex_vec) < INTENT_SIM_THRESHOLD:
|
|
271
|
+
continue
|
|
272
|
+
if intent_pending:
|
|
273
|
+
pending = True
|
|
274
|
+
elif intent_auto_left > 0:
|
|
275
|
+
pending = False
|
|
276
|
+
intent_auto_left -= 1
|
|
277
|
+
else:
|
|
278
|
+
pending = True
|
|
279
|
+
intent_edges.append(Edge(source=node.id, target=ex.id, kind=intent, pending=pending,
|
|
280
|
+
origin="intent"))
|
|
281
|
+
# Admit rule: explicitly declared relations (target_text|id, kind).
|
|
282
|
+
if relations:
|
|
283
|
+
for ref, kind in relations:
|
|
284
|
+
target = next((n for n in self.brain.read_nodes()
|
|
285
|
+
if n.id == ref or n.text.strip().lower() == ref.strip().lower()), None)
|
|
286
|
+
if target is not None and target.id != node.id:
|
|
287
|
+
intent_edges.append(Edge(source=node.id, target=target.id, kind=kind, pending=False,
|
|
288
|
+
origin=edge_origin))
|
|
289
|
+
# Similarity edges (V2#3): pending unless either the confidence band (>=0.95)
|
|
290
|
+
# or the env override (IDEAGRAPH_AUTO_ACCEPT) auto-accepts the edge.
|
|
291
|
+
# Tier-3 confidence floor (roadmap-confidence-floor): suggestions below the
|
|
292
|
+
# floor (per-call env IG_EDGE_CONF_FLOOR, default 0.0 = no filter) are
|
|
293
|
+
# dropped instead of landing pending — protects autonomous cycles from
|
|
294
|
+
# a flood of low-confidence edges.
|
|
295
|
+
# Tier-3 confidence floor: non-numeric values get an
|
|
296
|
+
# understandable error instead of a bare float() ValueError
|
|
297
|
+
# (Audit #20: "unexplained 500s").
|
|
298
|
+
_floor_raw = (env or {}).get("IG_EDGE_CONF_FLOOR",
|
|
299
|
+
os.environ.get("IG_EDGE_CONF_FLOOR", "0.0"))
|
|
300
|
+
try:
|
|
301
|
+
floor = float(_floor_raw)
|
|
302
|
+
except (TypeError, ValueError):
|
|
303
|
+
raise ValueError(
|
|
304
|
+
f"IG_EDGE_CONF_FLOOR must be a number, got: {_floor_raw!r}")
|
|
305
|
+
sim_edges = [Edge(source=s.source, target=s.target, kind=s.kind,
|
|
306
|
+
pending=not (is_auto_accept(s.confidence) or auto_accept),
|
|
307
|
+
confidence=s.confidence, origin="suggester")
|
|
308
|
+
for s in suggest(node.id, vec, candidates)
|
|
309
|
+
if s.confidence >= floor]
|
|
310
|
+
# Intent/admit-rule edges take precedence; similarity must not duplicate the same pair.
|
|
311
|
+
claimed = {(e.source, e.target) for e in intent_edges}
|
|
312
|
+
combined = list(intent_edges)
|
|
313
|
+
for e in sim_edges:
|
|
314
|
+
if (e.source, e.target) in claimed:
|
|
315
|
+
continue
|
|
316
|
+
claimed.add((e.source, e.target))
|
|
317
|
+
combined.append(e)
|
|
318
|
+
existing_pairs = {(e.source, e.target) for e in self.brain.read_edges()}
|
|
319
|
+
new_edges = [e for e in combined if (e.source, e.target) not in existing_pairs]
|
|
320
|
+
for e in new_edges:
|
|
321
|
+
self.brain.add_edge(e)
|
|
322
|
+
# Cache the new node's vector
|
|
323
|
+
cached = self.brain.read_vectors()
|
|
324
|
+
cached[node.id] = vec
|
|
325
|
+
self.brain.write_vectors(cached)
|
|
326
|
+
# Memory evolution (A-Mem lesson): strong new connection (similar,
|
|
327
|
+
# auto-accepted via confidence band OR env) → enrich related old
|
|
328
|
+
# nodes with a cross-reference.
|
|
329
|
+
# Audit #22: the rewrite must preserve the target node's status (and
|
|
330
|
+
# created) — previously it lost status → active fell back to
|
|
331
|
+
# probation (status erosion, verified live).
|
|
332
|
+
# Audit #19: the annotation grows unboundedly and drifts the node's
|
|
333
|
+
# own embedding → hard cap; further references go into the
|
|
334
|
+
# edges (which exist anyway), not into the text.
|
|
335
|
+
EVOLVED_ANNOTATION_CAP = 5
|
|
336
|
+
evolved = 0
|
|
337
|
+
for e in new_edges:
|
|
338
|
+
if not e.pending and e.kind == "similar":
|
|
339
|
+
target_node = next((n for n in self.brain.read_nodes()
|
|
340
|
+
if n.id == e.target), None)
|
|
341
|
+
if target_node is None or target_node.status == "tombstone" or target_node.ntype == "entity":
|
|
342
|
+
continue
|
|
343
|
+
existing = target_node.text.count("[evolved ")
|
|
344
|
+
if existing >= EVOLVED_ANNOTATION_CAP:
|
|
345
|
+
continue
|
|
346
|
+
ref = f"[evolved {self._now_short()}: connected to {node.id[:8]} “{_normalize(text)[:40]}…”]"
|
|
347
|
+
if "evolved" not in target_node.text or node.id[:8] not in target_node.text:
|
|
348
|
+
# Enrich the existing object: reconstructing a Node
|
|
349
|
+
# resets newer metadata (including pinned tiers).
|
|
350
|
+
target_node.text += "\n\n" + ref
|
|
351
|
+
self.brain.write_node(target_node)
|
|
352
|
+
evolved += 1
|
|
353
|
+
self.brain.rebuild_index()
|
|
354
|
+
suffix = f", {evolved} nodes evolved" if evolved else ""
|
|
355
|
+
try:
|
|
356
|
+
self.brain.commit_and_push(
|
|
357
|
+
f"ingest: {text[:50]}{'…' if len(text) > 50 else ''} (+{len(new_edges)} suggestions{suffix})")
|
|
358
|
+
except Exception:
|
|
359
|
+
# Audit #31: the mutation landed on disk but the commit/push
|
|
360
|
+
# failed. Heal the derived index and raise an actionable error
|
|
361
|
+
# instead of a bare CalledProcessError (no fake rollback — the
|
|
362
|
+
# next successful ingest commits whatever landed).
|
|
363
|
+
self._heal_after_failed_commit()
|
|
364
|
+
raise
|
|
365
|
+
return node, new_edges, False
|
|
366
|
+
|
|
367
|
+
def entities(self, text: str | None = None, **kwargs) -> dict:
|
|
368
|
+
"""Extract typed entities/facts and link them to an evidence node."""
|
|
369
|
+
from .entities import extract_entities
|
|
370
|
+
return extract_entities(self.brain, text, **kwargs)
|
|
371
|
+
|
|
372
|
+
def observe(self, text: str, source: str = "human", tags: list[str] | None = None,
|
|
373
|
+
observed_at: str | None = None, context: str | None = None,
|
|
374
|
+
allow_duplicates: bool = False) -> tuple[Node, bool]:
|
|
375
|
+
"""Store a raw episodic event (Phase 1: episodic layer).
|
|
376
|
+
|
|
377
|
+
Episodic nodes are NOT deduplicated — every observation is a distinct
|
|
378
|
+
event, even if it describes the same fact. They carry `observed_at`
|
|
379
|
+
(when the event happened) and `context` (where/how it was observed).
|
|
380
|
+
Returns (node, is_duplicate) — is_duplicate is always False unless
|
|
381
|
+
allow_duplicates=False and an identical text already exists.
|
|
382
|
+
"""
|
|
383
|
+
with BRAIN_LOCK:
|
|
384
|
+
self.brain.ensure_ready()
|
|
385
|
+
self.brain.pull()
|
|
386
|
+
text = text.strip()
|
|
387
|
+
if not text:
|
|
388
|
+
raise ValueError("Empty text cannot be observed.")
|
|
389
|
+
if not allow_duplicates:
|
|
390
|
+
# Exact-text dedupe for episodic (not cosine — events are unique)
|
|
391
|
+
existing = next(
|
|
392
|
+
(n for n in self.brain.read_nodes()
|
|
393
|
+
if n.ntype == "episodic" and n.text.strip() == text),
|
|
394
|
+
None)
|
|
395
|
+
if existing is not None:
|
|
396
|
+
return existing, True
|
|
397
|
+
node = Node(text=text, source=source, tags=tags, ntype="episodic",
|
|
398
|
+
status="active", observed_at=observed_at, context=context)
|
|
399
|
+
self.brain.write_node(node)
|
|
400
|
+
self.brain.rebuild_index()
|
|
401
|
+
self.brain.commit_and_push(
|
|
402
|
+
f"observe: {text[:50]}{'…' if len(text) > 50 else ''}")
|
|
403
|
+
return node, False
|
|
404
|
+
|
|
405
|
+
def extract(self, node_id: str, text: str | None = None,
|
|
406
|
+
source: str = "human", tags: list[str] | None = None) -> Node:
|
|
407
|
+
"""Extract a semantic fact from an episodic node (Phase 1).
|
|
408
|
+
|
|
409
|
+
Creates a new semantic node linked to the episodic source via an
|
|
410
|
+
`extends` edge. The original episodic node is NOT modified — the
|
|
411
|
+
extraction is a new, derived memory. If `text` is None, the episodic
|
|
412
|
+
node's text is used as the basis (the caller is expected to have
|
|
413
|
+
already refined it, e.g. via an LLM).
|
|
414
|
+
"""
|
|
415
|
+
with BRAIN_LOCK:
|
|
416
|
+
self.brain.ensure_ready()
|
|
417
|
+
self.brain.pull()
|
|
418
|
+
episodic = self.brain.read_node(node_id)
|
|
419
|
+
if episodic is None:
|
|
420
|
+
raise ValueError(f"No node with id {node_id!r}.")
|
|
421
|
+
if episodic.ntype != "episodic":
|
|
422
|
+
raise ValueError(
|
|
423
|
+
f"Node {node_id!r} is not episodic (type={episodic.ntype!r}).")
|
|
424
|
+
semantic_text = (text or episodic.text).strip()
|
|
425
|
+
if not semantic_text:
|
|
426
|
+
raise ValueError("Empty text cannot be extracted.")
|
|
427
|
+
node = Node(text=semantic_text, source=source, tags=tags,
|
|
428
|
+
ntype="semantic", status="probation")
|
|
429
|
+
self.brain.write_node(node)
|
|
430
|
+
edge = Edge(source=node.id, target=episodic.id, kind="extends",
|
|
431
|
+
pending=False, origin="consolidator")
|
|
432
|
+
self.brain.add_edge(edge)
|
|
433
|
+
self.brain.rebuild_index()
|
|
434
|
+
self.brain.commit_and_push(
|
|
435
|
+
f"extract: {node.id[:8]} from episodic {episodic.id[:8]}")
|
|
436
|
+
return node
|
|
437
|
+
|
|
438
|
+
def timeline(self, *, since: str | None = None, until: str | None = None,
|
|
439
|
+
limit: int = 50) -> list[Node]:
|
|
440
|
+
"""Query episodic nodes by time range (Phase 1: temporal reasoning).
|
|
441
|
+
|
|
442
|
+
Returns episodic nodes sorted by `observed_at` (newest first).
|
|
443
|
+
`since`/`until` are ISO-8601 date strings. Without arguments,
|
|
444
|
+
returns the most recent `limit` episodic nodes.
|
|
445
|
+
"""
|
|
446
|
+
nodes = [n for n in self.brain.read_nodes()
|
|
447
|
+
if n.ntype == "episodic" and n.status != "tombstone"]
|
|
448
|
+
if since:
|
|
449
|
+
nodes = [n for n in nodes
|
|
450
|
+
if (n.observed_at or n.created) >= since]
|
|
451
|
+
if until:
|
|
452
|
+
nodes = [n for n in nodes
|
|
453
|
+
if (n.observed_at or n.created) <= until]
|
|
454
|
+
nodes.sort(key=lambda n: n.observed_at or n.created, reverse=True)
|
|
455
|
+
return nodes[:limit]
|
|
456
|
+
|
|
457
|
+
def valid_at(self, timestamp: str) -> dict:
|
|
458
|
+
"""The graph as it was at a point in time (Phase 2: temporal reasoning).
|
|
459
|
+
|
|
460
|
+
Returns {nodes: [...], edges: [...]} — only nodes and edges that were
|
|
461
|
+
live at the given timestamp. Read-only.
|
|
462
|
+
"""
|
|
463
|
+
from .temporal import valid_at
|
|
464
|
+
return valid_at(self.brain, timestamp)
|
|
465
|
+
|
|
466
|
+
def history(self, node_id: str) -> list[dict]:
|
|
467
|
+
"""How a node's edges evolved over time (Phase 2: temporal reasoning).
|
|
468
|
+
|
|
469
|
+
Returns a chronological list of edge events (creation, invalidation)
|
|
470
|
+
involving this node. Read-only.
|
|
471
|
+
"""
|
|
472
|
+
from .temporal import history
|
|
473
|
+
return history(self.brain, node_id)
|
|
474
|
+
|
|
475
|
+
def when(self, query: str, at: str, k: int = 5) -> list[tuple[str, float]]:
|
|
476
|
+
"""Retrieval restricted to what was known at a point in time (Phase 2).
|
|
477
|
+
|
|
478
|
+
Runs the normal hybrid retrieval, then filters the results to only
|
|
479
|
+
include nodes that existed at the given timestamp. Read-only.
|
|
480
|
+
"""
|
|
481
|
+
from .temporal import when
|
|
482
|
+
return when(self, query, at, k=k)
|
|
483
|
+
|
|
484
|
+
def build_context(self, query: str, budget: int = 4000, **kwargs) -> dict:
|
|
485
|
+
"""Assemble a context window for a query (Phase 3: context management).
|
|
486
|
+
|
|
487
|
+
Returns {context, tokens, nodes, budget, truncated}. Read-only.
|
|
488
|
+
"""
|
|
489
|
+
from .context import build_context
|
|
490
|
+
return build_context(self, query, budget, **kwargs)
|
|
491
|
+
|
|
492
|
+
def context_for_prompt(self, query: str, max_tokens: int = 2000) -> str:
|
|
493
|
+
"""Build a context string ready for a prompt (Phase 3). Read-only."""
|
|
494
|
+
from .context import context_for_prompt
|
|
495
|
+
return context_for_prompt(self, query, max_tokens=max_tokens)
|
|
496
|
+
|
|
497
|
+
def context_stats(self, query: str, budget: int = 4000) -> dict:
|
|
498
|
+
"""Statistics about a context build (Phase 3). Read-only."""
|
|
499
|
+
from .context import context_stats
|
|
500
|
+
return context_stats(self, query, budget)
|
|
501
|
+
|
|
502
|
+
def _heal_after_failed_commit(self) -> None:
|
|
503
|
+
"""Audit #31: after a failed commit, restore the derived state to a
|
|
504
|
+
consistent, rebuildable condition and tell the operator what happened."""
|
|
505
|
+
try:
|
|
506
|
+
self.brain.rebuild_index()
|
|
507
|
+
except Exception:
|
|
508
|
+
pass # index rebuild is best-effort; nodes/edges/vectors are the truth
|
|
509
|
+
print("WARNING: brain mutated but commit/push failed — changes are on disk "
|
|
510
|
+
"UNCOMMITTED. Recovery: `git -C <brain> add -A && git commit`, or "
|
|
511
|
+
"retry the ingest (its commit will include these changes).", file=sys.stderr)
|
|
512
|
+
|
|
513
|
+
@staticmethod
|
|
514
|
+
def _now_short() -> str:
|
|
515
|
+
from datetime import datetime, timezone
|
|
516
|
+
return datetime.now(timezone.utc).strftime("%Y-%m-%d")
|
|
517
|
+
|
|
518
|
+
def resolve(self, edge_id: str, accept: bool) -> Edge | None:
|
|
519
|
+
with BRAIN_LOCK:
|
|
520
|
+
self.brain.ensure_ready()
|
|
521
|
+
self.brain.pull()
|
|
522
|
+
edge = self.brain.resolve_edge(edge_id, accept)
|
|
523
|
+
if edge is not None:
|
|
524
|
+
action = "accept" if accept else "reject"
|
|
525
|
+
self.brain.commit_and_push(f"edge {action}: {edge_id[:8]} [{edge.kind}]")
|
|
526
|
+
return edge
|
|
527
|
+
|
|
528
|
+
def undo(self, edge_id: str) -> Edge | None:
|
|
529
|
+
with BRAIN_LOCK:
|
|
530
|
+
self.brain.ensure_ready()
|
|
531
|
+
self.brain.pull()
|
|
532
|
+
edge = self.brain.restore_edge(edge_id)
|
|
533
|
+
if edge is not None:
|
|
534
|
+
self.brain.commit_and_push(f"edge undo: {edge_id[:8]} [{edge.kind}]")
|
|
535
|
+
return edge
|
|
536
|
+
|
|
537
|
+
def link(self, source_id: str, target_id: str,
|
|
538
|
+
kind: str = "same_as") -> Edge:
|
|
539
|
+
"""Create a manual edge (e.g. same_as for translation/alias pairs)."""
|
|
540
|
+
with BRAIN_LOCK:
|
|
541
|
+
self.brain.ensure_ready()
|
|
542
|
+
self.brain.pull()
|
|
543
|
+
ids = {n.id for n in self.brain.read_nodes()}
|
|
544
|
+
missing = [nid for nid in (source_id, target_id) if nid not in ids]
|
|
545
|
+
if missing:
|
|
546
|
+
raise ValueError(f"Node(s) not found: {', '.join(missing)}")
|
|
547
|
+
# Audit #23: the pair set was direction-less and kind-blind — a
|
|
548
|
+
# legitimate same_as AND similar between the same pair could not
|
|
549
|
+
# coexist, and A→B also blocked B→A. Dedupe is now
|
|
550
|
+
# kind-aware and direction-sensitive; only exact duplicates block.
|
|
551
|
+
existing = [(e.source, e.target, e.kind) for e in self.brain.read_edges()
|
|
552
|
+
if e.is_current and not e.rejected]
|
|
553
|
+
if (source_id, target_id, kind) in existing:
|
|
554
|
+
raise ValueError("This edge already exists.")
|
|
555
|
+
edge = Edge(source=source_id, target=target_id, kind=kind, pending=False,
|
|
556
|
+
origin="manual")
|
|
557
|
+
self.brain.add_edge(edge)
|
|
558
|
+
self.brain.commit_and_push(f"edge link: {source_id[:8]} --[{kind}]--> {target_id[:8]}")
|
|
559
|
+
return edge
|