sqlite-graph-memory 0.3.0__py3-none-any.whl → 0.3.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sqlite_graph_memory/__init__.py +1 -1
- sqlite_graph_memory/brain_ask.py +118 -5
- sqlite_graph_memory/mcp_server.py +40 -9
- sqlite_graph_memory/ppr.py +234 -0
- sqlite_graph_memory/vault_stats.py +217 -0
- {sqlite_graph_memory-0.3.0.dist-info → sqlite_graph_memory-0.3.2.dist-info}/METADATA +111 -19
- sqlite_graph_memory-0.3.2.dist-info/RECORD +14 -0
- {sqlite_graph_memory-0.3.0.dist-info → sqlite_graph_memory-0.3.2.dist-info}/entry_points.txt +2 -0
- sqlite_graph_memory-0.3.0.dist-info/RECORD +0 -12
- {sqlite_graph_memory-0.3.0.dist-info → sqlite_graph_memory-0.3.2.dist-info}/WHEEL +0 -0
- {sqlite_graph_memory-0.3.0.dist-info → sqlite_graph_memory-0.3.2.dist-info}/licenses/LICENSE +0 -0
sqlite_graph_memory/__init__.py
CHANGED
sqlite_graph_memory/brain_ask.py
CHANGED
|
@@ -9,12 +9,18 @@ USAGE:
|
|
|
9
9
|
python brain_ask.py --graph "question" # vector + graph expansion (Associative)
|
|
10
10
|
python brain_ask.py --ab "question" # run BOTH, diff, log to SQLite (ab_recall)
|
|
11
11
|
python brain_ask.py --ask "question" # emit a context bundle for an LLM agent
|
|
12
|
+
python brain_ask.py --ppr --graph "question" # + PageRank lane in the ORDERING (see ppr.py)
|
|
12
13
|
|
|
13
14
|
Config (env, all optional):
|
|
14
15
|
BRAIN_INDEX_DIR dir holding the embedding index (default: ./index)
|
|
15
16
|
TURNSTATE_DB SQLite db for A/B telemetry (default: ./turnstate.db)
|
|
16
17
|
BRAIN_ANSWER_OUT file to mirror the answer into (default: ./_brain_answer.txt)
|
|
17
18
|
BRAIN_AB_JSON if set, --ab also writes a structured JSON diff to this path
|
|
19
|
+
SGM_PPR=1 same as --ppr: fuse personalized PageRank over [[wikilinks]] with the
|
|
20
|
+
reranker by RRF, reranker top 5 protected (off by default)
|
|
21
|
+
SGM_NO_STAR_PROMPT=1 never print the one-time GitHub star line after --ab
|
|
22
|
+
SGM_CONFIG_DIR where that line's once-per-machine marker lives
|
|
23
|
+
(default: $XDG_CONFIG_HOME or ~/.config, /sqlite-graph-memory)
|
|
18
24
|
"""
|
|
19
25
|
import os, re, sys, pickle, datetime, sqlite3, time
|
|
20
26
|
from pathlib import Path
|
|
@@ -187,20 +193,113 @@ def expand_1hop(sims, meta, base_order, by_base=None, ghops=GHOPS, gmax=GMAX):
|
|
|
187
193
|
|
|
188
194
|
|
|
189
195
|
def rerank_candidates(ce, query, cand, meta, topn=TOPN):
|
|
190
|
-
"""Cross-encoder rerank of candidate chunks -> [(chunk index, score)], best first.
|
|
196
|
+
"""Cross-encoder rerank of candidate chunks -> [(chunk index, score)], best first.
|
|
197
|
+
topn=None returns the whole reranked list (the PPR fusion needs the tail)."""
|
|
191
198
|
pairs = [(query, meta[i]['title'] + '. ' + meta[i]['snippet']) for i in cand]
|
|
192
199
|
sc = ce.predict(pairs) if len(pairs) else []
|
|
193
200
|
return sorted(zip(cand, sc), key=lambda x: -x[1])[:topn]
|
|
194
201
|
|
|
195
202
|
|
|
203
|
+
def _ppr():
|
|
204
|
+
try:
|
|
205
|
+
from sqlite_graph_memory import ppr
|
|
206
|
+
except ImportError: # run as a loose script next to ppr.py
|
|
207
|
+
import ppr
|
|
208
|
+
return ppr
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
def ppr_enabled(env=None):
|
|
212
|
+
"""True when SGM_PPR asks for the PageRank lane. Never raises."""
|
|
213
|
+
try:
|
|
214
|
+
return _ppr().enabled(env)
|
|
215
|
+
except Exception:
|
|
216
|
+
return False
|
|
217
|
+
|
|
218
|
+
|
|
219
|
+
def ppr_lane(meta, sims, base_order):
|
|
220
|
+
"""PPR file order for this query, or [] (lane off) on ANY failure, including import."""
|
|
221
|
+
try:
|
|
222
|
+
return _ppr().ppr_file_order(meta, sims, base_order)
|
|
223
|
+
except Exception as e:
|
|
224
|
+
sys.stderr.write('PPR lane fell back to reranker order (%s)\n' % str(e)[:140])
|
|
225
|
+
return []
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
def fuse_ppr(ranked_full, meta, ppr_stems, topn=TOPN):
|
|
229
|
+
"""Reranker list fused with the PPR order, top `topn`. Any failure: reranker top `topn`."""
|
|
230
|
+
try:
|
|
231
|
+
return _ppr().fuse_after_rerank(ranked_full, meta, ppr_stems, topn)
|
|
232
|
+
except Exception as e:
|
|
233
|
+
sys.stderr.write('PPR fusion fell back to reranker order (%s)\n' % str(e)[:140])
|
|
234
|
+
return list(ranked_full)[:topn]
|
|
235
|
+
|
|
236
|
+
|
|
237
|
+
STAR_URL = 'https://github.com/tonydzi/sqlite-graph-memory'
|
|
238
|
+
STAR_MARKER = 'star_prompt_shown'
|
|
239
|
+
|
|
240
|
+
|
|
241
|
+
def _config_dir(env=None):
|
|
242
|
+
"""Per-user config folder: SGM_CONFIG_DIR, else $XDG_CONFIG_HOME or ~/.config."""
|
|
243
|
+
env = os.environ if env is None else env
|
|
244
|
+
if env.get('SGM_CONFIG_DIR'):
|
|
245
|
+
return Path(os.path.expanduser(env['SGM_CONFIG_DIR']))
|
|
246
|
+
base = env.get('XDG_CONFIG_HOME') or os.path.join(os.path.expanduser('~'), '.config')
|
|
247
|
+
return Path(base) / 'sqlite-graph-memory'
|
|
248
|
+
|
|
249
|
+
|
|
250
|
+
def maybe_star_prompt(stream=None, env=None, interactive=None):
|
|
251
|
+
"""After a successful --ab run, ask for a star once per machine, on stderr.
|
|
252
|
+
|
|
253
|
+
Silent when SGM_NO_STAR_PROMPT is set (anything but '' or '0'), when CI is set
|
|
254
|
+
(anything but '', '0' or 'false'), when BRAIN_AB_JSON is set (a machine reads the
|
|
255
|
+
result), when stdout or stderr is not a terminal, or when the marker file exists.
|
|
256
|
+
The marker is created with O_EXCL, so two first runs at once print one line.
|
|
257
|
+
CRASH-SAFETY: any failure here is swallowed; the answer was already printed.
|
|
258
|
+
Returns True only if the line was printed.
|
|
259
|
+
"""
|
|
260
|
+
env = os.environ if env is None else env
|
|
261
|
+
stream = sys.stderr if stream is None else stream
|
|
262
|
+
try:
|
|
263
|
+
if env.get('SGM_NO_STAR_PROMPT', '') not in ('', '0'):
|
|
264
|
+
return False
|
|
265
|
+
if env.get('CI', '').strip().lower() not in ('', '0', 'false'):
|
|
266
|
+
return False
|
|
267
|
+
if env.get('BRAIN_AB_JSON'):
|
|
268
|
+
return False
|
|
269
|
+
if interactive is None:
|
|
270
|
+
interactive = sys.stdout.isatty() and stream.isatty()
|
|
271
|
+
if not interactive:
|
|
272
|
+
return False
|
|
273
|
+
marker = _config_dir(env) / STAR_MARKER
|
|
274
|
+
marker.parent.mkdir(parents=True, exist_ok=True)
|
|
275
|
+
try:
|
|
276
|
+
fd = os.open(str(marker), os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o644)
|
|
277
|
+
except FileExistsError:
|
|
278
|
+
return False
|
|
279
|
+
with os.fdopen(fd, 'w', encoding='utf-8') as f:
|
|
280
|
+
f.write(time.strftime('%Y-%m-%d %H:%M:%S') + '\n')
|
|
281
|
+
try:
|
|
282
|
+
stream.write('\nIf the A/B diff above was useful, a star helps other people find it: '
|
|
283
|
+
'%s (shown once; SGM_NO_STAR_PROMPT=1 silences it)\n' % STAR_URL)
|
|
284
|
+
stream.flush()
|
|
285
|
+
except Exception:
|
|
286
|
+
try: marker.unlink() # the user never saw it: ask again next time
|
|
287
|
+
except Exception: pass
|
|
288
|
+
return False
|
|
289
|
+
return True
|
|
290
|
+
except Exception:
|
|
291
|
+
return False
|
|
292
|
+
|
|
293
|
+
|
|
196
294
|
def main():
|
|
197
295
|
args = sys.argv[1:]
|
|
198
296
|
ask = '--ask' in args
|
|
199
297
|
graph = '--graph' in args; ab = '--ab' in args
|
|
200
|
-
flags = {'--ask', '--graph', '--ab'}
|
|
298
|
+
flags = {'--ask', '--graph', '--ab', '--ppr'}
|
|
201
299
|
query = ' '.join(a for a in args if a not in flags)
|
|
202
300
|
if not query or not EMB.exists():
|
|
203
|
-
|
|
301
|
+
sys.stderr.write('need an embedding index (%s) and a query — run index_notes.py first\n' % EMB)
|
|
302
|
+
raise SystemExit(1)
|
|
204
303
|
|
|
205
304
|
dev = pick_device()
|
|
206
305
|
emb, meta = load_index()
|
|
@@ -223,10 +322,23 @@ def main():
|
|
|
223
322
|
added = expand_1hop(sims, meta, base_order)
|
|
224
323
|
graph_order = base_order + added; graph_set = set(added)
|
|
225
324
|
|
|
325
|
+
# PageRank lane (opt-in): a second ORDERING, not a candidate source. Crash-safe like the
|
|
326
|
+
# hop: any failure leaves ppr_stems empty and every list below keeps the reranker's order.
|
|
327
|
+
# Not in --ab: the A/B answers "does the 1-hop graph help?", and its rows in ab_recall
|
|
328
|
+
# carry no PPR column, so a lane on both arms would be logged as plain vector vs graph.
|
|
329
|
+
ppr_stems = []
|
|
330
|
+
if ab and ('--ppr' in args or ppr_enabled()):
|
|
331
|
+
sys.stderr.write('PPR lane is not applied in --ab (score it with eval/run_eval.py --ppr)\n')
|
|
332
|
+
elif order and ('--ppr' in args or ppr_enabled()):
|
|
333
|
+
ppr_stems = ppr_lane(meta, sims, base_order)
|
|
334
|
+
|
|
226
335
|
# rerank on the matched chunk (precise)
|
|
227
336
|
ce = load_reranker(dev)
|
|
228
337
|
def rerank(cand):
|
|
229
|
-
|
|
338
|
+
if not ppr_stems:
|
|
339
|
+
return rerank_candidates(ce, query, cand, meta)
|
|
340
|
+
full = rerank_candidates(ce, query, cand, meta, topn=None) # fusion needs the tail
|
|
341
|
+
return fuse_ppr(full, meta, ppr_stems, TOPN)
|
|
230
342
|
|
|
231
343
|
# --ab: run BOTH paths in ONE process, diff, log. Answers "does graph REALLY help?"
|
|
232
344
|
if ab:
|
|
@@ -279,6 +391,7 @@ def main():
|
|
|
279
391
|
pass
|
|
280
392
|
_out = Path(os.getenv('BRAIN_ANSWER_OUT', '_brain_answer.txt'))
|
|
281
393
|
_out.write_text(text, encoding='utf-8'); print(text)
|
|
394
|
+
maybe_star_prompt()
|
|
282
395
|
return
|
|
283
396
|
|
|
284
397
|
# GATE: wiki-graph helps THEME queries, hurts ENTITY/name queries -> skip graph for entities.
|
|
@@ -287,7 +400,7 @@ def main():
|
|
|
287
400
|
sys.stderr.write("wiki-graph gated OFF (entity-like query) -> Direct only\n")
|
|
288
401
|
order = graph_order if (graph and not gated) else base_order
|
|
289
402
|
reranked = rerank(order)
|
|
290
|
-
scope = 'all' + (' /graph+%d' % len(graph_set) if graph else '')
|
|
403
|
+
scope = 'all' + (' /graph+%d' % len(graph_set) if graph else '') + (' /ppr' if ppr_stems else '')
|
|
291
404
|
lines = [f'QUERY: {query}', f'(scope={scope}; e5 {len(order)} files -> rerank; {len(meta)} chunks)', '']
|
|
292
405
|
if ask:
|
|
293
406
|
lines.append('=== CONTEXT BUNDLE (paste to your agent) ===\n')
|
|
@@ -22,6 +22,26 @@ Config (env, optional):
|
|
|
22
22
|
import sys, os, json, subprocess, tempfile
|
|
23
23
|
from pathlib import Path
|
|
24
24
|
|
|
25
|
+
# Where the recall subprocess runs and where the package root resides.
|
|
26
|
+
# The server deliberately shells out instead of importing brain_ask: a model load
|
|
27
|
+
# that segfaults or runs out of memory then kills a child process and returns an
|
|
28
|
+
# MCP error, rather than taking the server down mid-session.
|
|
29
|
+
ROOT_DIR = Path(__file__).resolve().parent
|
|
30
|
+
|
|
31
|
+
# Ensure the package root (src/) is on sys.path when invoked directly as a script
|
|
32
|
+
# from a source checkout without an active editable install.
|
|
33
|
+
_SRC_DIR = str(ROOT_DIR.parent)
|
|
34
|
+
if _SRC_DIR not in sys.path:
|
|
35
|
+
sys.path.insert(0, _SRC_DIR)
|
|
36
|
+
|
|
37
|
+
try:
|
|
38
|
+
from sqlite_graph_memory import __version__
|
|
39
|
+
except ImportError:
|
|
40
|
+
try:
|
|
41
|
+
from . import __version__
|
|
42
|
+
except (ImportError, ValueError):
|
|
43
|
+
__version__ = "0.3.2"
|
|
44
|
+
|
|
25
45
|
# Ensure UTF-8 for stdio communication on all platforms (including Windows console)
|
|
26
46
|
try:
|
|
27
47
|
sys.stdin.reconfigure(encoding='utf-8')
|
|
@@ -30,20 +50,17 @@ try:
|
|
|
30
50
|
except Exception:
|
|
31
51
|
pass
|
|
32
52
|
|
|
33
|
-
# Where the recall subprocess runs. The server deliberately shells out instead
|
|
34
|
-
# of importing brain_ask: a model load that segfaults or runs out of memory then
|
|
35
|
-
# kills a child process and returns an MCP error, rather than taking the server
|
|
36
|
-
# down mid-session.
|
|
37
|
-
ROOT_DIR = Path(__file__).resolve().parent
|
|
38
|
-
|
|
39
53
|
|
|
40
54
|
def _brain_ask_cmd():
|
|
41
|
-
"""How to invoke brain_ask —
|
|
55
|
+
"""How to invoke brain_ask — prefers the sibling script when present, falls back to module."""
|
|
56
|
+
sibling = ROOT_DIR / "brain_ask.py"
|
|
57
|
+
if sibling.exists():
|
|
58
|
+
return [sys.executable, str(sibling)]
|
|
42
59
|
return [sys.executable, "-m", "sqlite_graph_memory.brain_ask"]
|
|
43
60
|
|
|
44
61
|
SERVER_INFO = {
|
|
45
62
|
"name": "sqlite-graph-memory",
|
|
46
|
-
"version":
|
|
63
|
+
"version": __version__
|
|
47
64
|
}
|
|
48
65
|
|
|
49
66
|
TOOLS = [
|
|
@@ -109,6 +126,9 @@ def run_recall(query: str, mode: str = "associative") -> tuple[str, bool]:
|
|
|
109
126
|
|
|
110
127
|
try:
|
|
111
128
|
call_env = {**os.environ, "BRAIN_ANSWER_OUT": str(temp_ans_file)}
|
|
129
|
+
src_dir = str(ROOT_DIR.parent)
|
|
130
|
+
existing_pp = call_env.get("PYTHONPATH", "")
|
|
131
|
+
call_env["PYTHONPATH"] = f"{src_dir}{os.pathsep}{existing_pp}" if existing_pp else src_dir
|
|
112
132
|
res = subprocess.run(
|
|
113
133
|
cmd,
|
|
114
134
|
cwd=str(ROOT_DIR),
|
|
@@ -147,6 +167,15 @@ def run_recall(query: str, mode: str = "associative") -> tuple[str, bool]:
|
|
|
147
167
|
|
|
148
168
|
def handle_request(req: dict) -> dict:
|
|
149
169
|
"""Handle a single JSON-RPC 2.0 MCP request."""
|
|
170
|
+
if not isinstance(req, dict):
|
|
171
|
+
return {
|
|
172
|
+
"jsonrpc": "2.0",
|
|
173
|
+
"id": None,
|
|
174
|
+
"error": {
|
|
175
|
+
"code": -32600,
|
|
176
|
+
"message": "Invalid Request: expected JSON object"
|
|
177
|
+
}
|
|
178
|
+
}
|
|
150
179
|
method = req.get("method")
|
|
151
180
|
req_id = req.get("id")
|
|
152
181
|
|
|
@@ -243,9 +272,11 @@ def main():
|
|
|
243
272
|
mode = "ab"
|
|
244
273
|
query_parts = [a for a in sys.argv[1:] if a not in flags]
|
|
245
274
|
q = " ".join(query_parts) if query_parts else "agent memory"
|
|
246
|
-
text,
|
|
275
|
+
text, is_err = run_recall(q, mode=mode)
|
|
247
276
|
print(f"=== MCP Test Query: {q} (mode={mode}) ===")
|
|
248
277
|
print(text)
|
|
278
|
+
if is_err:
|
|
279
|
+
sys.exit(1)
|
|
249
280
|
return
|
|
250
281
|
|
|
251
282
|
|
|
@@ -0,0 +1,234 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
"""ppr.py: personalized PageRank over the [[wikilink]] graph, fused with the reranker by RRF.
|
|
3
|
+
|
|
4
|
+
WHY IT EXISTS. On the home vault's frozen gold set (2026-09-16, 60 bridge questions) 97% of
|
|
5
|
+
the correct bridge answers were already in the candidate pool, and Recall@12 in that diagnosis
|
|
6
|
+
run was 0.225 (the README's 0.251 is the graph-mode baseline of the final measurement):
|
|
7
|
+
the cross-encoder dropped them, because the relation is structural, not textual. Adding more
|
|
8
|
+
candidates (more hops, larger GMAX) cannot fix that. This module gives the graph a vote in the
|
|
9
|
+
ORDERING instead: PPR seeded by the dense hits yields a structural file order, and that order
|
|
10
|
+
is fused with the reranker's order by Reciprocal Rank Fusion. Ranks, not scores, because
|
|
11
|
+
cosine, cross-encoder logits and PageRank mass are on incomparable scales.
|
|
12
|
+
|
|
13
|
+
THE HEAD IS PROTECTED. The first PPR_HEAD (5) positions of the reranker are never touched. When
|
|
14
|
+
a query names one thing, the right answer is already first, and any shuffling drops it (home
|
|
15
|
+
vault: without the protection title nDCG@12 fell by 0.076; with head=5 it rose by 0.006 at the
|
|
16
|
+
same bridge gain). PPR only reorders candidates already in the pool; it never brings new files,
|
|
17
|
+
because a file outside the pool has no chunk and no score to show.
|
|
18
|
+
|
|
19
|
+
OFF BY DEFAULT. Turn it on with SGM_PPR=1 (or `sgm-ask --ppr`). The home pipeline runs it on by
|
|
20
|
+
default; the package does not, because the measurement behind it was taken on one vault.
|
|
21
|
+
|
|
22
|
+
CRASH-SAFETY, same contract as the graph hop: no graph, no edges, a PPR failure or a fusion
|
|
23
|
+
failure each return the reranker's order unchanged and write one line to stderr. Worst case you
|
|
24
|
+
get the old result, never an empty or broken recall.
|
|
25
|
+
|
|
26
|
+
COST on the home vault (7.8k files, ~40k edges): building the graph 0.5 s once per process
|
|
27
|
+
(cached in memory by an index fingerprint), PPR itself ~3 ms per query.
|
|
28
|
+
|
|
29
|
+
Ported from the maintainer's home module brain_graph.py (2026-09-16) for issue #51.
|
|
30
|
+
"""
|
|
31
|
+
import hashlib
|
|
32
|
+
import os
|
|
33
|
+
import sys
|
|
34
|
+
from pathlib import Path
|
|
35
|
+
|
|
36
|
+
import numpy as np
|
|
37
|
+
|
|
38
|
+
PPR_ALPHA = 0.85 # damping: share of mass that walks the edges. High on purpose: the gain is
|
|
39
|
+
# in moving AWAY from the hits to their neighbours (0.5 gave half the bridge gain)
|
|
40
|
+
PPR_ITERS = 20 # power iteration; converges in <10 on a graph this size
|
|
41
|
+
PPR_SEED_TOPK = 15 # how many best dense FILES seed the personalization (same as GHOPS)
|
|
42
|
+
PPR_TOPM = 10 # how many PPR files enter the fusion. More is WORSE: a long structural tail
|
|
43
|
+
# dilutes the top (topm=30 gave bridge +0.053, topm=10 gave +0.103)
|
|
44
|
+
PPR_HEAD = 5 # reranker positions the fusion never touches
|
|
45
|
+
RRF_K = 60 # RRF constant (Cormack et al. 2009)
|
|
46
|
+
|
|
47
|
+
_CACHE = {} # index fingerprint -> Graph, in process memory only
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def enabled(env=None):
|
|
51
|
+
"""True when SGM_PPR is set to 1/on/yes/true. Default: off."""
|
|
52
|
+
env = os.environ if env is None else env
|
|
53
|
+
return str(env.get('SGM_PPR', '')).strip().lower() in ('1', 'on', 'yes', 'true')
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def _links_in(path):
|
|
57
|
+
# One wikilink parser for the whole package: the one the graph hop uses.
|
|
58
|
+
try:
|
|
59
|
+
from sqlite_graph_memory.brain_ask import _links_in as f
|
|
60
|
+
except ImportError: # run as a loose script next to brain_ask.py
|
|
61
|
+
from brain_ask import _links_in as f
|
|
62
|
+
return f(path)
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
class Graph:
|
|
66
|
+
"""File-level wikilink graph in CSR form: node = lowercase file stem."""
|
|
67
|
+
|
|
68
|
+
def __init__(self, stems, idx, indptr, indices):
|
|
69
|
+
self.stems = stems
|
|
70
|
+
self.idx = idx
|
|
71
|
+
self.indptr = indptr
|
|
72
|
+
self.indices = indices
|
|
73
|
+
self.n_edges = int(len(indices))
|
|
74
|
+
|
|
75
|
+
def ppr(self, seed, alpha=None, iters=None):
|
|
76
|
+
"""Personalized PageRank: p = alpha * A^T p + (1 - alpha) * seed.
|
|
77
|
+
|
|
78
|
+
Dangling nodes (no outgoing links) return their mass to the seed instead of spreading
|
|
79
|
+
it over the whole graph, otherwise the unlinked half of a vault would wash out the
|
|
80
|
+
personalization.
|
|
81
|
+
"""
|
|
82
|
+
alpha = PPR_ALPHA if alpha is None else alpha
|
|
83
|
+
iters = PPR_ITERS if iters is None else iters
|
|
84
|
+
n = len(self.stems)
|
|
85
|
+
s = np.asarray(seed, dtype='float64')
|
|
86
|
+
tot = s.sum()
|
|
87
|
+
if n == 0 or tot <= 0:
|
|
88
|
+
return np.zeros(n, dtype='float64')
|
|
89
|
+
s = s / tot
|
|
90
|
+
if self.n_edges == 0:
|
|
91
|
+
return s # no edges: PPR is the personalization itself
|
|
92
|
+
p = s.copy()
|
|
93
|
+
deg = np.diff(self.indptr)
|
|
94
|
+
live = deg > 0
|
|
95
|
+
src = np.repeat(np.arange(n), deg)
|
|
96
|
+
for _ in range(iters):
|
|
97
|
+
contrib = np.zeros(n, dtype='float64')
|
|
98
|
+
contrib[live] = p[live] / deg[live]
|
|
99
|
+
nxt = np.zeros(n, dtype='float64')
|
|
100
|
+
np.add.at(nxt, self.indices, contrib[src])
|
|
101
|
+
dangling = float(p[~live].sum())
|
|
102
|
+
p = alpha * nxt + (alpha * dangling + (1.0 - alpha)) * s
|
|
103
|
+
return p
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def _fingerprint(meta):
|
|
107
|
+
"""sha256 over (path, sha) of every chunk, so the cache expires when the index changes."""
|
|
108
|
+
h = hashlib.sha256()
|
|
109
|
+
for m in meta:
|
|
110
|
+
h.update(str(m.get('path', '')).encode('utf-8', 'replace'))
|
|
111
|
+
h.update(b'\x1f')
|
|
112
|
+
h.update(str(m.get('sha', '')).encode('utf-8', 'replace'))
|
|
113
|
+
h.update(b'\x1e')
|
|
114
|
+
return h.hexdigest()
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def build(meta):
|
|
118
|
+
"""Build the file graph from the notes' outgoing [[wikilinks]] (self-links and dups dropped).
|
|
119
|
+
|
|
120
|
+
A node is a file NAME, because that is what a wikilink addresses: a/index.md and
|
|
121
|
+
b/index.md are one node, and the outgoing links of both files become its edges.
|
|
122
|
+
"""
|
|
123
|
+
try:
|
|
124
|
+
fp = _fingerprint(meta)
|
|
125
|
+
except Exception:
|
|
126
|
+
fp = None
|
|
127
|
+
if fp and fp in _CACHE:
|
|
128
|
+
return _CACHE[fp]
|
|
129
|
+
stems, paths, idx = [], [], {}
|
|
130
|
+
for m in meta:
|
|
131
|
+
st = Path(m['path']).stem.lower()
|
|
132
|
+
if st not in idx:
|
|
133
|
+
idx[st] = len(stems)
|
|
134
|
+
stems.append(st)
|
|
135
|
+
paths.append([])
|
|
136
|
+
if m['path'] not in paths[idx[st]]:
|
|
137
|
+
paths[idx[st]].append(m['path'])
|
|
138
|
+
indptr = np.zeros(len(stems) + 1, dtype='int64')
|
|
139
|
+
indices = []
|
|
140
|
+
for u, files in enumerate(paths):
|
|
141
|
+
seen = set()
|
|
142
|
+
for p in files:
|
|
143
|
+
for tgt in _links_in(p):
|
|
144
|
+
v = idx.get(tgt.strip().lower())
|
|
145
|
+
if v is None or v == u or v in seen:
|
|
146
|
+
continue
|
|
147
|
+
seen.add(v)
|
|
148
|
+
indices.append(v)
|
|
149
|
+
indptr[u + 1] = len(indices)
|
|
150
|
+
g = Graph(stems, idx, indptr, np.asarray(indices, dtype='int64'))
|
|
151
|
+
if fp:
|
|
152
|
+
_CACHE[fp] = g
|
|
153
|
+
return g
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def ppr_file_order(meta, sims, base_order, graph=None, topm=None, seed_topk=None):
|
|
157
|
+
"""File stems ordered by personalized PageRank, seeded by the dense hits in base_order.
|
|
158
|
+
|
|
159
|
+
Seed weight = best chunk cosine + 1, so it is non-negative (PageRank needs a non-negative
|
|
160
|
+
teleport, cosine lives in [-1, 1]). Returns [] on any failure; the caller keeps its order.
|
|
161
|
+
"""
|
|
162
|
+
topm = PPR_TOPM if topm is None else topm
|
|
163
|
+
seed_topk = PPR_SEED_TOPK if seed_topk is None else seed_topk
|
|
164
|
+
try:
|
|
165
|
+
g = build(meta) if graph is None else graph
|
|
166
|
+
if not g.stems or g.n_edges == 0:
|
|
167
|
+
return []
|
|
168
|
+
seed = np.zeros(len(g.stems), dtype='float64')
|
|
169
|
+
seen, used = set(), 0
|
|
170
|
+
for i in base_order:
|
|
171
|
+
st = Path(meta[i]['path']).stem.lower()
|
|
172
|
+
if st in seen:
|
|
173
|
+
continue
|
|
174
|
+
seen.add(st)
|
|
175
|
+
u = g.idx.get(st)
|
|
176
|
+
if u is None:
|
|
177
|
+
continue
|
|
178
|
+
seed[u] = max(seed[u], float(sims[i]) + 1.0) # seed starts at 0: never negative
|
|
179
|
+
used += 1
|
|
180
|
+
if used >= seed_topk:
|
|
181
|
+
break
|
|
182
|
+
if seed.sum() <= 0:
|
|
183
|
+
return []
|
|
184
|
+
p = g.ppr(seed)
|
|
185
|
+
top = np.argsort(-p, kind='stable')[:max(1, topm)]
|
|
186
|
+
return [g.stems[int(u)] for u in top if p[int(u)] > 0]
|
|
187
|
+
except Exception as e:
|
|
188
|
+
sys.stderr.write('PPR lane failed (%s) -> reranker order unchanged\n' % str(e)[:140])
|
|
189
|
+
return []
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
def rrf_fuse(*ranked_lists, k=RRF_K):
|
|
193
|
+
"""Reciprocal Rank Fusion: score(d) = sum 1 / (k + rank_d + 1). Ties break by item value."""
|
|
194
|
+
score, order = {}, []
|
|
195
|
+
for lst in ranked_lists:
|
|
196
|
+
for rank, i in enumerate(lst):
|
|
197
|
+
if i not in score:
|
|
198
|
+
score[i] = 0.0
|
|
199
|
+
order.append(i)
|
|
200
|
+
score[i] += 1.0 / (k + rank + 1)
|
|
201
|
+
order.sort(key=lambda i: (-score[i], i))
|
|
202
|
+
return order
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
def fuse_after_rerank(ranked_idx, meta, ppr_stems, topn, head=None):
|
|
206
|
+
"""Pipeline hook: reorder [(chunk index, score)] from the reranker, then cut to topn.
|
|
207
|
+
|
|
208
|
+
`ranked_idx` must be the FULL reranked list, not the top-12: the point is to lift the right
|
|
209
|
+
neighbour out of the tail, which a fusion that only sees the top-12 cannot do. Empty PPR
|
|
210
|
+
returns the first topn exactly as given.
|
|
211
|
+
|
|
212
|
+
Every item is kept: two notes with the same file name in different folders are one graph
|
|
213
|
+
node (wikilinks address notes by name) but stay two results, so the output is never
|
|
214
|
+
shorter than the input. Positions, not names, go into RRF, so a tie breaks toward the
|
|
215
|
+
reranker's earlier position instead of the alphabetical order of file names.
|
|
216
|
+
"""
|
|
217
|
+
if not ppr_stems or not ranked_idx:
|
|
218
|
+
return list(ranked_idx)[:topn]
|
|
219
|
+
try:
|
|
220
|
+
items = list(ranked_idx)
|
|
221
|
+
head = PPR_HEAD if head is None else head
|
|
222
|
+
stems = [Path(meta[i]['path']).stem.lower() for i, _ in items]
|
|
223
|
+
kept = set(stems[:head])
|
|
224
|
+
first = {}
|
|
225
|
+
for p in range(head, len(items)):
|
|
226
|
+
first.setdefault(stems[p], p - head)
|
|
227
|
+
ppr_pos = [first[s] for s in ppr_stems if s in first and s not in kept]
|
|
228
|
+
if not ppr_pos:
|
|
229
|
+
return items[:topn]
|
|
230
|
+
fused = rrf_fuse(list(range(len(items) - head)), ppr_pos)
|
|
231
|
+
return (items[:head] + [items[head + p] for p in fused])[:topn]
|
|
232
|
+
except Exception as e:
|
|
233
|
+
sys.stderr.write('PPR fusion failed (%s) -> reranker order\n' % str(e)[:140])
|
|
234
|
+
return list(ranked_idx)[:topn]
|
|
@@ -0,0 +1,217 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""sgm-stats -- four health numbers for a folder of linked markdown notes.
|
|
3
|
+
|
|
4
|
+
sgm-stats <notes_dir> [--json] [--today YYYY-MM-DD] [--threshold-* N ...]
|
|
5
|
+
|
|
6
|
+
Measures the graph the retriever actually walks: the same note filter as the indexer
|
|
7
|
+
(``iter_notes``: no dot-directories, no sync-conflict copies) and the same wikilink parser
|
|
8
|
+
and target resolution as ``brain_ask`` (``_links_in``, basename match, case-insensitive).
|
|
9
|
+
A second parser would report on a graph recall never sees.
|
|
10
|
+
|
|
11
|
+
orphan rate pages with no inbound link ok <5% warn 5-15% fail >15%
|
|
12
|
+
avg degree resolved outgoing links per page ok 3-8 warn 2-3 / 8-10 fail <2 or >10
|
|
13
|
+
main component share of pages in the largest ok >=80% warn below
|
|
14
|
+
weakly connected component
|
|
15
|
+
stale concepts concept pages whose `updated` warn >50%
|
|
16
|
+
(or `date`) is older than 90 days
|
|
17
|
+
|
|
18
|
+
Pages are keyed by their path inside the vault, so two notes with the same name in
|
|
19
|
+
different folders stay two pages; a ``[[name]]`` link reaches every page with that name,
|
|
20
|
+
the same candidates ``brain_ask.index_by_basename`` keeps. Links to pages that do not
|
|
21
|
+
exist and links from a page to itself are not edges. A concept page is one with
|
|
22
|
+
``type: concept`` in its frontmatter or one that lives in a folder named ``concepts``
|
|
23
|
+
inside the vault. Stale share is over all concept pages; undated ones (no date, or a
|
|
24
|
+
date that is not a real day) are named in the detail and never count as stale.
|
|
25
|
+
Verdicts are taken on the same rounded value the report prints.
|
|
26
|
+
|
|
27
|
+
Exit code: 0 all ok, 1 any warn, 2 any fail -- so it can sit in cron as a watchdog.
|
|
28
|
+
Zero model calls, zero network, stdlib + the two sibling modules.
|
|
29
|
+
"""
|
|
30
|
+
import argparse
|
|
31
|
+
import datetime
|
|
32
|
+
import json
|
|
33
|
+
import re
|
|
34
|
+
import sys
|
|
35
|
+
from collections import defaultdict
|
|
36
|
+
from pathlib import Path
|
|
37
|
+
|
|
38
|
+
from sqlite_graph_memory.brain_ask import _links_in
|
|
39
|
+
from sqlite_graph_memory.index_notes import FM_RX, iter_notes, read_note
|
|
40
|
+
|
|
41
|
+
DEFAULTS = {
|
|
42
|
+
'orphan_warn': 5.0, 'orphan_fail': 15.0,
|
|
43
|
+
'degree_ok_min': 3.0, 'degree_ok_max': 8.0,
|
|
44
|
+
'degree_warn_min': 2.0, 'degree_warn_max': 10.0,
|
|
45
|
+
'component_ok': 80.0,
|
|
46
|
+
'stale_days': 90, 'stale_warn': 50.0,
|
|
47
|
+
}
|
|
48
|
+
RANK = {'ok': 0, 'warn': 1, 'fail': 2, 'n/a': 0}
|
|
49
|
+
|
|
50
|
+
TYPE_RX = re.compile(r"""(?m)^type:\s*["']?([^"'\n]+?)["']?\s*$""")
|
|
51
|
+
UPDATED_RX = re.compile(r"""(?m)^updated:\s*["']?(\d{4}-\d{2}-\d{2})""")
|
|
52
|
+
DATE_RX = re.compile(r"""(?m)^date:\s*["']?(\d{4}-\d{2}-\d{2})""")
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
class EmptyVault(Exception):
|
|
56
|
+
"""No note under the folder: an unmounted or wrong path, reported as fail."""
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def build_graph(notes_dir):
|
|
60
|
+
"""(pages, edges): pages = {relative path without .md: Path},
|
|
61
|
+
edges = {(src, dst)} between real pages."""
|
|
62
|
+
root = Path(notes_dir)
|
|
63
|
+
pages, by_stem = {}, defaultdict(list)
|
|
64
|
+
for p in iter_notes(root):
|
|
65
|
+
key = p.relative_to(root).with_suffix('').as_posix()
|
|
66
|
+
pages[key] = p
|
|
67
|
+
by_stem[p.stem.lower()].append(key)
|
|
68
|
+
edges = set()
|
|
69
|
+
for src, path in pages.items():
|
|
70
|
+
for tgt in _links_in(path):
|
|
71
|
+
for dst in by_stem.get(tgt.lower(), ()):
|
|
72
|
+
if dst != src:
|
|
73
|
+
edges.add((src, dst))
|
|
74
|
+
return pages, edges
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def largest_component(pages, edges):
|
|
78
|
+
"""Size of the largest weakly connected component (union-find)."""
|
|
79
|
+
parent = {p: p for p in pages}
|
|
80
|
+
|
|
81
|
+
def find(x):
|
|
82
|
+
while parent[x] != x:
|
|
83
|
+
parent[x] = parent[parent[x]]
|
|
84
|
+
x = parent[x]
|
|
85
|
+
return x
|
|
86
|
+
|
|
87
|
+
for a, b in edges:
|
|
88
|
+
ra, rb = find(a), find(b)
|
|
89
|
+
if ra != rb:
|
|
90
|
+
parent[ra] = rb
|
|
91
|
+
sizes = defaultdict(int)
|
|
92
|
+
for p in pages:
|
|
93
|
+
sizes[find(p)] += 1
|
|
94
|
+
return max(sizes.values()) if sizes else 0, len(sizes)
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def _parse_day(s):
|
|
98
|
+
try:
|
|
99
|
+
return datetime.date.fromisoformat(s)
|
|
100
|
+
except ValueError: # 2026-02-30: date-shaped, not a day
|
|
101
|
+
return None
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def concept_dates(pages):
|
|
105
|
+
"""[(key, date or None)] for every concept page."""
|
|
106
|
+
out = []
|
|
107
|
+
for key, path in pages.items():
|
|
108
|
+
text = read_note(path) or ''
|
|
109
|
+
m = FM_RX.match(text)
|
|
110
|
+
fm = m.group(1) if m else ''
|
|
111
|
+
t = TYPE_RX.search(fm)
|
|
112
|
+
is_concept = (t and t.group(1).strip().lower() == 'concept') or \
|
|
113
|
+
any(part.lower() == 'concepts' for part in Path(key).parent.parts)
|
|
114
|
+
if not is_concept:
|
|
115
|
+
continue
|
|
116
|
+
d = UPDATED_RX.search(fm) or DATE_RX.search(fm)
|
|
117
|
+
out.append((key, _parse_day(d.group(1)) if d else None))
|
|
118
|
+
return out
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def verdict_orphan(v, th):
|
|
122
|
+
return 'fail' if v > th['orphan_fail'] else 'warn' if v >= th['orphan_warn'] else 'ok'
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def verdict_degree(v, th):
|
|
126
|
+
if th['degree_ok_min'] <= v <= th['degree_ok_max']:
|
|
127
|
+
return 'ok'
|
|
128
|
+
if th['degree_warn_min'] <= v <= th['degree_warn_max']:
|
|
129
|
+
return 'warn'
|
|
130
|
+
return 'fail'
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def compute(notes_dir, today=None, th=None):
|
|
134
|
+
th = {**DEFAULTS, **(th or {})}
|
|
135
|
+
today = today or datetime.date.today()
|
|
136
|
+
pages, edges = build_graph(notes_dir)
|
|
137
|
+
n = len(pages)
|
|
138
|
+
if n == 0:
|
|
139
|
+
raise EmptyVault(f"sgm-stats: no notes under {notes_dir}")
|
|
140
|
+
|
|
141
|
+
indeg = defaultdict(int)
|
|
142
|
+
for _, dst in edges:
|
|
143
|
+
indeg[dst] += 1
|
|
144
|
+
orphans = sorted(p for p in pages if indeg[p] == 0)
|
|
145
|
+
orphan_pct = round(100.0 * len(orphans) / n, 1)
|
|
146
|
+
degree = round(len(edges) / n, 2)
|
|
147
|
+
main, n_comp = largest_component(pages, edges)
|
|
148
|
+
main_pct = round(100.0 * main / n, 1)
|
|
149
|
+
|
|
150
|
+
concepts = concept_dates(pages)
|
|
151
|
+
dated = [(s, d) for s, d in concepts if d is not None]
|
|
152
|
+
cutoff = today - datetime.timedelta(days=int(th['stale_days']))
|
|
153
|
+
stale = sorted(s for s, d in dated if d < cutoff)
|
|
154
|
+
if concepts:
|
|
155
|
+
stale_pct = round(100.0 * len(stale) / len(concepts), 1)
|
|
156
|
+
stale_v = 'warn' if stale_pct > th['stale_warn'] else 'ok'
|
|
157
|
+
else:
|
|
158
|
+
stale_pct, stale_v = None, 'n/a'
|
|
159
|
+
|
|
160
|
+
metrics = [
|
|
161
|
+
{'name': 'orphan_rate', 'value': orphan_pct, 'unit': '%',
|
|
162
|
+
'verdict': verdict_orphan(orphan_pct, th),
|
|
163
|
+
'detail': f"{len(orphans)} of {n} pages have no inbound link",
|
|
164
|
+
'examples': orphans[:5]},
|
|
165
|
+
{'name': 'avg_degree', 'value': degree, 'unit': 'links/page',
|
|
166
|
+
'verdict': verdict_degree(degree, th),
|
|
167
|
+
'detail': f"{len(edges)} resolved links over {n} pages"},
|
|
168
|
+
{'name': 'main_component', 'value': main_pct, 'unit': '%',
|
|
169
|
+
'verdict': 'ok' if main_pct >= th['component_ok'] else 'warn',
|
|
170
|
+
'detail': f"{main} of {n} pages; {n_comp} component{'' if n_comp == 1 else 's'}"},
|
|
171
|
+
{'name': 'stale_concepts',
|
|
172
|
+
'value': stale_pct, 'unit': '%',
|
|
173
|
+
'verdict': stale_v,
|
|
174
|
+
'detail': (f"{len(stale)} of {len(concepts)} concept pages older than "
|
|
175
|
+
f"{int(th['stale_days'])} days (as of {today.isoformat()})"
|
|
176
|
+
+ (f"; {len(concepts) - len(dated)} undated" if len(dated) < len(concepts) else '')
|
|
177
|
+
if concepts else "no concept pages"),
|
|
178
|
+
'examples': stale[:5]},
|
|
179
|
+
]
|
|
180
|
+
worst = max(RANK[m['verdict']] for m in metrics)
|
|
181
|
+
return {'notes_dir': str(notes_dir), 'pages': n, 'metrics': metrics,
|
|
182
|
+
'verdict': ['ok', 'warn', 'fail'][worst], 'exit_code': worst}
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
def render(report):
|
|
186
|
+
lines = [f"{report['pages']} pages in {report['notes_dir']}", '']
|
|
187
|
+
for m in report['metrics']:
|
|
188
|
+
val = 'n/a' if m['value'] is None else f"{m['value']}{'%' if m['unit'] == '%' else ''}"
|
|
189
|
+
lines.append(f"{m['verdict'].upper():<5} {m['name']:<15} {val:>7} {m['detail']}")
|
|
190
|
+
lines += ['', f"verdict: {report['verdict']}"]
|
|
191
|
+
return '\n'.join(lines)
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
def main(argv=None):
|
|
195
|
+
ap = argparse.ArgumentParser(prog='sgm-stats', description=__doc__.split('\n\n')[0])
|
|
196
|
+
ap.add_argument('notes_dir')
|
|
197
|
+
ap.add_argument('--json', action='store_true', help='machine-readable output')
|
|
198
|
+
ap.add_argument('--today', type=datetime.date.fromisoformat,
|
|
199
|
+
help='date to measure staleness against (YYYY-MM-DD; default: today)')
|
|
200
|
+
for key, val in DEFAULTS.items():
|
|
201
|
+
ap.add_argument('--threshold-' + key.replace('_', '-'), dest=key, type=float,
|
|
202
|
+
default=val, help=f'default {val}')
|
|
203
|
+
a = ap.parse_args(argv)
|
|
204
|
+
if not Path(a.notes_dir).is_dir():
|
|
205
|
+
ap.error(f"not a directory: {a.notes_dir}")
|
|
206
|
+
th = {k: getattr(a, k) for k in DEFAULTS}
|
|
207
|
+
try:
|
|
208
|
+
report = compute(a.notes_dir, today=a.today, th=th)
|
|
209
|
+
except EmptyVault as e:
|
|
210
|
+
print(e, file=sys.stderr)
|
|
211
|
+
return 2
|
|
212
|
+
print(json.dumps(report, indent=2, ensure_ascii=False) if a.json else render(report))
|
|
213
|
+
return report['exit_code']
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
if __name__ == '__main__':
|
|
217
|
+
sys.exit(main())
|
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: sqlite-graph-memory
|
|
3
|
-
Version: 0.3.
|
|
3
|
+
Version: 0.3.2
|
|
4
4
|
Summary: Associative memory for LLM agents: Graph RAG over a folder of markdown notes, in one SQLite file, with no server to run.
|
|
5
5
|
Project-URL: Homepage, https://github.com/tonydzi/sqlite-graph-memory
|
|
6
6
|
Project-URL: Repository, https://github.com/tonydzi/sqlite-graph-memory
|
|
7
7
|
Project-URL: Changelog, https://github.com/tonydzi/sqlite-graph-memory/blob/main/CHANGELOG.md
|
|
8
8
|
Project-URL: Issues, https://github.com/tonydzi/sqlite-graph-memory/issues
|
|
9
|
-
Author: Anton Dziatkovskii
|
|
9
|
+
Author: Anton Dziatkovskii, Vardhman Gupta
|
|
10
10
|
License: MIT License
|
|
11
11
|
|
|
12
12
|
Copyright (c) 2026 Palo Alto AI Research Lab
|
|
@@ -44,22 +44,81 @@ Requires-Python: >=3.9
|
|
|
44
44
|
Requires-Dist: numpy>=1.24
|
|
45
45
|
Provides-Extra: embeddings
|
|
46
46
|
Requires-Dist: numpy<2; (sys_platform == 'darwin' and platform_machine == 'x86_64') and extra == 'embeddings'
|
|
47
|
+
Requires-Dist: sentence-transformers<5; (sys_platform == 'darwin' and platform_machine == 'x86_64') and extra == 'embeddings'
|
|
47
48
|
Requires-Dist: sentence-transformers>=2.6; extra == 'embeddings'
|
|
49
|
+
Requires-Dist: transformers<4.50; (sys_platform == 'darwin' and platform_machine == 'x86_64') and extra == 'embeddings'
|
|
48
50
|
Description-Content-Type: text/markdown
|
|
49
51
|
|
|
50
|
-
# sqlite-graph-memory
|
|
52
|
+
# sqlite-graph-memory: persistent memory for Claude Code, Cursor and Codex over your own markdown notes
|
|
53
|
+
|
|
54
|
+
<!-- mcp-name: io.github.tonydzi/sqlite-graph-memory -->
|
|
55
|
+
|
|
56
|
+
[](https://pypi.org/project/sqlite-graph-memory/) [](https://github.com/tonydzi/sqlite-graph-memory/actions/workflows/tests.yml) [](https://registry.modelcontextprotocol.io/v0/servers?search=sqlite-graph-memory) [](LICENSE)
|
|
57
|
+
|
|
58
|
+
Long-term memory for coding agents that lives in the notes you already keep: an MCP server
|
|
59
|
+
and a CLI over any folder of markdown, Obsidian vaults included. `pip install`, point it at
|
|
60
|
+
the folder, and Claude Code, Cursor, Codex or any other MCP client can recall from it.
|
|
51
61
|
|
|
52
62
|
**Your agent forgets everything between sessions, and plain vector search gives it back
|
|
53
|
-
flat, unrelated chunks. This gives it associative recall over your own markdown notes
|
|
54
|
-
|
|
63
|
+
flat, unrelated chunks. This gives it associative recall over your own markdown notes:
|
|
64
|
+
vector search finds the entry points, the `[[wikilinks]]` you already wrote are the graph.
|
|
65
|
+
One SQLite file, no graph database, no ETL pass, no server.**
|
|
66
|
+
|
|
67
|
+
## Three commands
|
|
55
68
|
|
|
56
69
|
```bash
|
|
57
|
-
pip install sqlite-graph-memory
|
|
70
|
+
pip install 'sqlite-graph-memory[embeddings]'
|
|
71
|
+
sgm-index ./notes
|
|
72
|
+
sgm-ask --ab "In which programming language is the tool behind Project Ouzel written?"
|
|
58
73
|
```
|
|
59
74
|
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
75
|
+
`./notes` is any folder of markdown notes. To get exactly the output below, use the
|
|
76
|
+
202-note synthetic vault from this repo (it is not inside the pip package):
|
|
77
|
+
|
|
78
|
+
```bash
|
|
79
|
+
git clone --depth 1 https://github.com/tonydzi/sqlite-graph-memory && cp -R sqlite-graph-memory/bench/vault ./notes
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
Real output, top 5 of 12:
|
|
83
|
+
|
|
84
|
+
```
|
|
85
|
+
--- DIRECT memory (vector) ---
|
|
86
|
+
1. [ 1.66] project-ouzel
|
|
87
|
+
2. [ -1.33] project-avocet
|
|
88
|
+
3. [ -1.79] tool-cobbleset
|
|
89
|
+
4. [ -2.44] meeting-2025-10-21-lighthouse-sync
|
|
90
|
+
5. [ -2.70] person-severin-ibarrondo
|
|
91
|
+
|
|
92
|
+
--- ASSOCIATIVE memory (vector+graph) ---
|
|
93
|
+
1. [ 1.66] project-ouzel
|
|
94
|
+
2. [ 1.14] tool-lanternfish <== +graph
|
|
95
|
+
3. [ -1.33] project-avocet
|
|
96
|
+
4. [ -1.79] tool-cobbleset
|
|
97
|
+
5. [ -2.44] meeting-2025-10-21-lighthouse-sync
|
|
98
|
+
```
|
|
99
|
+
|
|
100
|
+
The answer lives in `tool-lanternfish` ("Lanternfish is written in Elixir"). The question
|
|
101
|
+
never names Lanternfish, so the vector pass does not find it in the top 12 at all.
|
|
102
|
+
`project-ouzel` links to it, so one graph hop puts it second.
|
|
103
|
+
|
|
104
|
+
If that hop found something your vector search missed, a
|
|
105
|
+
[star](https://github.com/tonydzi/sqlite-graph-memory) is how other people with a notes
|
|
106
|
+
folder find this repo. `sgm-ask --ab` asks the same once per machine, in a terminal only;
|
|
107
|
+
`SGM_NO_STAR_PROMPT=1` turns it off.
|
|
108
|
+
|
|
109
|
+
That is the idea in one query, and it is not a blanket win. On this same vault vector-only
|
|
110
|
+
ranks ahead of graph mode overall; the numbers are in [`bench/`](bench/README.md). `--ab`
|
|
111
|
+
exists so you can measure which one wins on your own notes.
|
|
112
|
+
|
|
113
|
+
**From any MCP client** (Claude Desktop, Cursor, Zed, Windsurf): `sgm-mcp` after the install
|
|
114
|
+
above, or `uvx --with 'sqlite-graph-memory[embeddings]' sqlite-graph-memory`. Set
|
|
115
|
+
`BRAIN_INDEX_DIR` to the `index` folder `sgm-index` wrote. Config:
|
|
116
|
+
[`examples/mcp-config.json`](examples/mcp-config.json). Claude Code:
|
|
117
|
+
`claude mcp add sqlite-graph-memory -e BRAIN_INDEX_DIR=/path/to/index -- sgm-mcp`.
|
|
118
|
+
Codex: a `[mcp_servers.sqlite-graph-memory]` table in `~/.codex/config.toml` with
|
|
119
|
+
`command = "sgm-mcp"`. Agents that read files first: [`llms.txt`](llms.txt).
|
|
120
|
+
|
|
121
|
+
## What it is
|
|
63
122
|
|
|
64
123
|
Graph RAG on SQLite for AI agents — a working pilot, not a framework. It is the extracted
|
|
65
124
|
memory layer of a personal "second brain" agent setup: vector search finds the entry
|
|
@@ -144,12 +203,13 @@ an edge table only when hop depth or corpus size demands it.
|
|
|
144
203
|
|
|
145
204
|
- No entity lane. The production setup has an extra retrieval lane over people/project
|
|
146
205
|
cards; it is too entangled with personal data to publish.
|
|
147
|
-
- No incremental indexing
|
|
206
|
+
- No incremental indexing (tracked in the issues). Packaging exists since v0.3.0 (`pip install sqlite-graph-memory`). There IS an eval suite now — see
|
|
148
207
|
[Measure before you swap](#measure-before-you-swap--eval) — but it scores retrieval on
|
|
149
208
|
wikilink-derived gold, which is weak supervision and says so out loud. Test coverage is uneven:
|
|
150
|
-
`index_notes.py`, `brain_ask.py` and `mcp_server.py` have unit tests,
|
|
151
|
-
has
|
|
152
|
-
prose here — that is how this file ended up claiming
|
|
209
|
+
`index_notes.py`, `brain_ask.py` and `mcp_server.py` have unit tests, and
|
|
210
|
+
`turnstate_hook.py` has write-path coverage. Run `pytest -q` for the current count rather
|
|
211
|
+
than trusting a number written in prose here — that is how this file ended up claiming
|
|
212
|
+
"no tests" while seven were passing.
|
|
153
213
|
The status line above says **pilot** on purpose.
|
|
154
214
|
|
|
155
215
|
## Quickstart
|
|
@@ -236,6 +296,26 @@ board with one adapter file and a pull request: [bench/CONTRIBUTING.md](bench/CO
|
|
|
236
296
|
CI recomputes every submitted number from the per-question outputs, so a typed-in number
|
|
237
297
|
cannot get onto the board.
|
|
238
298
|
|
|
299
|
+
## Health check — `sgm-stats`
|
|
300
|
+
|
|
301
|
+
Four numbers about the link graph itself, no model and no index needed. It reads the notes with
|
|
302
|
+
the same filter and the same `[[wikilink]]` parser as recall, so it measures the graph `--graph`
|
|
303
|
+
actually walks. Thresholds follow [second-brain-os metrics](https://github.com/undefined-ui/second-brain-os/blob/main/docs/05-graphs/metrics.md);
|
|
304
|
+
each is a `--threshold-*` flag. Exit code 0 ok, 1 warn, 2 fail, so it can sit in cron.
|
|
305
|
+
|
|
306
|
+
```text
|
|
307
|
+
$ sgm-stats bench/vault --today 2026-10-20
|
|
308
|
+
202 pages in bench/vault
|
|
309
|
+
|
|
310
|
+
FAIL orphan_rate 26.2% 53 of 202 pages have no inbound link
|
|
311
|
+
OK avg_degree 4.65 940 resolved links over 202 pages
|
|
312
|
+
OK main_component 100.0% 202 of 202 pages; 1 component
|
|
313
|
+
WARN stale_concepts 100.0% 12 of 12 concept pages older than 90 days (as of 2026-10-20)
|
|
314
|
+
```
|
|
315
|
+
|
|
316
|
+
The synthetic bench vault fails the orphan check for a plain reason: its 40 meeting and 12 decision notes link out and nothing
|
|
317
|
+
links back to them. `--json` gives the same report with the first orphan and stale page names.
|
|
318
|
+
|
|
239
319
|
## Measure before you swap — `eval/`
|
|
240
320
|
|
|
241
321
|
Every "obvious" upgrade to a RAG pipeline — a newer embedder, a stronger reranker, hybrid
|
|
@@ -297,6 +377,17 @@ of guessing — 97% of the correct `bridge` answers were *already in the candida
|
|
|
297
377
|
cross-encoder was dropping them, because the relation is structural, not textual. Widening
|
|
298
378
|
retrieval, the obvious fix, would have done nothing at all.
|
|
299
379
|
|
|
380
|
+
That lane ships in the package as `src/sqlite_graph_memory/ppr.py`, **off by default**: turn it
|
|
381
|
+
on with `SGM_PPR=1` or `sgm-ask --ppr`, and score it with `python eval/run_eval.py --gold ...
|
|
382
|
+
--ppr`, which adds a third mode next to `vector` and `graph`. The numbers above were measured on
|
|
383
|
+
the graph pool, so the matching command is `sgm-ask --graph --ppr`; `--ppr` alone reorders the
|
|
384
|
+
vector pool, and `--ab` leaves the lane out so its A/B log stays vector against graph. Any failure in it falls back to
|
|
385
|
+
the reranker's order, like the graph hop. Be clear about what that reproduces: the bridge
|
|
386
|
+
numbers above came from the maintainer's home vault and home pipeline on 2026-09-16, and they
|
|
387
|
+
have not been re-run from the published package. The code is the same algorithm with the same
|
|
388
|
+
constants (alpha 0.85, 10 PPR files, top 5 protected), but your vault will give your numbers,
|
|
389
|
+
and `bench/` does not score this lane yet.
|
|
390
|
+
|
|
300
391
|
Seven other changes were rejected by the same ruler, under one rule fixed in advance:
|
|
301
392
|
**a regression worse than 0.01 nDCG@12 on any class reverts the change.**
|
|
302
393
|
|
|
@@ -306,10 +397,10 @@ Seven other changes were rejected by the same ruler, under one rule fixed in adv
|
|
|
306
397
|
| Qwen3-Embedding-0.6B | body Recall 0.767 -> 0.683; reindex 2863 s vs 333 s -> **rejected** |
|
|
307
398
|
| bge-reranker-v2-m3 | −0.011…−0.018 nDCG on every class except English (+0.041) -> **rejected** |
|
|
308
399
|
| Qwen3-Reranker-0.6B | title −0.111 nDCG at 12x the cost -> **rejected** |
|
|
309
|
-
| BM25 + RRF hybrid | temporal −0.035 -> **merged switched off** |
|
|
400
|
+
| BM25 + RRF hybrid | temporal −0.035 -> **merged switched off** (in the home pipeline, not in this package) |
|
|
310
401
|
| heading-path prefix in every chunk | body −0.023, temporal −0.035 -> **reverted** |
|
|
311
402
|
| `sqlite-vec` (vec0) | full scan 11.99 s vs 0.14 s for a plain SQLite table, and its ANN missed 1.2 hits of 60 -> vectors live in SQLite, **the extension does not** |
|
|
312
|
-
| bi-temporal demotion of superseded notes | temporal +0.130 nDCG on a 41-question temporal set, but title −0.012 -> **merged behind a flag, default off** |
|
|
403
|
+
| bi-temporal demotion of superseded notes | temporal +0.130 nDCG on a 41-question temporal set, but title −0.012 -> **merged behind a flag, default off** (in the home pipeline, not in this package) |
|
|
313
404
|
|
|
314
405
|
None of those verdicts is a claim about the models. They are claims about *this corpus with
|
|
315
406
|
this ruler*, which is the only kind of claim a swap decision actually needs — and the reason
|
|
@@ -331,7 +422,7 @@ while the metric is broken never tested the metric.
|
|
|
331
422
|
|
|
332
423
|
## Roadmap
|
|
333
424
|
|
|
334
|
-
**Now — [
|
|
425
|
+
**Now — [latest release](https://github.com/tonydzi/sqlite-graph-memory/releases/latest)** (v0.3.1 on PyPI and in the MCP Registry as of 2026-10-19).
|
|
335
426
|
The pilot as it runs daily: retrieval pipeline, per-turn ledger and A/B telemetry, all in
|
|
336
427
|
`brain_ask.py` and `turnstate_hook.py`, plus the bi-temporal design note. Known defects live in
|
|
337
428
|
the [tracker](https://github.com/tonydzi/sqlite-graph-memory/issues) rather than in footnotes
|
|
@@ -369,9 +460,10 @@ Both are set as constants (`E5_MODEL`, `RERANK_MODEL`) in `brain_ask.py` and sma
|
|
|
369
460
|
to run on a laptop GPU; swap freely, since nothing in the pipeline depends on these
|
|
370
461
|
specific models.
|
|
371
462
|
|
|
372
|
-
##
|
|
463
|
+
## Maintainers
|
|
373
464
|
|
|
374
|
-
|
|
465
|
+
- **Anton Dziatkovskii** ([@tonydzi](https://github.com/tonydzi)): retrieval pipeline, ledger, bench, releases.
|
|
466
|
+
- **Vardhman Gupta** ([@Kaap10](https://github.com/Kaap10)): the MCP server (`sgm-mcp`, PRs #8, #10, #14) and its tests; reviews on `mcp_server.py` route to him via CODEOWNERS.
|
|
375
467
|
|
|
376
468
|
## AI contributors
|
|
377
469
|
|
|
@@ -383,7 +475,7 @@ content** — no decorative credits. Lab-wide policy, one source for every repo:
|
|
|
383
475
|
|
|
384
476
|
## Cite this work
|
|
385
477
|
|
|
386
|
-
The paper about this system — retrieval design, pilot telemetry, and the query-class-stratified evaluation protocol — is published: [DOI 10.5281/zenodo.22639718](https://doi.org/10.5281/zenodo.22639718) (preprint, CC BY 4.0). Machine-readable citation: [CITATION.cff](CITATION.cff).
|
|
478
|
+
The paper about this system — retrieval design, pilot telemetry, and the query-class-stratified evaluation protocol — is published: [DOI 10.5281/zenodo.22639718](https://doi.org/10.5281/zenodo.22639718) (preprint, CC BY 4.0). Machine-readable citation: [CITATION.cff](CITATION.cff) (GitHub's "Cite this repository" button). Academic identity: Anton Dzyatkovsky publishes as **Anton Dziatkovskii** ([ORCID 0000-0001-7408-3054](https://orcid.org/0000-0001-7408-3054)).
|
|
387
479
|
|
|
388
480
|
## License
|
|
389
481
|
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
sqlite_graph_memory/__init__.py,sha256=UjsgczAXDPxFURyM2QPQpy5D1LYuVFJ8x-7KWPMKsHY,1151
|
|
2
|
+
sqlite_graph_memory/brain_ask.py,sha256=v3Xn3uczigPxHvsfzoKtZOHmJdMwDjK9ZdP7HKUHkN4,19161
|
|
3
|
+
sqlite_graph_memory/index_notes.py,sha256=Hmt0_VVJHS-zXhPPw9h2te8iG1W48f1tJ_H0ALt9aLk,5333
|
|
4
|
+
sqlite_graph_memory/mcp_server.py,sha256=rYrEUVVnQ5KfZLGIt1Pcpm_6eGZIR37KDKDVSBtilWU,9848
|
|
5
|
+
sqlite_graph_memory/ppr.py,sha256=L7VSAiliXr8j3PH2hQqKbdgm08BPm6pZkLcckoMxAro,10001
|
|
6
|
+
sqlite_graph_memory/schema.sql,sha256=8aySXKw3Azaxyzqbd76mKTfOaDP7FlpkLXtgwA8ZVK4,2543
|
|
7
|
+
sqlite_graph_memory/turnstate_hook.py,sha256=fypGmUcHQkzlRN74tvIG07lr9uAXpp2GjetqthkuMc8,8641
|
|
8
|
+
sqlite_graph_memory/turnstate_show.py,sha256=IED7fmJQM5VgBNwxmEoNSZtujRmWja-qVPg5ckbWVnY,2909
|
|
9
|
+
sqlite_graph_memory/vault_stats.py,sha256=QYNGck556RDyLIVFH1C864su5tWehep1DHE21sSTi9g,8596
|
|
10
|
+
sqlite_graph_memory-0.3.2.dist-info/METADATA,sha256=ZZNMig7G0ZP5F0vHv7CPLLEXUpvQBWmWx6p6XTic3ME,32837
|
|
11
|
+
sqlite_graph_memory-0.3.2.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
|
|
12
|
+
sqlite_graph_memory-0.3.2.dist-info/entry_points.txt,sha256=ggzirSEagKB7DZnMli9NUDiMuaog1oeBu_NpRPiKmr0,387
|
|
13
|
+
sqlite_graph_memory-0.3.2.dist-info/licenses/LICENSE,sha256=p88nZdrfMUiES3_MpUEwhGmqz2hQFpyMh0k0cqdpfU8,1082
|
|
14
|
+
sqlite_graph_memory-0.3.2.dist-info/RECORD,,
|
{sqlite_graph_memory-0.3.0.dist-info → sqlite_graph_memory-0.3.2.dist-info}/entry_points.txt
RENAMED
|
@@ -2,5 +2,7 @@
|
|
|
2
2
|
sgm-ask = sqlite_graph_memory.brain_ask:main
|
|
3
3
|
sgm-index = sqlite_graph_memory.index_notes:main
|
|
4
4
|
sgm-mcp = sqlite_graph_memory.mcp_server:main
|
|
5
|
+
sgm-stats = sqlite_graph_memory.vault_stats:main
|
|
5
6
|
sgm-turnstate-hook = sqlite_graph_memory.turnstate_hook:main
|
|
6
7
|
sgm-turnstate-show = sqlite_graph_memory.turnstate_show:main
|
|
8
|
+
sqlite-graph-memory = sqlite_graph_memory.mcp_server:main
|
|
@@ -1,12 +0,0 @@
|
|
|
1
|
-
sqlite_graph_memory/__init__.py,sha256=T5govp5VGbquVPCrt9A206zFxMoSc9cyVZiP-1sMBcM,1151
|
|
2
|
-
sqlite_graph_memory/brain_ask.py,sha256=IU8AVk8VvDJr0XC02a7Foil_f0ui0N-Rr_jD734Onbk,14104
|
|
3
|
-
sqlite_graph_memory/index_notes.py,sha256=Hmt0_VVJHS-zXhPPw9h2te8iG1W48f1tJ_H0ALt9aLk,5333
|
|
4
|
-
sqlite_graph_memory/mcp_server.py,sha256=ncswz2La4JICUmvYhiOM5x9KipxbK7PXjxIyepKtgns,8771
|
|
5
|
-
sqlite_graph_memory/schema.sql,sha256=8aySXKw3Azaxyzqbd76mKTfOaDP7FlpkLXtgwA8ZVK4,2543
|
|
6
|
-
sqlite_graph_memory/turnstate_hook.py,sha256=fypGmUcHQkzlRN74tvIG07lr9uAXpp2GjetqthkuMc8,8641
|
|
7
|
-
sqlite_graph_memory/turnstate_show.py,sha256=IED7fmJQM5VgBNwxmEoNSZtujRmWja-qVPg5ckbWVnY,2909
|
|
8
|
-
sqlite_graph_memory-0.3.0.dist-info/METADATA,sha256=Ac_rhb4QZJz0bbKR2XqErNIKX2IiV3BDP9ilCBlSWzI,27081
|
|
9
|
-
sqlite_graph_memory-0.3.0.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
|
|
10
|
-
sqlite_graph_memory-0.3.0.dist-info/entry_points.txt,sha256=dyQ1KGmf1xgBfDKb3m_2GTXpF4IR4SDGm53oBNZy-CI,280
|
|
11
|
-
sqlite_graph_memory-0.3.0.dist-info/licenses/LICENSE,sha256=p88nZdrfMUiES3_MpUEwhGmqz2hQFpyMh0k0cqdpfU8,1082
|
|
12
|
-
sqlite_graph_memory-0.3.0.dist-info/RECORD,,
|
|
File without changes
|
{sqlite_graph_memory-0.3.0.dist-info → sqlite_graph_memory-0.3.2.dist-info}/licenses/LICENSE
RENAMED
|
File without changes
|