sqlite-graph-memory 0.3.0__py3-none-any.whl

sqlite_graph_memory/mcp_server.py

@@ -0,0 +1,271 @@
+# -*- coding: utf-8 -*-
+"""mcp_server.py — Model Context Protocol (MCP) server for sqlite-graph-memory.
+
+Exposes associative memory recall as a standard MCP tool over stdio transport.
+Compatible with any MCP client (Cursor, Claude Desktop, Antigravity, Zed, Windsurf).
+
+Protocol: JSON-RPC 2.0 over stdio (newline-delimited JSON).
+Zero third-party dependencies required (pure standard library).
+
+USAGE:
+  # Normal stdio mode (started by MCP client)
+  python examples/mcp_server.py
+
+  # Direct test mode (runs a recall query without an MCP client)
+  sgm-mcp --test "how do I think about agent memory"
+
+Config (env, optional):
+  BRAIN_INDEX_DIR   dir holding embedding index (default: ./index)
+  TURNSTATE_DB      SQLite db for telemetry (default: ./turnstate.db)
+  BRAIN_ANSWER_OUT  file to mirror answer into (default: ./_brain_answer.txt)
+"""
+import sys, os, json, subprocess, tempfile
+from pathlib import Path
+
+# Ensure UTF-8 for stdio communication on all platforms (including Windows console)
+try:
+    sys.stdin.reconfigure(encoding='utf-8')
+    sys.stdout.reconfigure(encoding='utf-8')
+    sys.stderr.reconfigure(encoding='utf-8')
+except Exception:
+    pass
+
+# Where the recall subprocess runs. The server deliberately shells out instead
+# of importing brain_ask: a model load that segfaults or runs out of memory then
+# kills a child process and returns an MCP error, rather than taking the server
+# down mid-session.
+ROOT_DIR = Path(__file__).resolve().parent
+
+
+def _brain_ask_cmd():
+    """How to invoke brain_ask — a guaranteed sibling module inside this package."""
+    return [sys.executable, "-m", "sqlite_graph_memory.brain_ask"]
+
+SERVER_INFO = {
+    "name": "sqlite-graph-memory",
+    "version": "0.3.0"
+}
+
+TOOLS = [
+    {
+        "name": "memory_recall",
+        "description": (
+            "Associative memory recall over linked markdown notes. "
+            "Retrieves relevant context using vector search, 1-hop [[wikilink]] graph expansion, "
+            "and cross-encoder reranking. Returns a formatted context bundle."
+        ),
+        "inputSchema": {
+            "type": "object",
+            "properties": {
+                "query": {
+                    "type": "string",
+                    "description": "The search query or topic to retrieve memory for."
+                },
+                "mode": {
+                    "type": "string",
+                    "enum": ["associative", "direct", "ab"],
+                    "default": "associative",
+                    "description": (
+                        "Retrieval mode: 'associative' (vector + graph expansion, recommended), "
+                        "'direct' (vector search only), or 'ab' (run both and log comparison telemetry)."
+                    )
+                }
+            },
+            "required": ["query"]
+        }
+    }
+]
+
+
+def run_recall(query: str, mode: str = "associative") -> tuple[str, bool]:
+    """Invoke brain_ask.py in a crash-safe subprocess and return (text, is_error).
+
+    An empty answer file means no notes matched; stdout is never promoted to an answer
+    because it carries startup notices, load reports, and tokenizer warnings.
+    """
+    base_cmd = _brain_ask_cmd()
+    if base_cmd is None:
+        return (
+            "Error: brain_ask not found. Install it with "
+            "`pip install sqlite-graph-memory[embeddings]`, or run this example "
+            "from a checkout of the repository.",
+            True,
+        )
+
+    if mode not in ("associative", "direct", "ab"):
+        mode = "associative"
+
+    cmd = list(base_cmd)
+    if mode == "ab":
+        cmd.append("--ab")
+    elif mode == "direct":
+        cmd.append("--ask")
+    else:  # associative (default)
+        cmd.extend(["--graph", "--ask"])
+    cmd.append(query)
+
+    with tempfile.NamedTemporaryFile(suffix=".txt", delete=False) as tf:
+        temp_ans_file = Path(tf.name)
+
+    try:
+        call_env = {**os.environ, "BRAIN_ANSWER_OUT": str(temp_ans_file)}
+        res = subprocess.run(
+            cmd,
+            cwd=str(ROOT_DIR),
+            env=call_env,
+            capture_output=True,
+            text=True,
+            encoding="utf-8",
+            errors="replace",
+            timeout=60
+        )
+        if res.returncode != 0:
+            err = (res.stderr or res.stdout or "").strip()
+            return f"Recall error (exit code {res.returncode}):\n{err}", True
+
+        # Read the scoped per-call answer file
+        try:
+            content = temp_ans_file.read_text(encoding="utf-8", errors="replace").strip()
+            if content:
+                return content, False
+        except Exception as e:
+            return f"Recall error: failed to read answer file: {e}", True
+
+        # Empty answer file: return explicit message and flag error so agent does not ingest stdout noise
+        return "(no matching notes found)", True
+    except subprocess.TimeoutExpired:
+        return "Recall error: timed out after 60 seconds.", True
+    except Exception as e:
+        return f"Recall exception: {e}", True
+    finally:
+        if temp_ans_file.exists():
+            try:
+                temp_ans_file.unlink()
+            except Exception:
+                pass
+
+
+def handle_request(req: dict) -> dict:
+    """Handle a single JSON-RPC 2.0 MCP request."""
+    method = req.get("method")
+    req_id = req.get("id")
+
+    if method == "initialize":
+        return {
+            "jsonrpc": "2.0",
+            "id": req_id,
+            "result": {
+                "protocolVersion": "2024-11-05",
+                "capabilities": {
+                    "tools": {}
+                },
+                "serverInfo": SERVER_INFO
+            }
+        }
+
+    elif method == "notifications/initialized":
+        # Notification from client, no response required
+        return None
+
+    elif method == "ping":
+        return {
+            "jsonrpc": "2.0",
+            "id": req_id,
+            "result": {}
+        }
+
+    elif method == "tools/list":
+        return {
+            "jsonrpc": "2.0",
+            "id": req_id,
+            "result": {
+                "tools": TOOLS
+            }
+        }
+
+    elif method == "tools/call":
+        params = req.get("params") or {}
+        tool_name = params.get("name")
+        args = params.get("arguments") or {}
+
+        if tool_name == "memory_recall":
+            query = args.get("query", "")
+            mode = args.get("mode", "associative")
+            if not query:
+                return {
+                    "jsonrpc": "2.0",
+                    "id": req_id,
+                    "result": {
+                        "content": [{"type": "text", "text": "Error: 'query' argument is required."}],
+                        "isError": True
+                    }
+                }
+            result_text, is_error = run_recall(query, mode=mode)
+            return {
+                "jsonrpc": "2.0",
+                "id": req_id,
+                "result": {
+                    "content": [{"type": "text", "text": result_text}],
+                    "isError": is_error
+                }
+            }
+        else:
+            return {
+                "jsonrpc": "2.0",
+                "id": req_id,
+                "error": {
+                    "code": -32601,
+                    "message": f"Tool not found: {tool_name}"
+                }
+            }
+
+    else:
+        if req_id is not None:
+            return {
+                "jsonrpc": "2.0",
+                "id": req_id,
+                "error": {
+                    "code": -32601,
+                    "message": f"Method not found: {method}"
+                }
+            }
+        return None
+
+
+def main():
+    # Direct CLI testing mode: sgm-mcp --test "query"
+    if len(sys.argv) > 1 and sys.argv[1] in ("--test", "-t", "--query"):
+        flags = {"--test", "-t", "--query", "--direct", "--ab", "--associative"}
+        mode = "associative"
+        if "--direct" in sys.argv:
+            mode = "direct"
+        elif "--ab" in sys.argv:
+            mode = "ab"
+        query_parts = [a for a in sys.argv[1:] if a not in flags]
+        q = " ".join(query_parts) if query_parts else "agent memory"
+        text, _ = run_recall(q, mode=mode)
+        print(f"=== MCP Test Query: {q} (mode={mode}) ===")
+        print(text)
+        return
+
+
+    # Standard MCP stdio loop
+    for line in sys.stdin:
+        line = line.strip()
+        if not line:
+            continue
+        try:
+            req = json.loads(line)
+        except Exception as e:
+            sys.stderr.write(f"JSON decode error: {e}\n")
+            continue
+
+        resp = handle_request(req)
+        if resp is not None:
+            sys.stdout.write(json.dumps(resp, ensure_ascii=False) + "\n")
+            sys.stdout.flush()
+
+
+if __name__ == "__main__":
+    main()
+

sqlite_graph_memory/schema.sql

@@ -0,0 +1,45 @@
+-- SQLite schema for the graph-memory pilot.
+-- One file (turnstate.db by default) holds both the per-turn ledger and the
+-- retrieval A/B telemetry. Both tables are auto-created by the scripts;
+-- this file is documentation, not a required migration.
+
+-- Per-turn semantic state ledger (written by turnstate_hook.py after every
+-- assistant turn; 0 LLM tokens, pure transcript parsing).
+CREATE TABLE IF NOT EXISTS turns(
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    session_id TEXT,          -- agent session id
+    ts TEXT,                  -- wall-clock 'YYYY-MM-DD HH:MM:SS'
+    cwd TEXT,                 -- working directory of the session
+    project TEXT,             -- basename(cwd), cheap grouping key
+    ask TEXT,                 -- last real user message in the turn (clipped)
+    summary TEXT,             -- assistant text of the turn (clipped)
+    files TEXT,               -- JSON array: files touched via Write/Edit tools
+    tools TEXT,               -- JSON array: tool names used this turn
+    commands TEXT,            -- JSON array: shell commands run (clipped)
+    decisions TEXT,           -- JSON array: lines matching the decision regex
+    evidence TEXT,            -- path to the raw transcript (.jsonl) = source of truth
+    byte_start INTEGER,       -- transcript byte range this row was derived from
+    byte_end INTEGER
+);
+CREATE INDEX IF NOT EXISTS ix_turns_sid ON turns(session_id);
+
+-- Retrieval A/B telemetry (written by brain_ask.py --ab).
+-- Each row = one real query answered by BOTH pipelines:
+--   A "Direct"      = vector retrieve -> rerank
+--   B "Associative" = vector + 1-hop wikilink graph expansion -> rerank
+CREATE TABLE IF NOT EXISTS ab_recall(
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    ts TEXT,
+    query TEXT,
+    cand_added INTEGER,           -- graph neighbours added to the candidate pool
+    new_in_top INTEGER,           -- notes in B's top-N that A's top-N missed
+    promoted_via_graph INTEGER,   -- of those, how many arrived via the graph hop
+    promoted_titles TEXT,         -- '; '-joined titles (clipped) for eyeballing
+    source TEXT                   -- 'work' = live usage, or a batch/eval label
+);
+
+-- The graph itself is deliberately NOT materialized in SQL in this pilot:
+-- edges are [[wikilinks]] parsed from the markdown notes at query time
+-- (bounded: 1 hop from top-15 hits, max 40 neighbours). Zero maintenance,
+-- always in sync with the notes. Materialize an edges(src, dst) table only
+-- when/if hop depth or corpus size makes query-time parsing too slow.

sqlite_graph_memory/turnstate_hook.py

@@ -0,0 +1,243 @@
+# -*- coding: utf-8 -*-
+"""
+Stop hook — PER-TURN SEMANTIC STATE LEDGER (Phase 1 of always-on agent memory).
+
+After EVERY assistant turn, write a small DETERMINISTIC "what just happened" row to a
+SQLite ledger (TurnState). 0 LLM tokens: everything is extracted by parsing the transcript
+tail (user ask, assistant summary, files touched, tools used, commands, decision/TODO lines,
+evidence pointer back to the raw transcript). The raw .jsonl stays the immutable source;
+this is the cheap DERIVED working-state (distilled, not a replacement).
+
+Written for Claude Code's Stop-hook protocol (JSON event on stdin with session_id,
+transcript_path, cwd), but the pattern ports to any agent harness that exposes a
+per-turn transcript.
+
+Design principles:
+- pure stdlib (sqlite3 + json + re), 0 deps, milliseconds.
+- INCREMENTAL: per-session byte offset in %temp% -> reads only NEW lines each turn.
+- NEVER breaks a session: ANY error -> exit 0, no output. Outputs nothing on success either
+  (a Stop hook that prints structured JSON could alter control flow; we only WRITE to SQLite).
+- LLM extraction is deliberately NOT here (cost gate). If the deterministic delta proves
+  too thin, a cheap model extractor can be added in a later pass.
+
+Config (env, optional):
+  TURNSTATE_DB  path to the ledger db (default: ~/.claude/turnstate/turnstate.db)
+"""
+import sys, os, json, re, sqlite3, tempfile, time
+
+
+def db_path():
+    env = os.getenv("TURNSTATE_DB")
+    if env:
+        return env
+    base = os.path.join(os.path.expanduser("~"), ".claude", "turnstate")
+    try:
+        os.makedirs(base, exist_ok=True)
+        return os.path.join(base, "turnstate.db")
+    except Exception:
+        return os.path.join(tempfile.gettempdir(), "turnstate.db")
+
+CLIP_ASK = 500
+CLIP_SUMMARY = 1200
+CLIP_CMD = 200
+MAX_NEW_BYTES = 4 * 1024 * 1024   # don't read more than 4MB of new tail in one turn
+
+# Lines that look like a decision / commitment / next step.
+# The alternation is DELIBERATELY BILINGUAL (English + Russian) because the sessions this was
+# built on are: a turn states its verdict in whichever language the human was using. Dropping
+# the non-English half does not raise an error and does not fail any check -- the ledger simply
+# stops recording decisions for half the turns, which looks exactly like a quiet week.
+# Add your own language here rather than translating these away.
+DECISION_RX = re.compile(
+    r"(реш(?:ил|ено|аем)|дел(?:аем|ать)\b|не дел(?:аем|ать)|выбра(?:л|ли|ем)|вердикт|"
+    r"\btodo\b|to-?do|next step|следующий шаг|\bвыбор\b|"
+    r"decision|decided|we (?:will|should)|рекомендаци)",
+    re.IGNORECASE)
+
+FILE_TOOLS = {"Write", "Edit", "MultiEdit", "NotebookEdit"}
+
+
+def text_of(ev):
+    msg = ev.get("message") or {}
+    c = msg.get("content")
+    if isinstance(c, str):
+        return c.strip()
+    if isinstance(c, list):
+        out = []
+        for b in c:
+            if isinstance(b, dict) and b.get("type") == "text":
+                out.append(b.get("text") or "")
+        return " ".join(p.strip() for p in out if p).strip()
+    return ""
+
+
+def is_real_user_text(t):
+    if not t or len(t) < 3:
+        return False
+    head = t.lstrip()[:40].lower()
+    if head.startswith("<") or "system-reminder" in head or "<command-" in head:
+        return False
+    if "caveat:" in head or "local-command" in head:
+        return False
+    return True
+
+
+def tool_uses(ev):
+    """Yield (name, input_dict) for tool_use blocks in an assistant event."""
+    msg = ev.get("message") or {}
+    c = msg.get("content")
+    if isinstance(c, list):
+        for b in c:
+            if isinstance(b, dict) and b.get("type") == "tool_use":
+                yield (b.get("name") or "", b.get("input") or {})
+
+
+def offset_file(sid):
+    sid = "".join(ch for ch in str(sid or "unknown") if ch.isalnum() or ch in "-_")[:80]
+    return os.path.join(tempfile.gettempdir(), "claude-turnstate-%s.off" % (sid or "unknown"))
+
+
+def read_offset(sid, fsize):
+    try:
+        with open(offset_file(sid), "r") as f:
+            off = int(f.read().strip())
+        if 0 <= off <= fsize:
+            return off
+    except Exception:
+        pass
+    return 0
+
+
+def write_offset(sid, off):
+    try:
+        with open(offset_file(sid), "w") as f:
+            f.write(str(off))
+    except Exception:
+        pass
+
+
+def ensure_db(con):
+    con.execute("""CREATE TABLE IF NOT EXISTS turns(
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        session_id TEXT, ts TEXT, cwd TEXT, project TEXT,
+        ask TEXT, summary TEXT,
+        files TEXT, tools TEXT, commands TEXT, decisions TEXT,
+        evidence TEXT, byte_start INTEGER, byte_end INTEGER
+    )""")
+    con.execute("CREATE INDEX IF NOT EXISTS ix_turns_sid ON turns(session_id)")
+    con.commit()
+
+
+def main():
+    raw = sys.stdin.buffer.read()
+    data = json.loads(raw.decode("utf-8", errors="replace"))
+
+    sid = str(data.get("session_id") or "")
+    tpath = data.get("transcript_path") or ""
+    cwd = data.get("cwd") or ""
+    if (not tpath or not os.path.isfile(tpath)) and sid:
+        # fallback: some hook events may omit transcript_path.
+        # locate the session's transcript by id under ~/.claude/projects/*/<sid>.jsonl
+        import glob as _glob
+        cands = _glob.glob(os.path.join(os.path.expanduser("~"), ".claude",
+                                        "projects", "*", sid + ".jsonl"))
+        if cands:
+            tpath = max(cands, key=os.path.getmtime)
+    if not tpath or not os.path.isfile(tpath):
+        return
+
+    fsize = os.path.getsize(tpath)
+    start = read_offset(sid, fsize)
+    if fsize - start > MAX_NEW_BYTES:        # huge gap -> skip body, just advance
+        write_offset(sid, fsize)
+        return
+    if fsize <= start:                        # nothing new
+        return
+
+    with open(tpath, "r", encoding="utf-8", errors="ignore") as fh:
+        fh.seek(start)
+        chunk = fh.read()
+    write_offset(sid, fsize)                   # advance regardless, never re-process
+
+    ask = summary = ""
+    files, tools, commands, decisions = [], [], [], []
+    summary_parts = []
+    for line in chunk.splitlines():
+        line = line.strip()
+        if not line:
+            continue
+        try:
+            ev = json.loads(line)
+        except Exception:
+            continue
+        t = ev.get("type")
+        if t == "user":
+            txt = text_of(ev)
+            if is_real_user_text(txt):
+                ask = txt           # last real user msg in this slice = the turn's ask
+        elif t == "assistant":
+            txt = text_of(ev)
+            if txt:
+                summary_parts.append(txt)
+            for name, inp in tool_uses(ev):
+                if name:
+                    tools.append(name)
+                if name in FILE_TOOLS:
+                    fp = inp.get("file_path") or inp.get("notebook_path")
+                    if fp:
+                        files.append(fp)
+                if name == "Bash":
+                    cmd = (inp.get("command") or "").strip()
+                    if cmd:
+                        commands.append(cmd[:CLIP_CMD])
+
+    summary = "\n".join(summary_parts).strip()
+    for ln in summary.splitlines():
+        ln = ln.strip(" -*#>").strip()
+        if 8 <= len(ln) <= 180 and DECISION_RX.search(ln):
+            decisions.append(ln)
+
+    # nothing meaningful happened (e.g. pure tool-result turn) -> skip writing noise
+    if not (ask or summary or files or commands):
+        return
+
+    def uniq(seq):
+        seen, out = set(), []
+        for x in seq:
+            if x not in seen:
+                seen.add(x); out.append(x)
+        return out
+
+    row = (
+        sid,
+        time.strftime("%Y-%m-%d %H:%M:%S"),
+        cwd,
+        os.path.basename(cwd.rstrip("\\/")) if cwd else "",
+        ask[:CLIP_ASK],
+        summary[:CLIP_SUMMARY],
+        json.dumps(uniq(files), ensure_ascii=False),
+        json.dumps(uniq(tools), ensure_ascii=False),
+        json.dumps(commands[:20], ensure_ascii=False),
+        json.dumps(uniq(decisions)[:10], ensure_ascii=False),
+        tpath,
+        start,
+        fsize,
+    )
+    con = sqlite3.connect(db_path(), timeout=3.0)
+    try:
+        ensure_db(con)
+        con.execute(
+            "INSERT INTO turns(session_id,ts,cwd,project,ask,summary,files,tools,"
+            "commands,decisions,evidence,byte_start,byte_end) "
+            "VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?)", row)
+        con.commit()
+    finally:
+        con.close()
+
+
+if __name__ == "__main__":
+    try:
+        main()
+    except Exception:
+        pass   # never break a session
+    sys.exit(0)

sqlite_graph_memory/turnstate_show.py

@@ -0,0 +1,85 @@
+# -*- coding: utf-8 -*-
+"""
+Viewer for the per-turn semantic-state ledger (TurnState).
+Read-only. 0 tokens.
+
+Usage:
+  python turnstate_show.py                 # last 15 turns (all sessions)
+  python turnstate_show.py --session <id>  # one session
+  python turnstate_show.py --n 40          # last N
+  python turnstate_show.py --stats         # totals
+
+Config (env, optional):
+  TURNSTATE_DB  path to the ledger db (default: ~/.claude/turnstate/turnstate.db)
+"""
+import sqlite3, json, os, sys, argparse
+
+DB = os.getenv("TURNSTATE_DB") or os.path.join(
+    os.path.expanduser("~"), ".claude", "turnstate", "turnstate.db")
+
+try:
+    sys.stdout.reconfigure(encoding="utf-8")
+except Exception:
+    pass
+
+
+def jl(s):
+    try:
+        return json.loads(s or "[]")
+    except Exception:
+        return []
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--session")
+    ap.add_argument("--n", type=int, default=15)
+    ap.add_argument("--stats", action="store_true")
+    a = ap.parse_args()
+
+    if not os.path.isfile(DB):
+        print("(no ledger yet:", DB, ")"); return
+    c = sqlite3.connect(DB)
+
+    if a.stats:
+        tot = c.execute("select count(*) from turns").fetchone()[0]
+        sess = c.execute("select count(distinct session_id) from turns").fetchone()[0]
+        files = c.execute("select count(*) from turns where files != '[]'").fetchone()[0]
+        print("ledger:", DB)
+        print("turns: %d | sessions: %d | turns-that-touched-files: %d" % (tot, sess, files))
+        for sid, n, last in c.execute(
+                "select session_id,count(*),max(ts) from turns group by session_id "
+                "order by max(ts) desc limit 12"):
+            print("  %-38s %4d turns  last %s" % (sid[:38], n, last))
+        return
+
+    q = ("select ts,project,ask,summary,files,tools,commands,decisions,session_id "
+         "from turns ")
+    args = ()
+    if a.session:
+        q += "where session_id=? "; args = (a.session,)
+    q += "order by id desc limit ?"; args = args + (a.n,)
+
+    rows = c.execute(q, args).fetchall()
+    for ts, project, ask, summary, files, tools, commands, decisions, sid in reversed(rows):
+        print("=" * 78)
+        print("🕒 %s   [%s]   sid %s" % (ts, project, sid[:12]))
+        if ask:
+            print("❓ ask: %s" % ask.replace("\n", " ")[:160])
+        f, t, cm, d = jl(files), jl(tools), jl(commands), jl(decisions)
+        if f:
+            print("📝 files: %s" % ", ".join(os.path.basename(x) for x in f[:6]))
+        if t:
+            print("🔧 tools: %s" % ", ".join(t[:12]))
+        if cm:
+            print("⌨  cmds: %d (%s ...)" % (len(cm), (cm[0][:60] if cm else "")))
+        if d:
+            print("✅ decisions:")
+            for x in d[:6]:
+                print("     - %s" % x)
+        if summary:
+            print("💬 summary: %s" % summary.replace("\n", " ")[:200])
+
+
+if __name__ == "__main__":
+    main()