engram-vault 0.1.0__tar.gz

engram_vault-0.1.0/PKG-INFO

@@ -0,0 +1,11 @@
+Metadata-Version: 2.4
+Name: engram-vault
+Version: 0.1.0
+Summary: Engram — local-first personal memory layer for AI agents: markdown vault + hybrid retrieval, exposed over MCP.
+Requires-Python: >=3.11
+Requires-Dist: mcp[cli]>=1.27.1
+Requires-Dist: python-ulid>=3.1.0
+Requires-Dist: pyyaml>=6.0.3
+Requires-Dist: fastembed>=0.4.0
+Requires-Dist: hnswlib>=0.8.0
+Requires-Dist: numpy>=2.4.6

engram_vault-0.1.0/backfill.py

@@ -0,0 +1,142 @@
+"""One-time backfill: embed the vault and build the FastKB index.
+
+Root cause of ticket #1: add() embedded a query vector to FIND neighbors but
+never PERSISTED the new fact's own vector, so the index only ever grew via the
+UI's transformers.js path (16 of 43.7k facts). This rebuilds from scratch by
+embedding every fact with the server's model (BAAI/bge-small-en-v1.5).
+
+Two phases, deliberately separated:
+  PHASE 1 — embed + append (streaming, resumable, O(N)). Vectors are appended to
+    vectors.f32 and metadata to manifest.jsonl in batches. A kill/crash keeps
+    progress; a re-run truncates vectors.f32 to the manifest row count (repairing
+    any half-written batch) and continues. NO hnsw here — building it per-batch
+    is what made the old version go O(N^2) (re-mmap + re-save the growing index
+    every batch, rate collapsing 72->11/s).
+  PHASE 2 — build hnsw.bin ONCE from the full vectors.f32 (a few seconds). It is
+    fully rebuildable from vectors+manifest, so it is fine to drop and redo.
+
+TEXT_CAP: bge-small only consumes the first 512 tokens, so we truncate each fact
+before embedding — lossless for the model, but huge scraped raw/ pages no longer
+cost seconds each to tokenize.
+
+Usage:
+  KB_DIR="$HOME/Library/Application Support/KB" \
+    uv run --project server python server/backfill.py [--limit N]
+        [--dirs knowledge,events,...] [--fresh] [--batch 512]
+"""
+from __future__ import annotations
+import argparse, json, sys, time
+from pathlib import Path
+
+import numpy as np
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+import server  # noqa
+import fastkb  # noqa
+
+TEXT_CAP = 2000  # chars; ~512 tokens — the model truncates past this anyway
+
+
+def fact_text(f: dict) -> str:
+    return (f["content"] + " " + " ".join(f.get("tags") or []))[:TEXT_CAP]
+
+
+def _read_manifest(fk) -> list[dict]:
+    if not fk.manifest_path.exists():
+        return []
+    out = []
+    with fk.manifest_path.open() as f:
+        for line in f:
+            line = line.strip()
+            if line:
+                out.append(json.loads(line))
+    return out
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--limit", type=int, default=0)
+    ap.add_argument("--dirs", type=str, default="")
+    ap.add_argument("--batch", type=int, default=512)
+    ap.add_argument("--fresh", action="store_true")
+    args = ap.parse_args()
+    allow = set(d.strip() for d in args.dirs.split(",") if d.strip()) or None
+
+    print(f"== backfill KB={server.KB} cap={TEXT_CAP} batch={args.batch} ==", flush=True)
+    t0 = time.perf_counter()
+    facts = [f for f in server._iter_facts() if (allow is None or f.get("dir") in allow)]
+    if args.limit:
+        facts = facts[: args.limit]
+    print(f"walked {len(facts)} facts in {time.perf_counter()-t0:.1f}s", flush=True)
+
+    fk = fastkb.FastKB(server.KB)
+    dim = fk.dim
+    if args.fresh:
+        fk.vec_path.write_bytes(b""); fk.manifest_path.write_text("")
+        if fk.hnsw_path.exists():
+            fk.hnsw_path.unlink()
+
+    # ---- resume: manifest is source of truth; repair vectors.f32 to match ----
+    manifest = _read_manifest(fk)
+    m_rows = len(manifest)
+    if fk.vec_path.exists():
+        good_bytes = m_rows * dim * 4
+        if fk.vec_path.stat().st_size != good_bytes:
+            with open(fk.vec_path, "r+b") as vf:
+                vf.truncate(good_bytes)
+            print(f"repaired vectors.f32 to {m_rows} rows", flush=True)
+    done = {m["id"] for m in manifest}
+    todo = [f for f in facts if f["id"] not in done]
+    print(f"phase1: indexed={m_rows} todo={len(todo)}", flush=True)
+
+    # ---- phase 1: embed + append ----
+    if todo:
+        from fastembed import TextEmbedding
+        emb = TextEmbedding(model_name="BAAI/bge-small-en-v1.5")
+        row = m_rows
+        n_done = 0
+        t_emb = time.perf_counter()
+        with open(fk.vec_path, "ab") as vf, fk.manifest_path.open("a") as mf:
+            for i in range(0, len(todo), args.batch):
+                batch = todo[i : i + args.batch]
+                vecs = np.asarray(
+                    list(emb.embed([fact_text(f) for f in batch], batch_size=args.batch)),
+                    dtype=np.float32,
+                )
+                vf.write(np.ascontiguousarray(vecs).tobytes()); vf.flush()
+                for f in batch:
+                    mf.write(json.dumps(fastkb.FastKB._meta_from_fact(f, row)) + "\n")
+                    row += 1
+                mf.flush()
+                n_done += len(batch)
+                rate = n_done / (time.perf_counter() - t_emb)
+                print(f"  embedded {n_done}/{len(todo)} ({rate:.0f}/s, "
+                      f"{(len(todo)-n_done)/max(rate,1):.0f}s left)", flush=True)
+
+    # ---- phase 2: build hnsw once from the full vector store ----
+    print("phase2: building hnsw index...", flush=True)
+    t_idx = time.perf_counter()
+    import hnswlib
+    manifest = _read_manifest(fk)
+    n = len(manifest)
+    vectors = np.memmap(fk.vec_path, dtype=np.float32, mode="r", shape=(n, dim))
+    idx = hnswlib.Index(space="cosine", dim=dim)
+    idx.init_index(max_elements=n + 8192,
+                   ef_construction=fastkb.HNSW_EF_CONSTRUCTION, M=fastkb.HNSW_M)
+    idx.add_items(np.asarray(vectors), np.arange(n))
+    idx.set_ef(fastkb.HNSW_EF_QUERY)
+    idx.save_index(str(fk.hnsw_path))
+    print(f"phase2: hnsw built ({n} elements) in {time.perf_counter()-t_idx:.1f}s", flush=True)
+
+    by_dir: dict[str, int] = {}
+    for m in manifest:
+        by_dir[m["dir"]] = by_dir.get(m["dir"], 0) + 1
+    print(f"\nDONE: {n} vectors in {time.perf_counter()-t0:.1f}s total")
+    print(f"by dir: {by_dir}")
+    print(f"artifacts: vectors.f32={fk.vec_path.stat().st_size//1024}KB "
+          f"manifest={fk.manifest_path.stat().st_size//1024}KB "
+          f"hnsw={fk.hnsw_path.stat().st_size//1024}KB")
+
+
+if __name__ == "__main__":
+    main()

engram_vault-0.1.0/cli.py

@@ -0,0 +1,155 @@
+#!/usr/bin/env python3
+"""kb — command-line doorway to the personal KB.
+
+The vault is exposed two ways, both calling the same logic in server.py:
+  - MCP  (server.py)  — for agents, over JSON-RPC stdio.
+  - CLI  (this file)  — for shell scripts / connectors, over stdin/stdout.
+
+A bash connector can't speak MCP mid-pipe, so this is how scripts write to
+the brain:
+
+    CANVAS_TOKEN=... ./connectors/canvas.sh | kb ingest --source canvas
+
+Connector contract (see connectors/README.md): a connector prints JSONL to
+stdout, one record per line:
+
+    {"id": "<stable external id>", "title": "...", "body": "...",
+     "tags": ["..."], "dir": "knowledge|events|...", "scope": null}
+
+`kb ingest` is the only trusted writer — connectors just fetch+emit, never
+touch the vault directly. `id` makes re-runs idempotent (a per-source
+manifest in <KB>/.connectors/<source>.json tracks what's been ingested).
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from pathlib import Path
+
+import server as kb  # same vault logic as the MCP server
+
+
+def _emit(obj) -> None:
+    print(json.dumps(obj, ensure_ascii=False))
+
+
+def cmd_add(args) -> None:
+    content = args.content if args.content is not None else sys.stdin.read()
+    if not content.strip():
+        print("kb add: empty content", file=sys.stderr)
+        sys.exit(1)
+    tags = [t.strip() for t in (args.tags or "").split(",") if t.strip()]
+    res = kb.add(
+        content=content,
+        tags=tags or None,
+        source=args.source,
+        dir=args.dir,
+        scope=args.scope,
+    )
+    _emit(res)
+
+
+def cmd_search(args) -> None:
+    res = kb.search(query=args.query, limit=args.limit, scope=args.scope)
+    _emit(res)
+
+
+def _manifest_path(source: str) -> Path:
+    d = kb.KB / ".connectors"
+    d.mkdir(parents=True, exist_ok=True)
+    return d / f"{source}.json"
+
+
+def _record_to_content(rec: dict) -> str:
+    title = (rec.get("title") or "").strip()
+    body = (rec.get("body") or "").strip()
+    if title and not body.lstrip().startswith("#"):
+        return f"# {title}\n\n{body}".rstrip()
+    return (body or title).rstrip()
+
+
+def cmd_ingest(args) -> None:
+    """Read JSONL from stdin and add each new record. Idempotent per-source
+    via a manifest keyed on each record's `id` (falls back to title)."""
+    mpath = _manifest_path(args.source)
+    seen: dict = {}
+    if mpath.exists():
+        try:
+            seen = json.loads(mpath.read_text())
+        except Exception:
+            seen = {}
+
+    new = skipped = errors = 0
+    for line in sys.stdin:
+        line = line.strip()
+        if not line:
+            continue
+        try:
+            rec = json.loads(line)
+        except Exception as e:
+            errors += 1
+            print(f"kb ingest: bad JSON line: {e}", file=sys.stderr)
+            continue
+        ext_id = str(rec.get("id") or rec.get("title") or "").strip()
+        if ext_id and ext_id in seen:
+            skipped += 1
+            continue
+        content = _record_to_content(rec)
+        if not content:
+            errors += 1
+            continue
+        tags = rec.get("tags") or []
+        if isinstance(tags, str):
+            tags = [t.strip() for t in tags.split(",") if t.strip()]
+        try:
+            res = kb.add(
+                content=content,
+                tags=tags or None,
+                source=args.source,
+                dir=rec.get("dir"),
+                scope=args.scope if args.scope is not None else rec.get("scope"),
+                link=args.link,
+            )
+            if ext_id:
+                seen[ext_id] = res.get("id")
+            new += 1
+        except Exception as e:
+            errors += 1
+            print(f"kb ingest: add failed: {e}", file=sys.stderr)
+
+    mpath.write_text(json.dumps(seen, indent=2))
+    _emit({"source": args.source, "new": new, "skipped": skipped, "errors": errors})
+
+
+def main() -> None:
+    p = argparse.ArgumentParser(prog="kb", description="Personal KB CLI")
+    sub = p.add_subparsers(dest="cmd", required=True)
+
+    a = sub.add_parser("add", help="add one fact (content via --content or stdin)")
+    a.add_argument("--content")
+    a.add_argument("--tags", help="comma-separated")
+    a.add_argument("--source")
+    a.add_argument("--dir", help="raw|events|knowledge|skills|thoughts")
+    a.add_argument("--scope")
+    a.set_defaults(func=cmd_add)
+
+    s = sub.add_parser("search", help="substring search, JSON out")
+    s.add_argument("query")
+    s.add_argument("--limit", type=int, default=10)
+    s.add_argument("--scope")
+    s.set_defaults(func=cmd_search)
+
+    i = sub.add_parser("ingest", help="ingest JSONL records from stdin (connector runner)")
+    i.add_argument("--source", required=True)
+    i.add_argument("--scope")
+    i.add_argument("--link", action="store_true",
+                   help="compute auto_related per fact (O(N)/fact — slow for bulk; off by default)")
+    i.set_defaults(func=cmd_ingest)
+
+    args = p.parse_args()
+    args.func(args)
+
+
+if __name__ == "__main__":
+    main()

engram_vault-0.1.0/engram_vault.egg-info/PKG-INFO

@@ -0,0 +1,11 @@
+Metadata-Version: 2.4
+Name: engram-vault
+Version: 0.1.0
+Summary: Engram — local-first personal memory layer for AI agents: markdown vault + hybrid retrieval, exposed over MCP.
+Requires-Python: >=3.11
+Requires-Dist: mcp[cli]>=1.27.1
+Requires-Dist: python-ulid>=3.1.0
+Requires-Dist: pyyaml>=6.0.3
+Requires-Dist: fastembed>=0.4.0
+Requires-Dist: hnswlib>=0.8.0
+Requires-Dist: numpy>=2.4.6

engram_vault-0.1.0/engram_vault.egg-info/SOURCES.txt

@@ -0,0 +1,15 @@
+backfill.py
+cli.py
+entities.py
+fastkb.py
+hybrid.py
+notifyd.py
+pyproject.toml
+reranker.py
+server.py
+engram_vault.egg-info/PKG-INFO
+engram_vault.egg-info/SOURCES.txt
+engram_vault.egg-info/dependency_links.txt
+engram_vault.egg-info/entry_points.txt
+engram_vault.egg-info/requires.txt
+engram_vault.egg-info/top_level.txt

engram_vault-0.1.0/engram_vault.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

engram_vault-0.1.0/engram_vault.egg-info/entry_points.txt

@@ -0,0 +1,3 @@
+[console_scripts]
+engram = cli:main
+engram-vault = server:main

engram_vault-0.1.0/engram_vault.egg-info/requires.txt

@@ -0,0 +1,6 @@
+mcp[cli]>=1.27.1
+python-ulid>=3.1.0
+pyyaml>=6.0.3
+fastembed>=0.4.0
+hnswlib>=0.8.0
+numpy>=2.4.6

engram_vault-0.1.0/engram_vault.egg-info/top_level.txt

@@ -0,0 +1,8 @@
+backfill
+cli
+entities
+fastkb
+hybrid
+notifyd
+reranker
+server

engram_vault-0.1.0/entities.py

@@ -0,0 +1,183 @@
+"""Entity linking — the "knows-you" layer.
+
+Extracts person/project entities from a fact's structured tags + project scope,
+maintains a sidecar entity index (entity -> fact ids), and resolves a free-text
+query to entities so retrieval can boost facts about the people/projects a query
+names ("what do I know about Priya", "nouva graph bug").
+
+Design: entities come from the HIGH-PRECISION signal already in the data —
+`person:`/`contact:` and `project:`/`area:` tags + the project `scope` — not from
+free-text NER over bodies (noisy; a deliberate non-goal here, LLM-upgradable
+later). So this whole layer is deterministic, needs no embedder/LLM, and is
+unit-testable offline.
+
+All functions are pure over a plain dict (`index`), shaped:
+
+    { "person:priya": ["<fact_id>", ...], "project:nouva-desktop": [...], ... }
+
+server.py owns persistence (atomic + file-locked writes beside the vault) so
+concurrent adds don't lose updates; this module never touches the filesystem.
+"""
+from __future__ import annotations
+
+import re
+
+# tag prefix -> entity type. Only these become entities.
+TAG_ENTITY_TYPES = {
+    "person": "person",    # named individuals (explicitly tagged)
+    "contact": "contact",  # chat / conversation identities (iMessage import:
+                           # group chats, phone numbers) — kept SEPARATE from
+                           # person so a few real people aren't buried under
+                           # high-volume chat slugs (mavs-vs-refs, brothaaassss).
+    "project": "project",
+    "area": "project",     # project areas/components (kb-server, embeddings, ...)
+}
+
+# prefixes that LOOK entity-ish but are not: dates, confidence levels, browsing-
+# history domains, status flags. Kept as an explicit denylist so a future tag
+# prefix doesn't silently leak in.
+NON_ENTITY_PREFIXES = frozenset({
+    "day", "conf", "host", "type", "status", "severity", "browser", "source",
+    "topic", "relationship", "group", "benchmark", "tool", "class", "course",
+})
+
+# entity name values that carry no identity — never index them.
+_JUNK_VALUES = frozenset({"", "untitled", "unknown", "none", "null", "na", "n-a"})
+
+_TOK = re.compile(r"[a-z0-9]+")
+# query tokens shorter than this don't resolve entities (avoid "vs"/"the" noise)
+_MIN_NAME_TOKEN = 4
+
+
+def _norm(val: str) -> str:
+    return (val or "").strip().lower()
+
+
+def entity_name(entity: str) -> str:
+    """'person:graham-neubig' -> 'graham-neubig'."""
+    return entity.split(":", 1)[1] if ":" in entity else entity
+
+
+def entity_type(entity: str) -> str:
+    return entity.split(":", 1)[0] if ":" in entity else ""
+
+
+def extract_entities(tags, scope=None) -> list[str]:
+    """Canonical entity ids ('type:name') for a fact, from its tags + scope.
+    Order-preserving and de-duplicated."""
+    out: list[str] = []
+    for t in tags or []:
+        if ":" not in t:
+            continue
+        pre, val = t.split(":", 1)
+        pre = pre.strip().lower()
+        if pre in NON_ENTITY_PREFIXES:
+            continue
+        et = TAG_ENTITY_TYPES.get(pre)
+        if not et:
+            continue
+        val = _norm(val)
+        if val in _JUNK_VALUES:
+            continue
+        out.append(f"{et}:{val}")
+    if scope:
+        sval = _norm(str(scope))
+        if sval not in _JUNK_VALUES:
+            out.append(f"project:{sval}")
+    seen: set[str] = set()
+    res: list[str] = []
+    for e in out:
+        if e not in seen:
+            seen.add(e)
+            res.append(e)
+    return res
+
+
+# ---------------- index mutation (pure over a dict) ----------------
+
+def index_add(index: dict, fact_id: str, entities) -> dict:
+    for e in entities:
+        lst = index.setdefault(e, [])
+        if fact_id not in lst:
+            lst.append(fact_id)
+    return index
+
+
+def index_remove(index: dict, fact_id: str, entities=None) -> dict:
+    """Drop fact_id from the given entities (or from ALL entities if None)."""
+    targets = list(entities) if entities is not None else list(index.keys())
+    for e in targets:
+        lst = index.get(e)
+        if not lst:
+            continue
+        index[e] = [f for f in lst if f != fact_id]
+        if not index[e]:
+            del index[e]
+    return index
+
+
+def rebuild(records) -> dict:
+    """Build an entity index from manifest-shaped records (each with `id`,
+    `tags`, `scope`). Manifest-backed, so a backfill needs no vault walk."""
+    index: dict = {}
+    for r in records:
+        ents = extract_entities(r.get("tags") or [], r.get("scope"))
+        if ents:
+            index_add(index, r["id"], ents)
+    return index
+
+
+# ---------------- query-time resolution ----------------
+
+def _token_map(index: dict) -> dict:
+    """name-token -> {entity ids containing it}. For O(query) resolution."""
+    tm: dict[str, set] = {}
+    for e in index:
+        for tok in _TOK.findall(entity_name(e)):
+            if len(tok) >= _MIN_NAME_TOKEN:
+                tm.setdefault(tok, set()).add(e)
+    return tm
+
+
+def resolve(index: dict, query: str, token_map: dict | None = None) -> list[str]:
+    """Entities a query names: an entity matches when one of its name-tokens
+    (length >= 4) appears in the query. Returns most-specific first (entities
+    matched by more query tokens rank higher)."""
+    qtokens = {t for t in _TOK.findall((query or "").lower()) if len(t) >= _MIN_NAME_TOKEN}
+    if not qtokens:
+        return []
+    tm = token_map if token_map is not None else _token_map(index)
+    hits: dict[str, int] = {}
+    for t in qtokens:
+        for e in tm.get(t, ()):  # type: ignore[union-attr]
+            hits[e] = hits.get(e, 0) + 1
+    return sorted(hits, key=lambda e: (-hits[e], -len(index.get(e, [])), e))
+
+
+def facts_for(index: dict, entities, cap: int | None = None) -> list[str]:
+    """Union of fact ids across the given entities, de-duplicated (first seen)."""
+    seen: set[str] = set()
+    out: list[str] = []
+    for e in entities:
+        for fid in index.get(e, []):
+            if fid not in seen:
+                seen.add(fid)
+                out.append(fid)
+                if cap and len(out) >= cap:
+                    return out
+    return out
+
+
+def summarize(index: dict, type: str | None = None, query: str | None = None, limit: int = 50) -> list[dict]:
+    """[{entity, type, name, count}] for the entities() tool, most-referenced
+    first. Optional `type` filter and `query` name-match."""
+    items = []
+    matched = set(resolve(index, query)) if query else None
+    for e, facts in index.items():
+        if type and entity_type(e) != type:
+            continue
+        if matched is not None and e not in matched:
+            continue
+        items.append({"entity": e, "type": entity_type(e), "name": entity_name(e), "count": len(facts)})
+    items.sort(key=lambda d: (-d["count"], d["entity"]))
+    return items[:limit]