vault-index 0.2.0__tar.gz

vault_index-0.2.0/.gitignore

@@ -0,0 +1,12 @@
+.venv/
+__pycache__/
+*.pyc
+.pytest_cache/
+.ruff_cache/
+.export-stamp
+# Your own MCP config (absolute paths); start from .mcp.json.example
+.mcp.json
+# Raw pre-compaction snapshots contain your prompts
+docs/handoffs/auto/
+.claude/settings.local.json
+dist/

vault_index-0.2.0/PKG-INFO

@@ -0,0 +1,48 @@
+Metadata-Version: 2.5
+Name: vault-index
+Version: 0.2.0
+Summary: Local hybrid (FTS5 + sqlite-vec) search MCP server for markdown vaults and codebases
+Project-URL: Homepage, https://github.com/GitGudatCodeing/claude-second-brain
+Project-URL: Issues, https://github.com/GitGudatCodeing/claude-second-brain/issues
+License-Expression: MIT
+Keywords: claude,markdown,mcp,obsidian,search,sqlite-vec
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Text Processing :: Indexing
+Requires-Python: <3.14,>=3.10
+Requires-Dist: fastembed>=0.4
+Requires-Dist: mcp<2,>=1.2
+Requires-Dist: sqlite-vec>=0.1.6
+Description-Content-Type: text/markdown
+
+# vault-index
+
+<!-- mcp-name: io.github.GitGudatCodeing/vault-index -->
+
+Local hybrid search for markdown vaults and codebases, served as an MCP server. Keyword search (SQLite FTS5) and vector search (sqlite-vec, `bge-small-en-v1.5` on CPU) are merged with reciprocal rank fusion. Everything lives in one SQLite file on your machine; no cloud services.
+
+Part of [claude-second-brain](https://github.com/GitGudatCodeing/claude-second-brain), where every search change is measured against golden question sets and compared with naive grep and basic-memory (see its README for the numbers).
+
+## Use with Claude Code
+
+```bash
+claude mcp add vault-index -- uvx --python 3.12 vault-index
+```
+
+Then ask Claude to index a folder, for example: "index_paths ~/notes". After that, `search_notes` finds paraphrased answers with file path and heading. The first run downloads the embedding model (about 65 MB).
+
+## Tools
+- `index_paths(paths, reindex=False)`: add folders or files as sources and sync them. With no paths, re-syncs every source (new, changed and deleted files).
+- `search_notes(query, limit=8, path_prefix=None)`: hybrid search; returns chunks with file path and heading path.
+- `get_note(path)`: full text of an indexed file.
+- `list_sources()`, `remove_source(path)`.
+
+Markdown is chunked by heading. A `.indexignore` in a source folder lists relative paths to skip.
+
+## Settings
+| Env var | Default |
+|---|---|
+| `VAULT_INDEX_DB` | `~/.mcp-memory/vault-index.db` |
+| `VAULT_MODEL_CACHE` | `~/.mcp-memory/models` |
+
+Python 3.10 to 3.13 (`--python 3.12` keeps uvx off newer Pythons that onnxruntime does not support yet). MIT license.

vault_index-0.2.0/README.md

@@ -0,0 +1,31 @@
+# vault-index
+
+<!-- mcp-name: io.github.GitGudatCodeing/vault-index -->
+
+Local hybrid search for markdown vaults and codebases, served as an MCP server. Keyword search (SQLite FTS5) and vector search (sqlite-vec, `bge-small-en-v1.5` on CPU) are merged with reciprocal rank fusion. Everything lives in one SQLite file on your machine; no cloud services.
+
+Part of [claude-second-brain](https://github.com/GitGudatCodeing/claude-second-brain), where every search change is measured against golden question sets and compared with naive grep and basic-memory (see its README for the numbers).
+
+## Use with Claude Code
+
+```bash
+claude mcp add vault-index -- uvx --python 3.12 vault-index
+```
+
+Then ask Claude to index a folder, for example: "index_paths ~/notes". After that, `search_notes` finds paraphrased answers with file path and heading. The first run downloads the embedding model (about 65 MB).
+
+## Tools
+- `index_paths(paths, reindex=False)`: add folders or files as sources and sync them. With no paths, re-syncs every source (new, changed and deleted files).
+- `search_notes(query, limit=8, path_prefix=None)`: hybrid search; returns chunks with file path and heading path.
+- `get_note(path)`: full text of an indexed file.
+- `list_sources()`, `remove_source(path)`.
+
+Markdown is chunked by heading. A `.indexignore` in a source folder lists relative paths to skip.
+
+## Settings
+| Env var | Default |
+|---|---|
+| `VAULT_INDEX_DB` | `~/.mcp-memory/vault-index.db` |
+| `VAULT_MODEL_CACHE` | `~/.mcp-memory/models` |
+
+Python 3.10 to 3.13 (`--python 3.12` keeps uvx off newer Pythons that onnxruntime does not support yet). MIT license.

vault_index-0.2.0/pyproject.toml

@@ -0,0 +1,42 @@
+[project]
+name = "vault-index"
+version = "0.2.0"
+description = "Local hybrid (FTS5 + sqlite-vec) search MCP server for markdown vaults and codebases"
+readme = "README.md"
+license = "MIT"
+requires-python = ">=3.10,<3.14"
+keywords = ["mcp", "claude", "search", "sqlite-vec", "markdown", "obsidian"]
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "Operating System :: OS Independent",
+    "Topic :: Text Processing :: Indexing",
+]
+dependencies = [
+    "mcp>=1.2,<2",
+    "sqlite-vec>=0.1.6",
+    "fastembed>=0.4",
+]
+
+[project.urls]
+Homepage = "https://github.com/GitGudatCodeing/claude-second-brain"
+Issues = "https://github.com/GitGudatCodeing/claude-second-brain/issues"
+
+[project.scripts]
+vault-index = "vault_index:main"
+
+[build-system]
+requires = ["hatchling>=1.27"]
+build-backend = "hatchling.build"
+
+# The repo runs server.py in place; the wheel ships it as the module vault_index.
+[tool.hatch.build.targets.wheel]
+bypass-selection = true
+
+[tool.hatch.build.targets.wheel.force-include]
+"server.py" = "vault_index.py"
+
+[tool.hatch.build.targets.sdist]
+include = ["server.py", "README.md", "pyproject.toml"]
+
+[tool.uv]
+package = false  # `uv run --project mcp` keeps using the source file, not an installed copy

vault_index-0.2.0/server.py

@@ -0,0 +1,403 @@
+"""Vault index MCP server: hybrid keyword (FTS5) + vector (sqlite-vec) search.
+
+Embeddings come from fastembed (ONNX, CPU). Everything lives in one SQLite file.
+Env:
+  VAULT_INDEX_DB     path to the SQLite file (default ~/.mcp-memory/vault-index.db)
+  VAULT_MODEL_CACHE  fastembed model cache dir (default ~/.mcp-memory/models)
+"""
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import re
+import sqlite3
+import sys
+from pathlib import Path
+
+import sqlite_vec
+from mcp.server.fastmcp import FastMCP
+
+MODEL_NAME = "BAAI/bge-small-en-v1.5"
+DIM = 384
+HOME = Path.home() / ".mcp-memory"
+DB_PATH = Path(os.environ.get("VAULT_INDEX_DB", HOME / "vault-index.db"))
+MODEL_CACHE = Path(os.environ.get("VAULT_MODEL_CACHE", HOME / "models"))
+
+EXTENSIONS = {
+    ".md", ".markdown", ".txt", ".rst",
+    ".py", ".js", ".ts", ".tsx", ".jsx", ".cs", ".java", ".go", ".rs", ".sql",
+    ".json", ".yaml", ".yml", ".toml", ".ps1", ".sh",
+}
+SKIP_DIRS = {
+    ".git", "node_modules", ".venv", "venv", "__pycache__", "bin", "obj",
+    "dist", "build", ".idea", ".vs", ".next", ".obsidian", ".trash",
+}
+MAX_FILE_BYTES = 1_000_000
+CHUNK_CHARS = 1200
+
+mcp = FastMCP("vault-index")
+_embedder = None
+_db: sqlite3.Connection | None = None
+
+
+def log(msg: str) -> None:
+    print(msg, file=sys.stderr, flush=True)
+
+
+def embedder():
+    global _embedder
+    if _embedder is None:
+        from fastembed import TextEmbedding
+
+        MODEL_CACHE.mkdir(parents=True, exist_ok=True)
+        _embedder = TextEmbedding(MODEL_NAME, cache_dir=str(MODEL_CACHE))
+    return _embedder
+
+
+def embed(texts: list[str]) -> list[bytes]:
+    return [sqlite_vec.serialize_float32(v.tolist()) for v in embedder().embed(texts)]
+
+
+def db() -> sqlite3.Connection:
+    global _db
+    if _db is None:
+        DB_PATH.parent.mkdir(parents=True, exist_ok=True)
+        con = sqlite3.connect(DB_PATH, check_same_thread=False, timeout=30)
+        con.execute("PRAGMA journal_mode=WAL")  # the PostToolUse hook writes concurrently
+        con.enable_load_extension(True)
+        sqlite_vec.load(con)
+        con.enable_load_extension(False)
+        con.executescript(
+            f"""
+            CREATE TABLE IF NOT EXISTS sources(path TEXT PRIMARY KEY);
+            CREATE TABLE IF NOT EXISTS files(
+                id INTEGER PRIMARY KEY, path TEXT UNIQUE NOT NULL,
+                mtime REAL NOT NULL, hash TEXT NOT NULL);
+            CREATE TABLE IF NOT EXISTS chunks(
+                id INTEGER PRIMARY KEY, file_id INTEGER NOT NULL,
+                ord INTEGER NOT NULL, heading TEXT NOT NULL, text TEXT NOT NULL);
+            CREATE INDEX IF NOT EXISTS chunks_file ON chunks(file_id);
+            CREATE VIRTUAL TABLE IF NOT EXISTS chunks_fts USING fts5(text, heading);
+            CREATE VIRTUAL TABLE IF NOT EXISTS chunks_vec USING vec0(embedding float[{DIM}]);
+            """
+        )
+        _db = con
+    return _db
+
+
+# ---------- chunking ----------
+
+_HEADING = re.compile(r"^(#{1,6})\s+(.*)$")
+
+
+def chunk_text(text: str, is_markdown: bool) -> list[tuple[str, str]]:
+    """Return (heading_path, chunk_text) pairs."""
+    sections: list[tuple[str, list[str]]] = []
+    stack: list[tuple[int, str]] = []
+    heading, buf = "", []
+    in_fence = False
+    for line in text.splitlines():
+        if line.lstrip().startswith("```"):
+            in_fence = not in_fence
+        m = None if (in_fence or not is_markdown) else _HEADING.match(line)
+        if m:
+            sections.append((heading, buf))
+            level = len(m.group(1))
+            while stack and stack[-1][0] >= level:
+                stack.pop()
+            stack.append((level, m.group(2).strip()))
+            heading, buf = " > ".join(h for _, h in stack), []
+        else:
+            buf.append(line)
+    sections.append((heading, buf))
+
+    out: list[tuple[str, str]] = []
+    for head, lines in sections:
+        body = "\n".join(lines).strip()
+        if not body:
+            continue
+        cur = ""
+        for para in re.split(r"\n\s*\n", body):
+            while len(para) > CHUNK_CHARS * 1.5:  # very long paragraph/code block
+                cut = para.rfind("\n", 0, CHUNK_CHARS)
+                cut = cut if cut > CHUNK_CHARS // 2 else CHUNK_CHARS
+                if cur:
+                    out.append((head, cur))
+                    cur = ""
+                out.append((head, para[:cut].strip()))
+                para = para[cut:]
+            if cur and len(cur) + len(para) > CHUNK_CHARS:
+                out.append((head, cur))
+                cur = ""
+            cur = f"{cur}\n\n{para}" if cur else para
+        if cur.strip():
+            out.append((head, cur.strip()))
+    return [(h, t) for h, t in out if t.strip()]
+
+
+# ---------- indexing ----------
+
+def load_ignores(root: Path) -> list[str]:
+    """Source-relative paths (files or folders) listed in <source>/.indexignore."""
+    f = root / ".indexignore"
+    if not root.is_dir() or not f.is_file():
+        return []
+    lines = (l.split("#", 1)[0].strip().replace("\\", "/").strip("/") for l in f.read_text(encoding="utf-8").splitlines())
+    return [l.lower() for l in lines if l]
+
+
+def is_ignored(p: Path, root: Path, ignores: list[str] | None = None) -> bool:
+    ignores = load_ignores(root) if ignores is None else ignores
+    try:
+        rel = p.relative_to(root).as_posix().lower()
+    except ValueError:
+        return False
+    return any(rel == ig or rel.startswith(ig + "/") for ig in ignores)
+
+
+def iter_files(root: Path):
+    if root.is_file():
+        yield root
+        return
+    ignores = load_ignores(root)
+    for dirpath, dirnames, filenames in os.walk(root):
+        dirnames[:] = [
+            d for d in dirnames if d not in SKIP_DIRS and not is_ignored(Path(dirpath) / d, root, ignores)
+        ]
+        for name in filenames:
+            p = Path(dirpath) / name
+            if p.suffix.lower() in EXTENSIONS and not is_ignored(p, root, ignores):
+                yield p
+
+
+def drop_file(con: sqlite3.Connection, file_id: int) -> None:
+    ids = [r[0] for r in con.execute("SELECT id FROM chunks WHERE file_id=?", (file_id,))]
+    for cid in ids:
+        con.execute("DELETE FROM chunks_fts WHERE rowid=?", (cid,))
+        con.execute("DELETE FROM chunks_vec WHERE rowid=?", (cid,))
+    con.execute("DELETE FROM chunks WHERE file_id=?", (file_id,))
+    con.execute("DELETE FROM files WHERE id=?", (file_id,))
+
+
+def index_file(con: sqlite3.Connection, p: Path, force: bool) -> str:
+    key = str(p)
+    st = p.stat()
+    row = con.execute("SELECT id, mtime, hash FROM files WHERE path=?", (key,)).fetchone()
+    if row and not force and abs(row[1] - st.st_mtime) < 1e-6:
+        return "unchanged"
+    try:
+        text = p.read_text(encoding="utf-8") if st.st_size <= MAX_FILE_BYTES else None
+    except (UnicodeDecodeError, OSError):
+        text = None
+    if text is None:  # too big or unreadable now: never keep serving its old chunks
+        if row:
+            drop_file(con, row[0])
+        return "skipped"
+    digest = hashlib.sha256(text.encode("utf-8")).hexdigest()
+    if row and not force and row[2] == digest:
+        con.execute("UPDATE files SET mtime=? WHERE id=?", (st.st_mtime, row[0]))
+        return "unchanged"
+    chunks = chunk_text(text, p.suffix.lower() in {".md", ".markdown"})
+    # Embed before touching the index, so a failure leaves the old version in place.
+    vecs = embed([f"{h}\n{t}" if h else t for h, t in chunks]) if chunks else []
+    if row:
+        drop_file(con, row[0])
+    cur = con.execute("INSERT INTO files(path, mtime, hash) VALUES(?,?,?)", (key, st.st_mtime, digest))
+    file_id = cur.lastrowid
+    if chunks:
+        for i, ((h, t), v) in enumerate(zip(chunks, vecs, strict=True)):
+            cid = con.execute(
+                "INSERT INTO chunks(file_id, ord, heading, text) VALUES(?,?,?,?)", (file_id, i, h, t)
+            ).lastrowid
+            con.execute("INSERT INTO chunks_fts(rowid, text, heading) VALUES(?,?,?)", (cid, t, h))
+            con.execute("INSERT INTO chunks_vec(rowid, embedding) VALUES(?,?)", (cid, v))
+    return "updated" if row else "added"
+
+
+def sync_source(con: sqlite3.Connection, root: Path, force: bool) -> dict:
+    stats = {"added": 0, "updated": 0, "unchanged": 0, "skipped": 0, "removed": 0}
+    seen: set[str] = set()
+    try:
+        for p in iter_files(root):
+            seen.add(str(p))
+            stats[index_file(con, p, force)] += 1
+        for fid, path in files_under(con, str(root)):
+            if path not in seen:
+                drop_file(con, fid)
+                stats["removed"] += 1
+    except BaseException:
+        con.rollback()  # the connection is long-lived; a later commit must never keep half a file
+        raise
+    con.commit()
+    return stats
+
+
+def files_under(con: sqlite3.Connection, source: str) -> list[tuple[int, str]]:
+    """(id, path) of indexed files at or below `source`. A plain string match: LIKE treats _ and % as wildcards."""
+    prefix = source.rstrip("\\/")
+    return [(fid, path) for fid, path in con.execute("SELECT id, path FROM files").fetchall()
+            if path == prefix or path.startswith(prefix + os.sep)]
+
+
+# ---------- search ----------
+
+STOPWORDS = frozenset("""
+a about after again all am an and any are as at be because been before being both but by can could
+did do does doing don't down during each few for from further had has have having he her here hers
+him his how i if in into is it its itself just let me more most my no nor not of off on once only or
+other our ours out over own same she should so some such than that the their theirs them then there
+these they this those through to too under until up very was we were what when where which while who
+whom why will with would you your yours
+""".split())
+
+
+def fts_query(q: str) -> str | None:
+    toks = re.findall(r"\w+", q)
+    content = [t for t in toks if t.lower() not in STOPWORDS] or toks
+    return " OR ".join(f'"{t}"' for t in content) if content else None
+
+
+@mcp.tool()
+def index_paths(paths: list[str] | None = None, reindex: bool = False) -> str:
+    """Add folders/files to the index (remembered as sources) and sync them.
+
+    With no `paths`, re-syncs every registered source (picks up new, changed and
+    deleted files). `reindex=True` forces re-embedding of everything.
+    """
+    con = db()
+    for raw in paths or []:
+        p = Path(raw).expanduser().resolve()
+        if not p.exists():
+            return f"Path does not exist: {p}"
+        con.execute("INSERT OR IGNORE INTO sources(path) VALUES(?)", (str(p),))
+    con.commit()
+    sources = [r[0] for r in con.execute("SELECT path FROM sources ORDER BY path")]
+    if not sources:
+        return "No sources registered. Call index_paths with one or more paths."
+    targets = [str(Path(x).expanduser().resolve()) for x in paths] if paths else sources
+    lines = []
+    for s in targets:
+        lines.append(f"{s}: {sync_source(con, Path(s), reindex)}")
+    return "\n".join(lines)
+
+
+RRF_K = 60
+# Keyword ranks count at a tenth of vector ranks: a tie-breaker and jargon booster, not an equal vote.
+# Beat equal weights on both golden sets (docs/adr/2026-10-23-fts-weight-in-fusion.md).
+FTS_WEIGHT = 0.1
+
+
+def hybrid_search(
+    query: str, limit: int, path_prefix: str | None = None, mode: str = "hybrid"
+) -> list[tuple[str, str, str, float]]:
+    """Ranked (path, heading, text, score). `mode` is hybrid, fts or vec; the
+    single-sided modes exist so eval/run_eval.py can measure each side's contribution."""
+    con = db()
+    pool = limit * 4
+    ranks: dict[int, float] = {}
+    # With a prefix, both sides rank only chunks under it, so a filtered search never comes back
+    # short just because other folders filled the candidate pool.
+    scope = json.dumps(prefix_chunk_ids(con, path_prefix)) if path_prefix else None
+
+    fq = fts_query(query)
+    if fq and mode in ("hybrid", "fts"):
+        sql = "SELECT rowid FROM chunks_fts WHERE chunks_fts MATCH ?"
+        args: tuple = (fq,)
+        if scope:
+            sql, args = sql + " AND rowid IN (SELECT value FROM json_each(?))", (fq, scope)
+        for rank, (cid,) in enumerate(con.execute(sql + " ORDER BY bm25(chunks_fts) LIMIT ?", (*args, pool))):
+            ranks[cid] = ranks.get(cid, 0) + (FTS_WEIGHT if mode == "hybrid" else 1) / (RRF_K + rank)
+    if mode in ("hybrid", "vec"):
+        qv = embed([query])[0]
+        if scope:  # exact distances over the scoped chunks (vec0's kNN index cannot filter by path)
+            rows = con.execute(
+                "SELECT rowid FROM chunks_vec WHERE rowid IN (SELECT value FROM json_each(?))"
+                " ORDER BY vec_distance_l2(embedding, ?) LIMIT ?", (scope, qv, pool))
+        else:
+            rows = con.execute("SELECT rowid FROM chunks_vec WHERE embedding MATCH ? AND k=? ORDER BY distance", (qv, pool))
+        for rank, (cid,) in enumerate(rows):
+            ranks[cid] = ranks.get(cid, 0) + 1 / (RRF_K + rank)
+
+    results = []
+    for cid, score in sorted(ranks.items(), key=lambda kv: -kv[1])[:limit]:
+        path, heading, text = con.execute(
+            "SELECT f.path, c.heading, c.text FROM chunks c JOIN files f ON f.id=c.file_id WHERE c.id=?", (cid,)
+        ).fetchone()
+        results.append((path, heading, text, score))
+    return results
+
+
+def prefix_chunk_ids(con: sqlite3.Connection, path_prefix: str) -> list[int]:
+    """Ids of chunks whose file path starts with `path_prefix`: an absolute path, or one relative
+    to any registered source (e.g. "docs/lessons.md"). Case-insensitive; / and \\ are the same."""
+    def norm(s: str) -> str:
+        return s.replace("\\", "/").lower()
+
+    p = Path(path_prefix).expanduser()
+    if p.is_absolute():
+        bases = [norm(str(p))]
+    else:
+        rel = norm(path_prefix).removeprefix("./")
+        bases = [norm(s).rstrip("/") + "/" + rel for (s,) in con.execute("SELECT path FROM sources")]
+    rows = con.execute("SELECT c.id, f.path FROM chunks c JOIN files f ON f.id=c.file_id")
+    return [cid for cid, path in rows if norm(path).startswith(tuple(bases))]
+
+
+@mcp.tool()
+def search_notes(query: str, limit: int = 8, path_prefix: str | None = None) -> str:
+    """Hybrid search (keyword + semantic, merged with reciprocal rank fusion).
+
+    Returns the best-matching chunks with file path and heading. Optionally restrict
+    to files whose path starts with `path_prefix`.
+    """
+    limit = max(1, min(limit, 25))
+    results = []
+    for i, (path, heading, text, score) in enumerate(hybrid_search(query, limit, path_prefix), 1):
+        snippet = text if len(text) <= 700 else text[:700] + "..."
+        results.append(f"[{i}] {path}" + (f"  # {heading}" if heading else "") + f"  (score {score:.4f})\n{snippet}")
+    return "\n\n".join(results) or "No matches. (Has index_paths been run?)"
+
+
+@mcp.tool()
+def get_note(path: str) -> str:
+    """Return the full text of an indexed file (read from disk)."""
+    p = Path(path).expanduser().resolve()
+    row = db().execute("SELECT 1 FROM files WHERE path=?", (str(p),)).fetchone()
+    if not row:
+        return f"Not in index: {p}. Only indexed files can be read with this tool."
+    return p.read_text(encoding="utf-8")
+
+
+@mcp.tool()
+def list_sources() -> str:
+    """List registered source paths with file and chunk counts."""
+    con = db()
+    out = []
+    for (s,) in con.execute("SELECT path FROM sources ORDER BY path").fetchall():
+        ids = [fid for fid, _ in files_under(con, s)]
+        nc = sum(con.execute("SELECT COUNT(*) FROM chunks WHERE file_id=?", (fid,)).fetchone()[0] for fid in ids)
+        out.append(f"{s}: {len(ids)} files, {nc} chunks")
+    return "\n".join(out) or "No sources registered."
+
+
+@mcp.tool()
+def remove_source(path: str) -> str:
+    """Unregister a source and delete its files/chunks from the index (files on disk are untouched)."""
+    con = db()
+    s = str(Path(path).expanduser().resolve())
+    con.execute("DELETE FROM sources WHERE path=?", (s,))
+    ids = [fid for fid, _ in files_under(con, s)]
+    for fid in ids:
+        drop_file(con, fid)
+    con.commit()
+    return f"Removed {s} ({len(ids)} files dropped from index)."
+
+
+def main() -> None:
+    """Console entry point (`uvx vault-index`): serve over stdio."""
+    mcp.run()
+
+
+if __name__ == "__main__":
+    main()