sift-rag 0.1.0__py3-none-any.whl

sift/__init__.py

@@ -0,0 +1,3 @@
+"""sift: fast local search over text and Markdown files, built for AI agents."""
+
+__version__ = "0.1.0"

sift/__main__.py

@@ -0,0 +1,3 @@
+from .cli import main
+
+main()

sift/cli.py

@@ -0,0 +1,109 @@
+"""Command-line interface."""
+
+import argparse
+import json
+import sqlite3
+import sys
+import textwrap
+
+from . import __version__, store
+
+
+def _filters(pairs):
+    out = {}
+    for pair in pairs:
+        key, sep, value = pair.partition("=")
+        if not sep:
+            raise SystemExit(f"filter must be KEY=VALUE, got {pair!r}")
+        try:
+            out[key] = json.loads(value)
+        except ValueError:
+            out[key] = value
+    return out
+
+
+def _print_hits(hits, as_json):
+    if as_json:
+        print(json.dumps(hits, ensure_ascii=False, indent=2))
+        return
+    for h in hits:
+        print(f"{h['score']:>8.4f}  {h['id']}:{h['lines']}  {h['title']}")
+        print(textwrap.indent(textwrap.shorten(h["text"], 300), " " * 10))
+
+
+def main(argv=None):
+    p = argparse.ArgumentParser(prog="sift", description="Fast local search over text and Markdown files.")
+    p.add_argument("--version", action="version", version=f"sift {__version__}")
+    sub = p.add_subparsers(dest="cmd", required=True)
+
+    s = sub.add_parser("add", help="register a folder as a collection")
+    s.add_argument("path")
+    s.add_argument("--name", help="collection name (default: folder name)")
+    s = sub.add_parser("rm", help="remove a collection from the index (its files stay on disk)")
+    s.add_argument("name")
+    s = sub.add_parser("index", help="scan collections and update the keyword index")
+    s.add_argument("names", nargs="*", help="collections to scan (default: all)")
+    s = sub.add_parser("embed", help="embed all chunks that have no vector yet")
+    s.add_argument("--reset", action="store_true", help="drop all vectors first (after changing the model)")
+    for name, desc in (
+        ("search", "exact keyword search (BM25)"),
+        ("vsearch", "semantic search"),
+        ("query", "hybrid search with reranking"),
+    ):
+        s = sub.add_parser(name, help=desc)
+        s.add_argument("query")
+        s.add_argument("-k", type=int, default=10, help="number of results (default: 10)")
+        s.add_argument("-c", "--collection")
+        s.add_argument("--path", help='path glob within the collection, e.g. "2024/*"')
+        s.add_argument(
+            "-f", "--filter", action="append", default=[], metavar="KEY=VALUE", help="front-matter filter, repeatable"
+        )
+        s.add_argument("--json", action="store_true", help="print results as JSON")
+    s = sub.add_parser("get", help="print a document by its id (collection/path)")
+    s.add_argument("id")
+    sub.add_parser("status", help="show collections and index statistics")
+    s = sub.add_parser("serve", help="run the MCP server")
+    s.add_argument("--host", default="127.0.0.1")
+    s.add_argument("--port", type=int, default=8765)
+    s.add_argument("--stdio", action="store_true", help="serve over stdio instead of HTTP")
+
+    a = p.parse_args(argv)
+    if (sys.stdout.encoding or "").lower() != "utf-8":  # e.g. a Windows pipe: titles may be any script
+        sys.stdout.reconfigure(encoding="utf-8")
+    db = store.connect()
+    try:
+        if a.cmd == "add":
+            name = store.add_collection(db, a.path, a.name)
+            print(f"added collection {name!r}; run `sift index` next", file=sys.stderr)
+        elif a.cmd == "rm":
+            n = store.remove_collection(db, a.name)
+            documents = "document" if n == 1 else "documents"
+            print(f"removed collection {a.name!r} and its {n} {documents} from the index;", end=" ", file=sys.stderr)
+            print("the files on disk are untouched", file=sys.stderr)
+        elif a.cmd == "index":
+            store.index(db, a.names)
+        elif a.cmd == "embed":
+            from .embed import embed_pending
+
+            embed_pending(db, a.reset)
+        elif a.cmd in ("search", "vsearch", "query"):
+            from .search import Index
+
+            fn = getattr(Index(db), a.cmd)
+            _print_hits(fn(a.query, a.k, a.collection, a.path, _filters(a.filter)), a.json)
+        elif a.cmd == "get":
+            from .search import Index
+
+            sys.stdout.write(Index(db).get(a.id))
+        elif a.cmd == "status":
+            from .search import Index
+
+            print(json.dumps(Index(db).status(), indent=2))
+        else:  # serve
+            from .server import serve
+
+            serve(a.host, a.port, a.stdio)
+    except (LookupError, ValueError, sqlite3.Error) as e:
+        raise SystemExit(f"error: {e}") from None
+    except KeyboardInterrupt:
+        raise SystemExit(130) from None

sift/embed.py

@@ -0,0 +1,259 @@
+"""Embedding and reranking models, and the batched GPU embedding pipeline."""
+
+import contextlib
+import logging
+import os
+import sys
+import time
+from functools import cache
+
+import numpy as np
+import torch
+
+from . import store
+
+# Loading Sentence Transformers makes PyTorch log a harmless note about distributed training on Windows and macOS.
+logging.getLogger("torch.distributed.elastic.multiprocessing.redirects").setLevel(logging.ERROR)
+# The MCP server logs at INFO, where loading a model logs every HTTP request to Hugging Face.
+for _name in ("httpx", "huggingface_hub", "sentence_transformers"):
+    logging.getLogger(_name).setLevel(logging.WARNING)
+
+GiB = 2**30
+
+
+def _device():
+    """An NVIDIA GPU (CUDA) if there is one, else an Apple GPU (MPS), else the CPU."""
+    if torch.cuda.is_available():
+        return "cuda"
+    return "mps" if torch.backends.mps.is_available() else "cpu"
+
+
+EMBED_MODEL = os.environ.get("SIFT_EMBED_MODEL", "Qwen/Qwen3-Embedding-0.6B")
+RERANK_MODEL = os.environ.get("SIFT_RERANK_MODEL", "Alibaba-NLP/gte-reranker-modernbert-base")
+DEVICE = _device()
+MAX_TOKENS = 1024
+SLAB = 20000  # chunks per commit; length sorting happens within a slab
+HEADROOM = 0.1  # share of the GPU left free for other programs, at least 1 GiB
+MIN_BUDGET = 2 * GiB  # with less GPU memory than this, the models run on the CPU
+FILL = 0.9  # share of the memory budget a calibrated batch may use
+MAX_BATCH = 1024
+PROGRESS_BATCHES = 8  # report progress after this many batches
+
+_batch_size = 128  # until calibrated on CUDA; a good size for CPUs and Apple GPUs
+
+
+def fp16():
+    """Whether models and vectors use fp16: yes on GPUs, no on CPUs, which are slow at fp16 matmul."""
+    return torch.device(DEVICE).type in ("cuda", "mps")
+
+
+def _free_gpu_memory():
+    """Free and total memory of the GPU. On Windows (WDDM), CUDA counts memory other programs hold as free,
+    since the driver could page it out; NVML, which nvidia-smi reads, does not. The smaller value wins."""
+    free, total = torch.cuda.mem_get_info(DEVICE)
+    with contextlib.suppress(Exception):  # NVML missing or unusable: CUDA's figure is all there is
+        free = min(free, total - torch.cuda.device_memory_used(DEVICE))
+    return free, total
+
+
+def cuda_pytorch_command(platform=sys.platform):
+    """How to replace a CPU-only PyTorch with the CUDA build. Windows wheels bundle their CUDA libraries, so
+    torch's other dependencies can stay; Linux wheels depend on nvidia-* packages, which must come along."""
+    no_deps = "--no-deps " if platform == "win32" else ""
+    return f"pip install --force-reinstall {no_deps}torch --index-url https://download.pytorch.org/whl/cu128"
+
+
+def _unused_nvidia_gpu():
+    """Why an NVIDIA GPU sits unused, if there is one; None otherwise."""
+    try:
+        import pynvml
+
+        pynvml.nvmlInit()
+        if not pynvml.nvmlDeviceGetCount():
+            return None
+    except Exception:  # no NVML, no driver, no GPU
+        return None
+    if torch.version.cuda is None:
+        return (
+            "found an NVIDIA GPU, but this PyTorch build has no CUDA support, so sift runs on the CPU (much slower). "
+            "To use the GPU, reinstall with uv: `uv tool install --reinstall --torch-backend auto sift-rag` "
+            f"(or run `uvx --torch-backend auto sift-rag ...`), or with pip: `{cuda_pytorch_command()}`"
+        )
+    return f"found an NVIDIA GPU, but PyTorch (CUDA {torch.version.cuda}) can't use it; update the NVIDIA driver"
+
+
+@cache
+def limit_gpu_memory():
+    """Cap PyTorch's GPU memory at what is free now, minus headroom for other programs, and return that budget.
+
+    Past the cap, PyTorch raises out-of-memory inside sift instead of taking memory that other programs need.
+    With too little free memory, sift runs on the CPU instead.
+    """
+    global DEVICE
+    if torch.device(DEVICE).type != "cuda":
+        if DEVICE == "cpu" and (reason := _unused_nvidia_gpu()):
+            print(reason, file=sys.stderr)
+        return None
+    free, total = _free_gpu_memory()
+    budget = free - max(HEADROOM * total, GiB)
+    if budget < MIN_BUDGET:
+        print(f"only {free / GiB:.1f} GB of GPU memory is free; using the CPU (slower)", file=sys.stderr)
+        DEVICE = "cpu"
+        return None
+    index = torch.device(DEVICE).index  # PyTorch wants an index here; plain "cuda" means the current device
+    torch.cuda.set_per_process_memory_fraction(budget / total, torch.cuda.current_device() if index is None else index)
+    print(f"using at most {budget / GiB:.1f} of {total / GiB:.1f} GB of GPU memory", file=sys.stderr)
+    return budget
+
+
+@cache
+def embedder():
+    from sentence_transformers import SentenceTransformer
+
+    limit_gpu_memory()
+    model = SentenceTransformer(EMBED_MODEL, device=DEVICE)
+    if fp16():
+        model.half()
+    model.max_seq_length = min(model.max_seq_length or MAX_TOKENS, MAX_TOKENS)
+    return model
+
+
+@cache
+def reranker():
+    from sentence_transformers import CrossEncoder
+
+    limit_gpu_memory()
+    model = CrossEncoder(RERANK_MODEL, device=DEVICE, max_length=MAX_TOKENS)
+    if fp16():
+        model.model.half()
+    return model
+
+
+def _oom_safe(fn, items):
+    """Run fn(items, batch_size), halving the batch size when the GPU runs out of memory."""
+    global _batch_size
+    while True:
+        try:
+            return fn(items, _batch_size)
+        except RuntimeError as e:  # CUDA raises OutOfMemoryError, a RuntimeError; MPS a plain RuntimeError
+            cuda = isinstance(e, torch.cuda.OutOfMemoryError)
+            if _batch_size == 1 or not (cuda or "MPS backend out of memory" in str(e)):
+                raise
+            (torch.cuda if cuda else torch.mps).empty_cache()
+            _batch_size //= 2
+            print(f"\nout of GPU memory, batch size now {_batch_size}", file=sys.stderr)
+
+
+def calibrate_batch_size(sample=None):
+    """Measure the largest batch that fits the GPU memory budget, and use it from now on.
+
+    Batches of the sample, which should be the longest text to embed (default: a text of the model's maximum
+    length), run with sizes 1, 2, 4, ... until one uses half of the budget. Memory grows linearly with the batch
+    size, so the last measurement predicts the size that fills FILL of the budget. A size that ran out of memory
+    is never chosen.
+    """
+    global _batch_size
+    model = embedder()
+    budget = limit_gpu_memory()
+    if budget is None:
+        return _batch_size
+    longest = sample or "x " * (4 * model.max_seq_length)  # the default is truncated to the longest input
+    base = torch.cuda.memory_allocated(DEVICE)
+    target = FILL * budget - base
+
+    def cost(n):
+        torch.cuda.reset_peak_memory_stats(DEVICE)
+        model.encode_document([longest] * n, batch_size=n, show_progress_bar=False)
+        return torch.cuda.max_memory_allocated(DEVICE) - base
+
+    one = used = cost(1)
+    n, failed = 1, MAX_BATCH + 1
+    while n < MAX_BATCH and used < target / 2:
+        try:
+            used = cost(2 * n)
+        except torch.cuda.OutOfMemoryError:
+            failed = 2 * n
+            break
+        n *= 2
+    torch.cuda.empty_cache()
+    per_text = (used - one) / (n - 1) if n > 1 else one
+    _batch_size = max(1, min(failed - 1, MAX_BATCH, 1 + int((target - one) / per_text)))
+    print(f"batch size {_batch_size}, calibrated to the GPU memory budget", file=sys.stderr)
+    return _batch_size
+
+
+def encode(texts, query=False):
+    """Normalised fp16 embeddings, one row per text. Sentence Transformers sorts by length to minimise padding."""
+    model = embedder()
+    fn = model.encode_query if query else model.encode_document
+    return _oom_safe(
+        lambda t, bs: fn(t, batch_size=bs, normalize_embeddings=True, convert_to_numpy=True, show_progress_bar=False),
+        texts,
+    ).astype(np.float16)
+
+
+def rerank(query, texts):
+    model = reranker()
+    return _oom_safe(
+        lambda t, bs: model.predict([(query, x) for x in t], batch_size=bs, show_progress_bar=False), texts
+    )
+
+
+def _text(seq, title, text):
+    """What gets embedded: chunks after the first repeat their document title, unless they already start with it."""
+    return text if seq == 0 or text.lstrip("# ").startswith(title) else f"{title}\n\n{text}"
+
+
+def embed_pending(db, reset=False):
+    """Embed every chunk without a vector, slab by slab. Safe to interrupt and resume."""
+    if reset:
+        db.execute("DELETE FROM vectors")
+        db.execute("DELETE FROM meta WHERE key IN ('model', 'dim')")
+        store.touch(db)
+        db.commit()
+    model = store.get_meta(db, "model")
+    if model and model != EMBED_MODEL:
+        raise SystemExit(f"index was embedded with {model}; run `sift embed --reset` to switch to {EMBED_MODEL}")
+    pending = """SELECT c.id, c.seq, fts.title, fts.text FROM chunks c JOIN fts ON fts.rowid = c.id
+                 WHERE c.id > ? AND NOT EXISTS (SELECT 1 FROM vectors v WHERE v.chunk = c.id) ORDER BY c.id LIMIT ?"""
+    total = db.execute(
+        "SELECT count(*) FROM chunks c WHERE NOT EXISTS (SELECT 1 FROM vectors v WHERE v.chunk = c.id)"
+    ).fetchone()[0]
+    if not total:
+        print("nothing to embed", file=sys.stderr)
+        return
+    # calibrate on the longest text of this run: no batch can need more memory per text
+    candidates = db.execute(
+        """SELECT c.seq, fts.title, fts.text FROM chunks c JOIN fts ON fts.rowid = c.id
+           WHERE NOT EXISTS (SELECT 1 FROM vectors v WHERE v.chunk = c.id)
+           ORDER BY length(fts.text) + length(fts.title) DESC LIMIT 100"""
+    ).fetchall()
+    calibrate_batch_size(max((_text(*row) for row in candidates), key=len))
+    done, last, t0 = 0, 0, time.perf_counter()
+
+    def progress(n):
+        rate = n / (time.perf_counter() - t0)
+        left = max(total - n, 0) / rate / 60
+        print(f"\rembedded {n}/{total} chunks, {rate:.0f}/s, {left:.1f} min left ", end="", file=sys.stderr, flush=True)
+
+    while rows := db.execute(pending, (last, SLAB)).fetchall():
+        texts = [_text(seq, title, text) for _, seq, title, text in rows]
+        # longest first, in steps of a few batches: similar lengths per batch, and progress every few seconds
+        order = sorted(range(len(texts)), key=lambda i: -len(texts[i]))
+        vecs, step = None, _batch_size * PROGRESS_BATCHES
+        for start in range(0, len(order), step):
+            part = order[start : start + step]
+            out = encode([texts[i] for i in part])
+            if vecs is None:
+                vecs = np.empty((len(texts), out.shape[1]), np.float16)
+            vecs[part] = out
+            progress(done + start + len(part))
+        db.executemany(
+            "INSERT OR REPLACE INTO vectors(chunk, vec) VALUES(?, ?)", [(r[0], v.tobytes()) for r, v in zip(rows, vecs)]
+        )
+        store.set_meta(db, "model", EMBED_MODEL)
+        store.set_meta(db, "dim", vecs.shape[1])
+        store.touch(db)  # tells a running server to reload vectors
+        db.commit()
+        done, last = done + len(rows), rows[-1][0]
+    print(f"\rembedded {done} chunks in {(time.perf_counter() - t0) / 60:.1f} min" + " " * 20, file=sys.stderr)

sift/search.py

@@ -0,0 +1,176 @@
+"""The three search modes: exact keyword (BM25), semantic (exact GPU vector search) and hybrid with reranking."""
+
+import re
+import sqlite3
+from pathlib import Path
+
+import numpy as np
+
+from . import store
+
+STOPWORDS = set(
+    "a an and are as at be by can do does for from has have how i in is it of on or that the this to "
+    "was what when where which who why with".split()
+)
+RRF_K = 60
+
+
+def _where(db, collection=None, path=None, filters=None):
+    """SQL condition on docs `d`. Filters match front-matter fields; list fields match if they contain the value."""
+    sql, args = ["1"], []
+    if collection:
+        row = db.execute("SELECT id FROM collections WHERE name = ?", (collection,)).fetchone()
+        if not row:
+            raise LookupError(f"no collection named {collection!r}")
+        sql.append("d.collection = ?")
+        args.append(row[0])
+    if path:
+        sql.append("d.path GLOB ?")
+        args.append(path)
+    for key, value in (filters or {}).items():
+        if not re.fullmatch(r"\w+", key):
+            raise ValueError(f"invalid filter field {key!r}")
+        if not isinstance(value, str | int | float):
+            raise ValueError(f"filter {key!r} needs a single string or number, got {value!r}")
+        sql.append(f"EXISTS (SELECT 1 FROM json_each(d.meta, '$.{key}') WHERE value = ?)")
+        args.append(value)
+    return " AND ".join(sql), args
+
+
+def _check_k(k):
+    if not isinstance(k, int) or k < 1:
+        raise ValueError(f"k must be a positive integer, got {k!r}")
+    return k
+
+
+def _phrases(query):
+    """Fallback for input that isn't valid FTS5 syntax: every word becomes an exact phrase, all required."""
+    return " ".join('"' + w.replace('"', '""') + '"' for w in query.split())
+
+
+def _any_word(query):
+    """OR query over the meaningful words, used to collect keyword candidates for hybrid search."""
+    words = [w for w in re.findall(r"\w+", query.lower()) if w not in STOPWORDS and len(w) > 1]
+    return " OR ".join(f'"{w}"' for w in dict.fromkeys(words))
+
+
+class Index:
+    def __init__(self, db=None):
+        self.db = db or store.connect()
+        self.version = None
+
+    # --- candidates as [(chunk_id, score)] -----------------------------------
+
+    def _keyword(self, match, k, where, args):
+        if not match.strip():
+            return []
+        sql = f"""SELECT fts.rowid, -bm25(fts, 2.0, 1.0) FROM fts JOIN chunks c ON c.id = fts.rowid
+                  JOIN docs d ON d.id = c.doc WHERE fts MATCH ? AND {where} ORDER BY bm25(fts, 2.0, 1.0) LIMIT ?"""
+        for attempt in (match, _phrases(match)):
+            try:
+                return self.db.execute(sql, [attempt, *args, k]).fetchall()
+            except sqlite3.OperationalError as e:
+                error = e
+        raise ValueError(f"invalid search query {match!r}: {error}")
+
+    def _vectors(self):
+        """Load all vectors into one (GPU) matrix; reload when the index has changed since the last load."""
+        import torch
+
+        from .embed import DEVICE, fp16
+
+        version = store.get_meta(self.db, "version")
+        if version is None or version != self.version:
+            n = self.db.execute("SELECT count(*) FROM vectors").fetchone()[0]
+            if not n:
+                raise LookupError("no embeddings yet, run `sift embed`")
+            ids, mat = np.empty(n, np.int64), np.empty((n, int(store.get_meta(self.db, "dim"))), np.float16)
+            for row, (chunk, vec) in enumerate(self.db.execute("SELECT chunk, vec FROM vectors ORDER BY chunk")):
+                ids[row], mat[row] = chunk, np.frombuffer(vec, np.float16)
+            self.ids, self.mat = ids, torch.from_numpy(mat).to(DEVICE, torch.float16 if fp16() else torch.float32)
+            self.version = version
+        return self.ids, self.mat
+
+    def _semantic(self, query, k, where, args):
+        import torch
+
+        from .embed import encode
+
+        ids, mat = self._vectors()
+        q = torch.from_numpy(encode([query], query=True)[0]).to(mat.device, mat.dtype)
+        scores = mat @ q
+        if args:  # restrict to chunks of matching documents
+            sql = f"SELECT c.id FROM chunks c JOIN docs d ON d.id = c.doc WHERE {where}"
+            allowed = np.array([c for (c,) in self.db.execute(sql, args)], np.int64)
+            pos = np.searchsorted(ids, allowed)
+            pos = pos[(pos < len(ids)) & (ids[np.minimum(pos, len(ids) - 1)] == allowed)]
+            keep = torch.zeros(len(ids), dtype=torch.bool, device=mat.device)
+            keep[torch.from_numpy(pos).to(mat.device)] = True
+            scores = scores.masked_fill(~keep, float("-inf"))
+        top = torch.topk(scores.float(), min(k, len(ids)))
+        return [(int(ids[i]), s) for s, i in zip(top.values.tolist(), top.indices.tolist()) if s > float("-inf")]
+
+    def _hits(self, ranked):
+        sql = """SELECT co.name, d.path, d.title, c.line_start, c.line_end, fts.text FROM chunks c
+                 JOIN docs d ON d.id = c.doc JOIN collections co ON co.id = d.collection JOIN fts ON fts.rowid = c.id
+                 WHERE c.id = ?"""
+        hits = []
+        for chunk, score in ranked:
+            row = self.db.execute(sql, (chunk,)).fetchone()
+            if row is None:  # removed by a concurrent `sift index`
+                continue
+            name, path, title, start, end, text = row
+            hit = {"id": f"{name}/{path}", "title": title, "lines": f"{start}-{end}", "score": round(score, 4)}
+            hits.append(hit | {"text": text})
+        return hits
+
+    # --- public API ----------------------------------------------------------
+
+    def search(self, query, k=10, collection=None, path=None, filters=None):
+        """Exact keyword search, ranked by BM25. Accepts FTS5 syntax: "phrases", prefix*, AND, OR, NOT."""
+        return self._hits(self._keyword(query, _check_k(k), *_where(self.db, collection, path, filters)))
+
+    def vsearch(self, query, k=10, collection=None, path=None, filters=None):
+        """Semantic search: cosine similarity between the query and every chunk."""
+        return self._hits(self._semantic(query, _check_k(k), *_where(self.db, collection, path, filters)))
+
+    def query(self, query, k=10, collection=None, path=None, filters=None, candidates=40):
+        """Hybrid search for RAG: keyword and semantic candidates, fused by reciprocal rank, then reranked."""
+        from .embed import rerank
+
+        k, where = _check_k(k), _where(self.db, collection, path, filters)
+        fused = {}
+        for ranked in (self._keyword(_any_word(query), candidates, *where), self._semantic(query, candidates, *where)):
+            for rank, (chunk, _) in enumerate(ranked):
+                fused[chunk] = fused.get(chunk, 0) + 1 / (RRF_K + rank)
+        hits = self._hits(sorted(fused.items(), key=lambda x: -x[1]))
+        if not hits:
+            return []
+        scores = rerank(query, [f"{h['title']}\n\n{h['text']}" for h in hits])
+        for h, s in zip(hits, scores):
+            h["score"] = round(float(s), 4)
+        return sorted(hits, key=lambda h: -h["score"])[:k]
+
+    def get(self, id):
+        """Full text of a document, by its `collection/path` id."""
+        name, _, rel = id.partition("/")
+        row = self.db.execute("SELECT path FROM collections WHERE name = ?", (name,)).fetchone()
+        root = Path(row[0]).resolve() if row else None
+        path = (root / rel).resolve() if root and rel else None
+        if path is None or not path.is_relative_to(root):
+            raise LookupError(f"unknown document {id!r}")
+        try:
+            return path.read_text("utf-8-sig", "replace")
+        except OSError as e:
+            raise LookupError(f"cannot read {id!r}: {e.strerror or e}") from None
+
+    def status(self):
+        rows = self.db.execute("""
+            SELECT co.name, co.path, count(DISTINCT d.id), count(c.id), count(v.chunk) FROM collections co
+            LEFT JOIN docs d ON d.collection = co.id LEFT JOIN chunks c ON c.doc = d.id
+            LEFT JOIN vectors v ON v.chunk = c.id GROUP BY co.id ORDER BY co.name""").fetchall()
+        return {
+            "index": self.db.execute("PRAGMA database_list").fetchone()[2],
+            "model": store.get_meta(self.db, "model"),
+            "collections": [{"name": n, "path": p, "docs": d, "chunks": c, "embedded": e} for n, p, d, c, e in rows],
+        }

sift/server.py

@@ -0,0 +1,110 @@
+"""MCP server exposing search, vsearch, query, get and status."""
+
+import sys
+from typing import Any
+
+from mcp.server.mcpserver import MCPServer
+from mcp.server.mcpserver.exceptions import ToolError
+
+from . import embed
+from .search import Index
+
+INSTRUCTIONS = """Search the user's local text and Markdown files, indexed by sift.
+When a question touches what these collections cover, search here before answering from memory or searching the web,
+and cite what you find; say so when nothing relevant turns up.
+
+{collections}
+
+- `query`: hybrid search with reranking. Best default for questions; slower than the others.
+- `search`: exact keyword search (BM25). Best for names, identifiers, exact terms and "quoted phrases".
+- `vsearch`: semantic search. Best when the wording of the answer is unknown.
+Results carry `id` (collection/path), `lines`, `title`, `score` and chunk `text`. Use `get` for a full document.
+All search tools accept `collection`, a `path` glob such as "2024/*", and `filters` on front-matter fields,
+e.g. {{"categories": "cs.RO"}}; a list field matches when it contains the value."""
+
+
+def _collections(index):
+    """The indexed collections with their sizes, read when the server starts."""
+    rows = index.db.execute("""SELECT co.name, count(d.id) FROM collections co LEFT JOIN docs d ON d.collection = co.id
+                               GROUP BY co.id ORDER BY co.name""").fetchall()
+    if not rows:
+        return "No collections are indexed yet."
+    lines = [f"- {name}: {n:,} document{'' if n == 1 else 's'}" for name, n in rows]
+    return "Indexed collections:\n" + "\n".join(lines)
+
+
+def _run(fn, *args):
+    """Call an Index method, turning expected failures into ToolError, the only kind whose message reaches the
+    agent; anything else counts as a crash, and the MCP library tells the agent just "Error executing tool"."""
+    try:
+        return fn(*args)
+    except (LookupError, ValueError) as e:
+        raise ToolError(str(e)) from None
+
+
+def build(index=None):
+    # Tools are async, so they run one at a time on the event-loop thread. That serialises GPU and SQLite work, and
+    # keeps PyTorch out of worker threads, where it crashes on Python 3.15 (Windows, macOS).
+    index = index or Index()
+    mcp = MCPServer("sift", instructions=INSTRUCTIONS.format(collections=_collections(index)))
+
+    @mcp.tool()
+    async def search(
+        query: str,
+        k: int = 10,
+        collection: str | None = None,
+        path: str | None = None,
+        filters: dict[str, Any] | None = None,
+    ) -> list[dict]:
+        """Exact keyword search ranked by BM25. Supports "phrases", prefix*, AND, OR, NOT."""
+        return _run(index.search, query, k, collection, path, filters)
+
+    @mcp.tool()
+    async def vsearch(
+        query: str,
+        k: int = 10,
+        collection: str | None = None,
+        path: str | None = None,
+        filters: dict[str, Any] | None = None,
+    ) -> list[dict]:
+        """Semantic search: chunks closest in meaning to the query."""
+        return _run(index.vsearch, query, k, collection, path, filters)
+
+    @mcp.tool()
+    async def query(
+        query: str,
+        k: int = 10,
+        collection: str | None = None,
+        path: str | None = None,
+        filters: dict[str, Any] | None = None,
+    ) -> list[dict]:
+        """Hybrid search for RAG: keyword and semantic candidates, reranked by a cross-encoder."""
+        return _run(index.query, query, k, collection, path, filters)
+
+    @mcp.tool()
+    async def get(id: str) -> str:
+        """Full text of a document, by the `id` shown in search results."""
+        return _run(index.get, id)
+
+    @mcp.tool()
+    async def status() -> dict:
+        """Collections, document and chunk counts, and the embedding model."""
+        return _run(index.status)
+
+    return mcp, index
+
+
+def serve(host="127.0.0.1", port=8765, stdio=False):
+    mcp, index = build()
+    print("loading models and vectors ...", file=sys.stderr)
+    embed.embedder()
+    embed.reranker()
+    try:
+        index._vectors()
+    except LookupError as e:
+        print(f"warning: {e}", file=sys.stderr)
+    if stdio:
+        mcp.run("stdio")
+    else:
+        print(f"sift MCP server on http://{host}:{port}/mcp", file=sys.stderr)
+        mcp.run("streamable-http", host=host, port=port)

sift/store.py

@@ -0,0 +1,296 @@
+"""SQLite store: collections, documents, chunks, keyword index and vectors."""
+
+import hashlib
+import json
+import os
+import re
+import sqlite3
+import sys
+import time
+import uuid
+from concurrent.futures import ProcessPoolExecutor
+from contextlib import nullcontext
+from pathlib import Path
+
+HOME = Path(os.environ.get("SIFT_HOME", Path.home() / ".sift"))
+EXTS = (".md", ".markdown", ".txt")
+SKIP_DIRS = {"node_modules", "__pycache__"}
+CHUNK_CHARS, OVERLAP_CHARS = 3000, 300  # ~750 tokens, well inside the models' 1024-token window
+WRITE_BATCH = 5000
+FILES_PER_WORKER = 1000  # a worker process only pays off for at least this many files
+
+SCHEMA = """
+CREATE TABLE IF NOT EXISTS meta(key TEXT PRIMARY KEY, value TEXT);
+CREATE TABLE IF NOT EXISTS collections(id INTEGER PRIMARY KEY, name TEXT UNIQUE NOT NULL, path TEXT NOT NULL);
+CREATE TABLE IF NOT EXISTS docs(id INTEGER PRIMARY KEY, collection INTEGER NOT NULL, path TEXT NOT NULL,
+    mtime INTEGER, size INTEGER, hash TEXT, title TEXT, meta TEXT, UNIQUE(collection, path));
+CREATE TABLE IF NOT EXISTS chunks(id INTEGER PRIMARY KEY, doc INTEGER NOT NULL, seq INTEGER,
+    line_start INTEGER, line_end INTEGER);
+CREATE INDEX IF NOT EXISTS chunks_doc ON chunks(doc);
+CREATE TABLE IF NOT EXISTS vectors(chunk INTEGER PRIMARY KEY, vec BLOB NOT NULL);
+CREATE VIRTUAL TABLE IF NOT EXISTS fts USING fts5(title, text, tokenize='unicode61 remove_diacritics 2');
+"""
+
+
+def connect(home=None) -> sqlite3.Connection:
+    home = Path(home or HOME)
+    home.mkdir(parents=True, exist_ok=True)
+    db = sqlite3.connect(home / "sift.db", timeout=30, check_same_thread=False)
+    db.execute("PRAGMA journal_mode=WAL")
+    db.execute("PRAGMA synchronous=NORMAL")
+    db.execute("PRAGMA cache_size=-262144")  # 256 MB
+    db.executescript(SCHEMA)
+    return db
+
+
+def get_meta(db, key, default=None):
+    row = db.execute("SELECT value FROM meta WHERE key=?", (key,)).fetchone()
+    return row[0] if row else default
+
+
+def set_meta(db, key, value):
+    db.execute("INSERT OR REPLACE INTO meta(key, value) VALUES(?, ?)", (key, str(value)))
+
+
+def touch(db):
+    """Mark the vectors as changed, so a running server reloads them. Random, as clocks can tick too slowly."""
+    set_meta(db, "version", uuid.uuid4().hex)
+
+
+# --- collections -------------------------------------------------------------
+
+
+def add_collection(db, path, name=None):
+    path = Path(path).resolve()
+    if not path.is_dir():
+        raise SystemExit(f"not a directory: {path}")
+    name = name or path.name
+    if not re.fullmatch(r"[\w.-]+", name):
+        raise SystemExit(f"invalid collection name {name!r}: use letters, digits, '_', '-' and '.'")
+    try:
+        db.execute("INSERT INTO collections(name, path) VALUES(?, ?)", (name, str(path)))
+    except sqlite3.IntegrityError:
+        raise SystemExit(f"collection {name!r} already exists") from None
+    db.commit()
+    return name
+
+
+def remove_collection(db, name):
+    """Remove a collection from the index; its files stay on disk. Returns the number of documents removed."""
+    row = db.execute("SELECT id FROM collections WHERE name=?", (name,)).fetchone()
+    if not row:
+        raise SystemExit(f"no collection named {name!r}")
+    docs = [d for (d,) in db.execute("SELECT id FROM docs WHERE collection=?", row)]
+    delete_docs(db, docs)
+    db.execute("DELETE FROM collections WHERE id=?", row)
+    db.commit()
+    return len(docs)
+
+
+def delete_docs(db, doc_ids):
+    for i in range(0, len(doc_ids), 500):
+        batch = doc_ids[i : i + 500]
+        marks = ",".join("?" * len(batch))
+        chunk_ids = f"SELECT id FROM chunks WHERE doc IN ({marks})"
+        db.execute(f"DELETE FROM fts WHERE rowid IN ({chunk_ids})", batch)
+        db.execute(f"DELETE FROM vectors WHERE chunk IN ({chunk_ids})", batch)
+        db.execute(f"DELETE FROM chunks WHERE doc IN ({marks})", batch)
+        db.execute(f"DELETE FROM docs WHERE id IN ({marks})", batch)
+    if doc_ids:
+        touch(db)
+
+
+# --- parsing and chunking ----------------------------------------------------
+
+FRONT_MATTER = re.compile(r"\A---\r?\n(.*?)\r?\n---[ \t]*(?:\r?\n|\Z)", re.DOTALL)
+HEADING = re.compile(r"^#{1,6}\s+(.+?)\s*#*\s*$", re.MULTILINE)
+
+
+def _value(text):
+    """A front-matter value: JSON if it parses (lists, numbers, quoted strings), else plain text."""
+    try:
+        return json.loads(text, parse_constant=str)  # NaN/Infinity stay strings: SQLite's JSON rejects them
+    except ValueError:
+        return text.strip("'\"")
+
+
+def parse(text, fallback_title):
+    """Split off YAML front matter (flat `key: value` lines). Returns title, meta, body, body's first line."""
+    meta, first_line = {}, 1
+    m = FRONT_MATTER.match(text)
+    if m:
+        for line in m.group(1).splitlines():
+            key, sep, value = line.partition(":")
+            if sep and key.strip() and not key[0].isspace():
+                meta[key.strip()] = _value(value.strip())
+        first_line += text.count("\n", 0, m.end())
+        text = text[m.end() :]
+    title = meta.get("title") if isinstance(meta.get("title"), str) and meta["title"].strip() else None
+    if not title:
+        h = HEADING.search(text)
+        title = h.group(1) if h else fallback_title
+    return title, meta, text, first_line
+
+
+def chunk(text, first_line=1):
+    """Split text into pieces of at most CHUNK_CHARS on line boundaries, preferring headings, with a small overlap.
+
+    Yields (first line, last line, text).
+    """
+    lines = []
+    for n, line in enumerate(text.splitlines(keepends=True), first_line):
+        while len(line) > CHUNK_CHARS:  # break overlong lines at whitespace
+            cut = line.rfind(" ", 0, CHUNK_CHARS) + 1 or CHUNK_CHARS
+            lines.append((n, line[:cut]))
+            line = line[cut:]
+        lines.append((n, line))
+
+    pieces, buf, size = [], [], 0
+    for n, line in lines:
+        heading = line.startswith("#")
+        if buf and (size + len(line) > CHUNK_CHARS or (heading and size > CHUNK_CHARS // 2)):
+            pieces.append(buf)
+            buf, size = [], 0  # carry trailing lines over as overlap, unless a new section starts
+            for item in reversed(pieces[-1] if not heading else []):
+                if size + len(item[1]) > min(OVERLAP_CHARS, CHUNK_CHARS - len(line)):
+                    break
+                buf.insert(0, item)
+                size += len(item[1])
+        buf.append((n, line))
+        size += len(line)
+    pieces.append(buf)
+
+    for piece in pieces:
+        body = "".join(text for _, text in piece).strip()
+        if body:
+            yield piece[0][0], piece[-1][0], body
+
+
+def load(path):
+    """Read and parse one file (runs in worker processes). Returns None if unreadable."""
+    try:
+        raw = Path(path).read_bytes()
+    except OSError:
+        return None
+    text = raw.decode("utf-8-sig", "replace").replace("\r\n", "\n")  # same chunks on every OS
+    title, meta, body, first_line = parse(text, Path(path).stem)
+    return hashlib.blake2b(raw, digest_size=16).hexdigest(), title, meta, list(chunk(body, first_line))
+
+
+# --- indexing ----------------------------------------------------------------
+
+
+def _cpus():
+    """CPUs this process may use, honouring affinity and container limits where the platform reports them."""
+    if hasattr(os, "process_cpu_count"):  # Python 3.13+
+        return os.process_cpu_count() or 1
+    if hasattr(os, "sched_getaffinity"):  # Linux
+        return len(os.sched_getaffinity(0))
+    return os.cpu_count() or 1
+
+
+def workers(files):
+    """Worker processes for parsing `files` files; 0 or 1 means parse in-process.
+
+    One per CPU except the one the main process uses for writing, and at most one per FILES_PER_WORKER files.
+    """
+    return max(0, min(_cpus() - 1, files // FILES_PER_WORKER))
+
+
+def walk(root):
+    """Yield (path, mtime_ns, size) for matching files. DirEntry.stat() is free on Windows."""
+    stack = [root]
+    while stack:
+        try:
+            entries = os.scandir(stack.pop())
+        except OSError:
+            continue
+        with entries:
+            for e in entries:
+                if e.is_dir(follow_symlinks=False):
+                    if not e.name.startswith(".") and e.name not in SKIP_DIRS:
+                        stack.append(e.path)
+                elif e.name.lower().endswith(EXTS):
+                    try:
+                        st = e.stat()
+                    except OSError:  # e.g. a broken symlink
+                        continue
+                    yield e.path, st.st_mtime_ns, st.st_size
+
+
+def index(db, names=None):
+    """Bring the keyword index in sync with the files on disk. Only new or changed files are read."""
+    collections = db.execute("SELECT id, name, path FROM collections ORDER BY name").fetchall()
+    unknown = set(names or ()) - {name for _, name, _ in collections}
+    if unknown:
+        raise SystemExit(f"no collection named {', '.join(map(repr, sorted(unknown)))}")
+    for cid, name, root in collections:
+        if names and name not in names:
+            continue
+        if not os.path.isdir(root):  # never treat an unplugged drive as "all files deleted"
+            print(f"{name}: folder not found, skipped: {root}", file=sys.stderr)
+            continue
+        t0 = time.perf_counter()
+        known = {
+            p: (i, m, s, h)
+            for i, p, m, s, h in db.execute("SELECT id, path, mtime, size, hash FROM docs WHERE collection=?", (cid,))
+        }
+        todo, seen = [], set()
+        for path, mtime, size in walk(root):
+            rel = os.path.relpath(path, root).replace(os.sep, "/")
+            seen.add(rel)
+            k = known.get(rel)
+            if not k or k[1] != mtime or k[2] != size:
+                todo.append((path, rel, mtime, size))
+        gone = [known[p][0] for p in known.keys() - seen]
+        delete_docs(db, gone)
+
+        added = changed = failed = 0
+        batches = [todo[i : i + WRITE_BATCH] for i in range(0, len(todo), WRITE_BATCH)]
+        n = workers(len(todo))  # parsing is CPU-bound, so it runs in processes rather than threads
+        with ProcessPoolExecutor(n) if n > 1 else nullcontext() as pool:
+
+            def read(batch):  # pool.map starts all work at once; builtin map reads lazily
+                paths = [t[0] for t in batch]
+                return pool.map(load, paths, chunksize=64) if pool else map(load, paths)
+
+            # double buffering: the next batch is parsed while the current one is written
+            pending = read(batches[0]) if batches else None
+            for b, batch in enumerate(batches):
+                results = pending
+                if b + 1 < len(batches):
+                    pending = read(batches[b + 1])
+                for (path, rel, mtime, size), res in zip(batch, results):
+                    if res is None:  # unreadable now; retried on the next run
+                        print(f"\nunreadable, skipped: {path}", file=sys.stderr)
+                        failed += 1
+                        continue
+                    digest, title, meta, chunks = res
+                    k = known.get(rel)
+                    if k and k[3] == digest:  # touched but unchanged
+                        db.execute("UPDATE docs SET mtime=?, size=? WHERE id=?", (mtime, size, k[0]))
+                        continue
+                    if k:
+                        delete_docs(db, [k[0]])
+                        changed += 1
+                    else:
+                        added += 1
+                    doc = db.execute(
+                        "INSERT INTO docs(collection, path, mtime, size, hash, title, meta) VALUES(?,?,?,?,?,?,?)",
+                        (cid, rel, mtime, size, digest, title, json.dumps(meta, ensure_ascii=False)),
+                    ).lastrowid
+                    for seq, (start, end, text) in enumerate(chunks):
+                        c = db.execute(
+                            "INSERT INTO chunks(doc, seq, line_start, line_end) VALUES(?,?,?,?)", (doc, seq, start, end)
+                        ).lastrowid
+                        db.execute("INSERT INTO fts(rowid, title, text) VALUES(?,?,?)", (c, title, text))
+                db.commit()
+                done = min((b + 1) * WRITE_BATCH, len(todo))
+                print(f"\r{name}: {done}/{len(todo)} files read", end="", file=sys.stderr, flush=True)
+        db.commit()
+        print(
+            f"\r{name}: {len(seen)} files, {added} added, {changed} changed, {len(gone)} removed"
+            + (f", {failed} unreadable" if failed else "")
+            + f" ({time.perf_counter() - t0:.1f}s)",
+            file=sys.stderr,
+        )

sift_rag-0.1.0.dist-info/METADATA

@@ -0,0 +1,210 @@
+Metadata-Version: 2.4
+Name: sift-rag
+Version: 0.1.0
+Summary: Fast local search over text and Markdown files, built for AI agents.
+Author: Tobias Würth
+License-Expression: MIT
+Project-URL: Homepage, https://github.com/tobiaswuerth/sift
+Project-URL: Documentation, https://github.com/tobiaswuerth/sift#readme
+Project-URL: Issues, https://github.com/tobiaswuerth/sift/issues
+Project-URL: Changelog, https://github.com/tobiaswuerth/sift/blob/main/CHANGELOG.md
+Keywords: search,rag,embeddings,mcp,markdown,bm25,semantic-search
+Classifier: Development Status :: 3 - Alpha
+Classifier: Environment :: Console
+Classifier: Environment :: GPU :: NVIDIA CUDA
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: Operating System :: MacOS
+Classifier: Operating System :: Microsoft :: Windows
+Classifier: Operating System :: POSIX :: Linux
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Programming Language :: Python :: 3.15
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Topic :: Text Processing :: Indexing
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: numpy>=1.26
+Requires-Dist: torch>=2.4
+Requires-Dist: sentence-transformers>=5.0
+Requires-Dist: mcp>=2.0
+Requires-Dist: nvidia-ml-py>=12
+Provides-Extra: dev
+Requires-Dist: pytest>=8; extra == "dev"
+Requires-Dist: pytest-cov>=5; extra == "dev"
+Requires-Dist: ruff>=0.6; extra == "dev"
+Requires-Dist: pre-commit>=4; extra == "dev"
+Requires-Dist: build>=1.2; extra == "dev"
+Requires-Dist: twine>=6; extra == "dev"
+Dynamic: license-file
+
+# sift
+
+[![PyPI](https://img.shields.io/pypi/v/sift-rag)](https://pypi.org/project/sift-rag/)
+[![Python](https://img.shields.io/pypi/pyversions/sift-rag)](https://pypi.org/project/sift-rag/)
+[![CI](https://github.com/tobiaswuerth/sift/actions/workflows/ci.yml/badge.svg)](https://github.com/tobiaswuerth/sift/actions/workflows/ci.yml)
+[![Coverage](https://img.shields.io/badge/coverage-100%25-brightgreen)](https://github.com/tobiaswuerth/sift/actions/workflows/ci.yml)
+[![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff)
+[![License: MIT](https://img.shields.io/badge/license-MIT-blue.svg)](LICENSE)
+
+Fast local search over your text and Markdown files, built for AI agents.
+
+sift indexes folders of `.md` and `.txt` files, embeds them on your GPU, and runs a server that agents query over MCP.
+It offers three kinds of search: exact keyword matching, semantic search, and hybrid retrieval with reranking for RAG.
+Everything runs locally. No cloud services, no API keys.
+
+## Install
+
+Requires Python 3.11 or newer. With [uv](https://docs.astral.sh/uv/):
+
+```bash
+uv tool install --torch-backend auto sift-rag   # installs the `sift` command
+```
+
+Or run it without installing: `uvx --torch-backend auto sift-rag serve`. With pip, in a virtual environment:
+`pip install sift-rag`.
+
+On first use, sift downloads its two models from Hugging Face (about 1.5 GB).
+
+### GPU support
+
+sift uses an NVIDIA GPU (CUDA) if there is one, else an Apple GPU, else the CPU, which works but embeds slowly.
+
+- **Linux, macOS:** nothing to do. PyTorch from PyPI includes CUDA on Linux and uses the Apple GPU on macOS.
+- **Windows:** PyTorch from PyPI has no CUDA support. uv's `--torch-backend auto`, used above, picks the CUDA build
+  that matches your driver. With pip, install it afterwards:
+  `pip install --force-reinstall --no-deps torch --index-url https://download.pytorch.org/whl/cu128`.
+
+If an NVIDIA GPU sits unused, sift says so on start-up and prints the command that fixes it.
+
+## Usage
+
+```bash
+sift add ~/papers --name papers        # register a folder as a collection
+sift index                             # scan files, chunk them, build the keyword index
+sift embed                             # embed every chunk that has no vector yet
+sift search '"reward shaping"'         # exact keyword search
+sift vsearch "how do robots plan brush strokes"
+sift query "mixing pigments to match a target colour"
+sift get papers/2401.02225.md          # full document
+sift status                            # collections and counts
+sift rm papers                         # remove a collection from the index; its files stay on disk
+sift serve                             # MCP server on http://127.0.0.1:8765/mcp
+```
+
+`sift index` and `sift embed` are incremental: run them again after files change, and only new or changed files are
+processed. Deleted files are removed. An interrupted `sift embed` continues where it stopped. A running server picks up
+new vectors on its own.
+
+Search commands accept `-k` (number of results), `-c` (collection), `--path` (glob within the collection, e.g.
+`"2024/*"`), `-f KEY=VALUE` (front-matter filter, repeatable) and `--json`.
+
+### Front matter
+
+YAML front matter with flat `key: value` lines becomes filterable metadata. A list field matches when it contains the
+value:
+
+```markdown
+---
+title: "Brush stroke planning"
+categories: ["cs.RO", "cs.GR"]
+year: 2024
+---
+```
+
+```bash
+sift query "stroke order" -f categories=cs.RO -f year=2024
+```
+
+The `title` field, or else the first heading, or else the file name, becomes the document title.
+
+## MCP
+
+| Tool | Purpose |
+|---|---|
+| `search` | Exact keyword search, ranked by BM25. Supports `"phrases"`, `prefix*`, `AND`, `OR`, `NOT`. |
+| `vsearch` | Semantic search. |
+| `query` | Hybrid search for RAG: keyword and semantic candidates, reranked by a cross-encoder. |
+| `get` | Full text of a document by its id. |
+| `status` | Collections and counts. |
+
+The search tools take `query`, `k`, `collection`, `path` and `filters`. Every result carries `id`
+(`collection/path`), `title`, `lines`, `score` and the chunk `text`, so agents can cite their sources.
+
+Claude Code, with the server running (`sift serve`), shared by any number of agents:
+
+```bash
+claude mcp add --transport http sift http://127.0.0.1:8765/mcp
+```
+
+Or let Claude Code start its own server over stdio:
+
+```bash
+claude mcp add sift -- sift serve --stdio
+```
+
+## Configuration
+
+None needed: sift picks the device, the GPU memory budget, the batch size and the number of worker processes itself.
+Three optional settings exist for special cases:
+
+| Variable | Default | Purpose |
+|---|---|---|
+| `SIFT_HOME` | `~/.sift` | Where the index lives |
+| `SIFT_EMBED_MODEL` | `Qwen/Qwen3-Embedding-0.6B` | Any Sentence Transformers embedding model |
+| `SIFT_RERANK_MODEL` | `Alibaba-NLP/gte-reranker-modernbert-base` | Any Sentence Transformers cross-encoder |
+
+After changing the embedding model, run `sift embed --reset`. Models are downloaded from Hugging Face on first use.
+
+## How it works
+
+- **Indexing:** worker processes read and parse files in parallel while the previous batch is written; small
+  updates skip the process start-up. Files are compared by modification time and size, then by content hash, so
+  unchanged files cost nothing. Markdown is split into chunks of about 3,000 characters (about 750 tokens) on line
+  boundaries, preferring headings, with a small overlap.
+- **Storage:** one SQLite database in WAL mode holds documents, chunks, metadata, an FTS5 keyword index and the
+  vectors (fp16).
+- **Embedding:** PyTorch in fp16 on the GPU (NVIDIA, else Apple), in batches sorted by length to keep padding low.
+  Each slab of 20,000 chunks is committed, so progress is never lost.
+- **GPU memory:** sift shares the GPU with other programs. On NVIDIA GPUs it caps itself at the memory that is free
+  when it starts, minus headroom (10% of the GPU, at least 1 GB), asking NVML as well as CUDA, since on Windows CUDA
+  counts memory held by other programs as free. It then measures how much memory its longest text needs at growing
+  batch sizes and uses the largest batch that fills 90% of that budget; if a batch still doesn't fit, it is halved.
+  With less than 2 GB to spare, sift uses the CPU instead.
+- **`search`:** SQLite FTS5 with BM25, titles weighted double. Input that isn't valid FTS5 syntax is treated as exact
+  phrases.
+- **`vsearch`:** exact nearest-neighbour search as one matrix multiply over all vectors on the GPU. No approximate
+  index, so nothing to tune and no recall loss; a million 1024-dimensional vectors take about 2 GB of VRAM.
+- **`query`:** the top 40 keyword and top 40 semantic candidates are merged by reciprocal rank fusion and reranked in
+  one GPU batch.
+- **Server:** `sift serve` loads models and vectors once and keeps them in memory.
+
+## Scope
+
+In scope: plain text and Markdown files; Linux, macOS and Windows; NVIDIA GPUs (CUDA) and Apple GPUs (MPS), otherwise
+CPU; one user, any number of local agents.
+
+Out of scope for now: PDF, Word and HTML parsing (convert to Markdown first); answer generation (sift retrieves, the
+agent writes); approximate nearest-neighbour indexes; live file watching; remote access and authentication.
+
+## Development
+
+```bash
+git clone https://github.com/tobiaswuerth/sift
+cd sift
+uv venv && uv pip install --torch-backend auto -e ".[dev]"   # or with pip: pip install -r requirements.txt -e ".[dev]"
+pre-commit install      # lint, format and config checks on every commit
+pytest --cov            # 100% line and branch coverage is required
+```
+
+The tests replace the embedding and reranking models with small deterministic fakes, so they need no GPU and no
+model downloads; everything else, including the MCP server, runs for real. See [CONTRIBUTING.md](CONTRIBUTING.md)
+for how changes and releases work.
+
+## License
+
+[MIT](LICENSE)

sift_rag-0.1.0.dist-info/RECORD

@@ -0,0 +1,13 @@
+sift/__init__.py,sha256=2Vu_dwmy3r1As0_rBVLNHdF9RJHX5PaWCGcLV2zLV4U,104
+sift/__main__.py,sha256=bYt9eEaoRQWdejEHFD8REx9jxVEdZptECFsV7F49Ink,30
+sift/cli.py,sha256=0q4WgMC2Vbv8nI1y2WOjsOEIxUjTYHRr2L5zPPxUws0,4488
+sift/embed.py,sha256=EtaQsVCwagDGNU5uVFyLu7Ze1p7jbNam0Crc-7WihHI,11111
+sift/search.py,sha256=flzCKi7UFW6_Lc3qOvosfyhOG59ddS3zOnRYeko5nYA,8337
+sift/server.py,sha256=ZpdaDx9nbqJ38gyUgR0KTzfwp5Qdelco6gnQn-lm5S8,4306
+sift/store.py,sha256=Qqdb84wY574OMVWBrPlTPDTCZDMksHdbglAq4Uo-oRY,12582
+sift_rag-0.1.0.dist-info/licenses/LICENSE,sha256=duqhUf4jtG5_2fPTQdbZYfACgSSQqDZ8cjiEe_Aj9Qo,1070
+sift_rag-0.1.0.dist-info/METADATA,sha256=XvJJo9FXBNTWSdc8JHUCnziV5_Q9Uua47fTDqbS08ls,9803
+sift_rag-0.1.0.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
+sift_rag-0.1.0.dist-info/entry_points.txt,sha256=EU1NrF_e-YlCf89B36yoGyBsCW_4UAUwBY3X0I3O5LY,64
+sift_rag-0.1.0.dist-info/top_level.txt,sha256=g_pdIrH3sx3vFPzSrmKWbngar3ays85WLehOhjxYlys,5
+sift_rag-0.1.0.dist-info/RECORD,,

sift_rag-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (84.0.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

sift_rag-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,3 @@
+[console_scripts]
+sift = sift.cli:main
+sift-rag = sift.cli:main

sift_rag-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Tobias Würth
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

sift_rag-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+sift