sqlite-sparse 0.1.0__py3-none-any.whl

sqlite_sparse/__init__.py

@@ -0,0 +1,6 @@
+from .api import SparseIndex
+from .loadable import load, loadable_path
+from .models import MODELS, fetch, register
+
+__all__ = ["SparseIndex", "load", "loadable_path", "MODELS", "fetch", "register"]
+__version__ = "0.1.0"

sqlite_sparse/api.py

@@ -0,0 +1,118 @@
+"""User-facing API: SparseIndex with create/attach/add/commit/search/sync."""
+import numpy as np
+
+from .store import SparseStore
+from .search import QueryEngine
+
+
+class SparseIndex:
+    def __init__(self, path, model=None, max_seq=256, device=None):
+        self.store = SparseStore(path)
+        self._enc = None
+        self._model = model or self.store.get_meta("model_id")
+        self._max_seq, self._device = max_seq, device
+        if self.store.get_meta("format"):
+            self.engine = QueryEngine(self.store.db)
+        else:
+            assert model, "new index needs model= ('mini' | 'base' | HF id)"
+            enc = self.encoder()
+            self.store.init_model(enc.model_id, enc.vocab(), enc.qlut())
+            self.engine = QueryEngine(self.store.db)
+
+    @classmethod
+    def create(cls, path, model="mini", **kw):
+        return cls(path, model=model, **kw)
+
+    def encoder(self):
+        if self._enc is None:
+            from .encoder import TorchEncoder
+            self._enc = TorchEncoder(self._model, max_seq=self._max_seq, device=self._device)
+        return self._enc
+
+    def add(self, id, text, title=""):
+        self.store.queue(id, text, title)
+
+    def commit(self, batch_size=256):
+        return self.sync(batch_size=batch_size)
+
+    def sync(self, batch_size=256):
+        """Drain the pending queue through the encoder. Returns docs synced."""
+        total = 0
+        while True:
+            rows = self.store.drain_pending(limit=batch_size)
+            if not rows:
+                break
+            enc = self.encoder()
+            terms = enc.encode([r[3] for r in rows])
+            self.store.clear_pending([r[0] for r in rows])
+            self.store.add_encoded(
+                [(r[1], r[2], r[3], t) for r, t in zip(rows, terms)])
+            total += len(rows)
+        self.engine.reload()
+        return total
+
+    def delete(self, id):
+        self.store.delete(id)
+
+    def attach(self, table, columns, id_col="rowid"):
+        cols = " || ' ' || ".join(f"COALESCE(NEW.{c},'')" for c in columns)
+        db = self.store.db
+        for ev in ("INSERT", "UPDATE"):
+            db.execute(f"""
+CREATE TRIGGER IF NOT EXISTS sparse_cap_{table}_{ev.lower()}
+AFTER {ev} ON {table} BEGIN
+  INSERT INTO pending(ext_id, title, body) VALUES (NEW.{id_col}, '', {cols});
+END""")
+        db.execute(f"""
+CREATE TRIGGER IF NOT EXISTS sparse_cap_{table}_delete
+AFTER DELETE ON {table} BEGIN
+  UPDATE docs SET deleted=1 WHERE ext_id = CAST(OLD.{id_col} AS TEXT) AND deleted=0;
+END""")
+        sel = " || ' ' || ".join(f"COALESCE({c},'')" for c in columns)
+        db.execute(f"INSERT INTO pending(ext_id, title, body) SELECT {id_col}, '', {sel} FROM {table}")
+        db.commit()
+
+    def search(self, text, k=10, auto_sync=False):
+        if auto_sync and self.store.pending_count():
+            self.sync()
+        return self.engine.search(text, k=k)
+
+
+def build_bulk(path, docs, ids, model="mini", batch_size=256, device=None,
+               titles=None, log=None):
+    """Bulk load: encode everything, accumulate postings in memory, write once."""
+    from collections import defaultdict
+    from .encoder import TorchEncoder
+    from .store import SparseStore
+
+    if log is None:
+        def log(m):
+            print(m, flush=True)
+    enc = TorchEncoder(model, device=device)
+    st = SparseStore(path)
+    st.init_model(enc.model_id, enc.vocab(), enc.qlut())
+    scale = float(st.get_meta("weight_scale", 40.0))
+    titles = titles or ["" for _ in docs]
+    acc_d, acc_w = defaultdict(list), defaultdict(list)
+    st.db.executemany("INSERT INTO docs(id, ext_id, title) VALUES(?,?,?)",
+                      [(i + 1, str(ids[i]), titles[i]) for i in range(len(docs))])
+    import time
+    t0 = time.time()
+    for c0 in range(0, len(docs), batch_size * 8):
+        chunk = docs[c0:c0 + batch_size * 8]
+        for off, terms in enumerate(enc.encode(chunk, batch_size=batch_size)):
+            did = c0 + off + 1
+            for t, w in terms.items():
+                acc_d[t].append(did)
+                acc_w[t].append(w)
+        if (c0 // (batch_size * 8)) % 10 == 0:
+            log(f"[bulk] {min(c0 + batch_size*8, len(docs))}/{len(docs)} ({time.time()-t0:.0f}s)")
+    for t in sorted(acc_d):
+        d = np.array(acc_d[t], dtype="<i4")
+        w = np.clip(np.rint(np.array(acc_w[t]) * scale), 1, 255).astype(np.uint8)
+        st.db.execute("INSERT INTO postings VALUES(?,?,?)", (t, d.tobytes(), w.tobytes()))
+    st.set_meta("ndocs", len(docs))
+    st.db.commit()
+    st.db.execute("VACUUM")
+    log(f"[bulk] DONE {len(docs)} docs ({time.time()-t0:.0f}s)")
+    return SparseIndex(path)

sqlite_sparse/cli.py

@@ -0,0 +1,50 @@
+"""CLI: sqlite-sparse build|search|sync|attach|info"""
+import argparse
+import json
+import sys
+
+
+def main():
+    p = argparse.ArgumentParser(prog="sqlite-sparse")
+    sub = p.add_subparsers(dest="cmd", required=True)
+    b = sub.add_parser("build", help="index a JSONL file of {id, text[, title]}")
+    b.add_argument("db"); b.add_argument("jsonl"); b.add_argument("--model", default="mini")
+    s = sub.add_parser("search"); s.add_argument("db"); s.add_argument("query"); s.add_argument("-k", type=int, default=10)
+    y = sub.add_parser("sync"); y.add_argument("db")
+    a = sub.add_parser("attach"); a.add_argument("db"); a.add_argument("table"); a.add_argument("columns", help="comma-separated"); a.add_argument("--id-col", default="rowid"); a.add_argument("--model", default="mini")
+    i = sub.add_parser("info"); i.add_argument("db")
+    c = sub.add_parser("convert", help="write the .sprs sidecar for an inference-free HF checkpoint")
+    c.add_argument("model_id"); c.add_argument("out")
+    c.add_argument("--double-log", action="store_true", help="v3 models (log1p applied twice)")
+    args = p.parse_args()
+    from .api import SparseIndex
+
+    if args.cmd == "build":
+        ix = SparseIndex(args.db, model=args.model)
+        for line in open(args.jsonl):
+            r = json.loads(line)
+            ix.add(id=r["id"], text=r["text"], title=r.get("title", ""))
+        n = ix.commit()
+        print(f"indexed {n} docs -> {args.db}")
+    elif args.cmd == "search":
+        ix = SparseIndex(args.db)
+        for r in ix.search(args.query, k=args.k):
+            print(f"{r['score']:8.3f}  {r['ext_id']}  {r['title'][:70]}")
+    elif args.cmd == "sync":
+        print(f"synced {SparseIndex(args.db).sync()} docs")
+    elif args.cmd == "attach":
+        ix = SparseIndex(args.db, model=args.model)
+        ix.attach(args.table, args.columns.split(","), id_col=args.id_col)
+        print(f"attached to {args.table}({args.columns}); synced {ix.sync()} rows")
+    elif args.cmd == "convert":
+        from .convert import extract
+        extract(args.model_id, args.out, act_flag=1 if args.double_log else 0)
+    elif args.cmd == "info":
+        ix = SparseIndex(args.db)
+        st = ix.store
+        print(json.dumps({"model": st.get_meta("model_id"), "ndocs": st.get_meta("ndocs"),
+                          "pending": st.pending_count(), "weight_mode": st.get_meta("weight_mode")}, indent=2))
+
+
+if __name__ == "__main__":
+    sys.exit(main())

sqlite_sparse/convert.py

@@ -0,0 +1,52 @@
+"""Write the .sprs sidecar: the MLM head llama.cpp's converter drops, the static
+query weight table and the vocabulary. Little-endian throughout.
+
+  'SPRS' | u32 version=1 | u32 hidden | u32 vocab_n | u32 weight fmt (0=f32)
+  | u32 activation (0 log1p(relu), 1 log1p(log1p(relu)) for v3) | 8 pad
+  | dense.W (H*H f32) | dense.b (H) | ln.gamma (H) | ln.beta (H)
+  | decoder.W (vocab_n*H f32, row-major) | decoder.b (vocab_n)
+  | qlut (vocab_n f32) | u32 len | vocab blob (tokens joined by \n)
+"""
+import struct
+import sys
+
+import numpy as np
+
+
+def extract(model_id, out_path, act_flag=0):
+    from transformers import AutoModelForMaskedLM, AutoTokenizer
+    from sentence_transformers import SparseEncoder
+
+    se = SparseEncoder(model_id, device="cpu")
+    qlut = se[0].sub_modules["query"][0].weight.detach().float().numpy().ravel()
+    m = AutoModelForMaskedLM.from_pretrained(model_id)
+    sd = {k: v.detach().float().numpy() for k, v in m.state_dict().items()}
+    H = sd["cls.predictions.transform.dense.weight"].shape[0]
+    dec_w = sd["bert.embeddings.word_embeddings.weight"]
+    vocab_n = dec_w.shape[0]
+    tok = AutoTokenizer.from_pretrained(model_id)
+    vocab = tok.convert_ids_to_tokens(list(range(vocab_n)))
+    vocab_blob = "\n".join(vocab).encode("utf-8")
+
+    with open(out_path, "wb") as f:
+        f.write(b"SPRS")
+        f.write(struct.pack("<IIIII8x", 1, H, vocab_n, 0, act_flag))
+        for arr in (sd["cls.predictions.transform.dense.weight"],
+                    sd["cls.predictions.transform.dense.bias"],
+                    sd["cls.predictions.transform.LayerNorm.weight"],
+                    sd["cls.predictions.transform.LayerNorm.bias"],
+                    dec_w,
+                    sd["cls.predictions.bias"],
+                    qlut.astype(np.float32)):
+            f.write(np.ascontiguousarray(arr, dtype="<f4").tobytes())
+        f.write(struct.pack("<I", len(vocab_blob)))
+        f.write(vocab_blob)
+    import os
+    print(f"wrote {out_path}: H={H} vocab={vocab_n} bytes={os.path.getsize(out_path)}")
+    return H, vocab_n
+
+
+if __name__ == "__main__":
+    extract(sys.argv[1] if len(sys.argv) > 1
+            else "opensearch-project/opensearch-neural-sparse-encoding-doc-v2-mini",
+            sys.argv[2] if len(sys.argv) > 2 else "mini.sprs")

sqlite_sparse/encoder.py

@@ -0,0 +1,39 @@
+"""Document encoders for the reference implementation (torch)."""
+
+MODELS = {
+    "mini": "opensearch-project/opensearch-neural-sparse-encoding-doc-v2-mini",
+    "base": "opensearch-project/opensearch-neural-sparse-encoding-doc-v3-distill",
+    "multilingual": "opensearch-project/opensearch-neural-sparse-encoding-multilingual-v1",
+}
+
+
+def resolve(model):
+    return MODELS.get(model, model)
+
+
+class TorchEncoder:
+    def __init__(self, model, max_seq=256, device=None):
+        import torch
+        from sentence_transformers import SparseEncoder
+        self.model_id = resolve(model)
+        device = device or ("cuda" if torch.cuda.is_available() else "cpu")
+        kw = {"trust_remote_code": True} if "gte" in self.model_id or "multilingual" in self.model_id else {}
+        self.m = SparseEncoder(self.model_id, device=device, **kw)
+        self.m.max_seq_length = max_seq
+
+    def qlut(self):
+        return self.m[0].sub_modules["query"][0].weight.detach().float().cpu().numpy().ravel()
+
+    def vocab(self):
+        return self.m.tokenizer.convert_ids_to_tokens(list(range(len(self.qlut()))))
+
+    def encode(self, texts, batch_size=32, wmin=0.01):
+        """returns list[dict[term_id, weight]]"""
+        emb = self.m.encode_document(texts, batch_size=batch_size,
+                                     convert_to_sparse_tensor=True, show_progress_bar=False).coalesce()
+        idx, val = emb.indices().cpu().numpy(), emb.values().cpu().numpy()
+        out = [dict() for _ in texts]
+        keep = val >= wmin
+        for r, c, v in zip(idx[0][keep], idx[1][keep], val[keep]):
+            out[r][int(c)] = float(v)
+        return out

sqlite_sparse/loadable.py

@@ -0,0 +1,45 @@
+"""Locate and load the sparse0 extension. Search order: the copy bundled in
+the package, $SQLITE_SPARSE_EXT, then the repo's build tree."""
+import os
+import sys
+from pathlib import Path
+
+_SUFFIX = {"darwin": ".dylib", "win32": ".dll"}.get(sys.platform, ".so")
+
+
+def _candidates():
+    here = Path(__file__).parent
+    yield here / f"sparse0{_SUFFIX}"
+    env = os.environ.get("SQLITE_SPARSE_EXT")
+    if env:
+        yield Path(env)
+    yield here.parents[2] / "build" / f"sparse0{_SUFFIX}"
+
+
+def loadable_path():
+    """Absolute path of the extension binary."""
+    tried = []
+    for p in _candidates():
+        tried.append(str(p))
+        if p.exists():
+            return str(p)
+    raise FileNotFoundError(
+        "sparse0 extension not found. Tried:\n  " + "\n  ".join(tried) +
+        "\nBuild it with `make` at the repo root, or set SQLITE_SPARSE_EXT to the binary."
+    )
+
+
+def load(conn):
+    """Load sparse0 into a connection. Returns the path used."""
+    if not hasattr(conn, "enable_load_extension"):
+        raise RuntimeError(
+            "this Python's sqlite3 module was built without loadable-extension "
+            "support (common with the python.org and GitHub Actions macOS builds). "
+            "Use a Python from Homebrew or conda, or `pip install sqlean.py` and "
+            "connect with `import sqlean as sqlite3`."
+        )
+    path = loadable_path()
+    conn.enable_load_extension(True)
+    conn.load_extension(path)
+    conn.enable_load_extension(False)
+    return path

sqlite_sparse/models.py

@@ -0,0 +1,69 @@
+"""Named models: download the GGUF and .sprs for an alias into
+~/.cache/sqlite-sparse and register them. urllib only; HF_TOKEN honored."""
+import os
+import sys
+import urllib.request
+from pathlib import Path
+
+HUB = "https://huggingface.co"
+
+# alias -> (repo, file stem, encoder params)
+MODELS = {
+    "mini":         ("arbazsiddiqui/opensearch-neural-sparse-doc-v2-mini-GGUF", "mini", "23M"),
+    "base":         ("arbazsiddiqui/opensearch-neural-sparse-doc-v3-distill-GGUF", "base", "67M"),
+    "multilingual": ("arbazsiddiqui/opensearch-neural-sparse-multilingual-v1-GGUF", "multilingual", "168M"),
+}
+
+
+def cache_dir():
+    root = os.environ.get("SQLITE_SPARSE_CACHE") or os.path.join(
+        os.environ.get("XDG_CACHE_HOME", os.path.expanduser("~/.cache")), "sqlite-sparse")
+    Path(root).mkdir(parents=True, exist_ok=True)
+    return root
+
+
+def _download(url, dest, label):
+    req = urllib.request.Request(url)
+    tok = os.environ.get("HF_TOKEN")
+    if tok:
+        req.add_header("Authorization", f"Bearer {tok}")
+    tmp = dest + ".part"
+    with urllib.request.urlopen(req) as r, open(tmp, "wb") as f:
+        total = int(r.headers.get("Content-Length") or 0)
+        done = 0
+        while True:
+            chunk = r.read(1 << 20)
+            if not chunk:
+                break
+            f.write(chunk)
+            done += len(chunk)
+            if total and sys.stderr.isatty():
+                sys.stderr.write(f"\r{label}: {done * 100 // total}%")
+        if total and sys.stderr.isatty():
+            sys.stderr.write("\n")
+    os.replace(tmp, dest)
+
+
+def fetch(alias="mini", quant="q8"):
+    """Download the GGUF and sidecar for `alias` if not cached. Returns (gguf, sprs)."""
+    if alias not in MODELS:
+        raise KeyError(f"unknown model alias {alias!r}; known: {sorted(MODELS)}")
+    repo, stem, _ = MODELS[alias]
+    if quant not in ("q8", "f16"):
+        raise ValueError("quant must be 'q8' or 'f16'")
+    d = os.path.join(cache_dir(), alias)
+    Path(d).mkdir(parents=True, exist_ok=True)
+    out = []
+    for fname in (f"{stem}_{quant}.gguf", f"{stem}.sprs"):
+        dest = os.path.join(d, fname)
+        if not os.path.exists(dest):
+            _download(f"{HUB}/{repo}/resolve/main/{fname}", dest, fname)
+        out.append(dest)
+    return tuple(out)
+
+
+def register(conn, alias="mini", quant="q8", max_seq=512):
+    """fetch() then sparse_register(). Returns the alias."""
+    gguf, sprs = fetch(alias, quant)
+    conn.execute("SELECT sparse_register(?, ?, ?, ?)", (alias, gguf, sprs, max_seq)).fetchone()
+    return alias

sqlite_sparse/search.py

@@ -0,0 +1,144 @@
+"""Query engine: tokenizer + static table + exact scoring over an open
+sqlite-sparse database. numpy only. No model, no network."""
+import json
+import unicodedata
+
+import numpy as np
+
+_MAX_WORD_CHARS = 100
+
+
+def _is_punctuation(ch):
+    cp = ord(ch)
+    if 33 <= cp <= 47 or 58 <= cp <= 64 or 91 <= cp <= 96 or 123 <= cp <= 126:
+        return True
+    return unicodedata.category(ch).startswith("P")
+
+
+def _is_cjk(cp):
+    return (0x4E00 <= cp <= 0x9FFF or 0x3400 <= cp <= 0x4DBF or 0x20000 <= cp <= 0x2A6DF
+            or 0x2A700 <= cp <= 0x2B73F or 0x2B740 <= cp <= 0x2B81F or 0x2B820 <= cp <= 0x2CEAF
+            or 0xF900 <= cp <= 0xFAFF or 0x2F800 <= cp <= 0x2FA1F)
+
+
+def _basic_tokenize(text):
+    """BERT BasicTokenizer: clean, isolate CJK, split on whitespace, lowercase,
+    strip combining marks, split punctuation."""
+    cleaned = []
+    for ch in text:
+        cp = ord(ch)
+        cat = unicodedata.category(ch)
+        if cp == 0 or cp == 0xFFFD or (cat.startswith("C") and ch not in "\t\n\r"):
+            continue
+        if ch in " \t\n\r" or cat == "Zs":
+            cleaned.append(" ")
+        elif _is_cjk(cp):
+            cleaned.append(f" {ch} ")
+        else:
+            cleaned.append(ch)
+    words = []
+    for w in "".join(cleaned).split():
+        w = w.lower()
+        w = "".join(c for c in unicodedata.normalize("NFD", w) if unicodedata.category(c) != "Mn")
+        cur = []
+        for c in w:
+            if _is_punctuation(c):
+                if cur:
+                    words.append("".join(cur))
+                    cur = []
+                words.append(c)
+            else:
+                cur.append(c)
+        if cur:
+            words.append("".join(cur))
+    return words
+
+
+class QueryEngine:
+    def __init__(self, db):
+        self.db = db
+        self.reload()
+
+    def reload(self):
+        meta = dict(self.db.execute("SELECT k, v FROM meta"))
+        assert meta.get("format") == "sqlite-sparse/1", f"not sqlite-sparse: {meta.get('format')}"
+        self.model_id = meta["model_id"]
+        self.ndocs = int(meta.get("ndocs") or 0)
+        self.weight_mode = meta.get("weight_mode", "u8")
+        self.weight_scale = float(meta.get("weight_scale", 40.0))
+        self._v2i = {t: i for i, t in enumerate(json.loads(meta["vocab"]))}
+        self._unk = self._v2i.get("[UNK]")
+        self._qlut = dict(self.db.execute("SELECT t, w FROM qlut"))
+        self._dead = {r[0] for r in self.db.execute("SELECT id FROM docs WHERE deleted=1")}
+
+    def _wordpiece_word(self, w):
+        if len(w) > _MAX_WORD_CHARS:
+            return [self._unk]
+        ids, s = [], 0
+        while s < len(w):
+            e = len(w)
+            while e > s:
+                piece = w[s:e] if s == 0 else "##" + w[s:e]
+                if piece in self._v2i:
+                    ids.append(self._v2i[piece])
+                    break
+                e -= 1
+            else:
+                return [self._unk]
+            s = e
+        return ids
+
+    def _wordpiece(self, text):
+        out = []
+        for w in _basic_tokenize(text):
+            for t in self._wordpiece_word(w):
+                if t is not None:
+                    out.append(t)
+            if len(out) >= 512:
+                return out[:512]
+        return out
+
+    def encode_query(self, text):
+        qw = {}
+        for t in self._wordpiece(text):
+            w = self._qlut.get(t)
+            if w:
+                qw[t] = qw.get(t, 0.0) + w
+        return qw
+
+    def search(self, text, k=10):
+        qw = self.encode_query(text)
+        if not qw or not self.ndocs:
+            return []
+        score = np.zeros(self.ndocs + 1, dtype=np.float64)
+        for t, w in qw.items():
+            row = self.db.execute("SELECT docs, ws FROM postings WHERE t=?", (t,)).fetchone()
+            if not row:
+                continue
+            docs = np.frombuffer(row[0], dtype="<i4")
+            if self.weight_mode == "u8":
+                ws = np.frombuffer(row[1], dtype=np.uint8).astype(np.float32) / self.weight_scale
+            else:
+                ws = np.frombuffer(row[1], dtype="<f4")
+            np.add.at(score, docs, ws * w)
+        want = k + len(self._dead)
+        if want >= len(score):
+            top = np.argsort(-score)
+        else:
+            part = np.argpartition(score, -want)[-want:]
+            top = part[np.argsort(-score[part])]
+        out = []
+        for d in top:
+            d = int(d)
+            if score[d] <= 0 or d in self._dead:
+                continue
+            out.append(d)
+            if len(out) == k:
+                break
+        if not out:
+            return []
+        ph = ",".join("?" * len(out))
+        rows = {r[0]: r for r in self.db.execute(
+            f"SELECT id, ext_id, title FROM docs WHERE id IN ({ph})", out)}
+        return [{"id": d, "ext_id": rows[d][1], "title": rows[d][2], "score": float(score[d])}
+                for d in out if d in rows]

sqlite_sparse/store.py

@@ -0,0 +1,126 @@
+"""Storage layer for the sqlite-sparse/1 format: schema, incremental posting
+merges, deletions, compaction. All writes go through SparseStore."""
+import json
+import sqlite3
+import time
+from collections import defaultdict
+
+import numpy as np
+
+SCHEMA = """
+CREATE TABLE IF NOT EXISTS meta(k TEXT PRIMARY KEY, v TEXT);
+CREATE TABLE IF NOT EXISTS qlut(t INTEGER PRIMARY KEY, w REAL);
+CREATE TABLE IF NOT EXISTS postings(t INTEGER PRIMARY KEY, docs BLOB, ws BLOB);
+CREATE TABLE IF NOT EXISTS docs(id INTEGER PRIMARY KEY AUTOINCREMENT,
+    ext_id TEXT UNIQUE, title TEXT, body TEXT, meta TEXT, deleted INTEGER DEFAULT 0,
+    ntokens INTEGER, truncated INTEGER);
+CREATE TABLE IF NOT EXISTS pending(id INTEGER PRIMARY KEY AUTOINCREMENT,
+    ext_id TEXT, title TEXT, body TEXT NOT NULL);
+"""
+
+
+class SparseStore:
+    def __init__(self, path):
+        self.path = path
+        self.db = sqlite3.connect(path)
+        self.db.executescript(SCHEMA)
+
+    def get_meta(self, k, default=None):
+        r = self.db.execute("SELECT v FROM meta WHERE k=?", (k,)).fetchone()
+        return r[0] if r else default
+
+    def set_meta(self, k, v):
+        self.db.execute("INSERT OR REPLACE INTO meta VALUES(?,?)", (k, str(v)))
+
+    def init_model(self, model_id, vocab, qlut_weights, weight_mode="u8", weight_scale=40.0):
+        if self.get_meta("format"):
+            assert self.get_meta("model_id") == model_id, \
+                f"index built with {self.get_meta('model_id')}, not {model_id}"
+            return
+        for k, v in [("format", "sqlite-sparse/1"), ("model_id", model_id),
+                     ("weight_mode", weight_mode), ("weight_scale", weight_scale),
+                     ("vocab", json.dumps(vocab)),
+                     ("created_utc", time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()))]:
+            self.set_meta(k, v)
+        nz = np.nonzero(qlut_weights)[0]
+        self.db.executemany("INSERT OR REPLACE INTO qlut VALUES(?,?)",
+                            [(int(t), float(qlut_weights[t])) for t in nz])
+        self.db.commit()
+
+    def queue(self, ext_id, text, title=""):
+        self.db.execute("INSERT INTO pending(ext_id, title, body) VALUES(?,?,?)",
+                        (str(ext_id), title, text))
+        self.db.commit()
+
+    def pending_count(self):
+        return self.db.execute("SELECT COUNT(*) FROM pending").fetchone()[0]
+
+    def drain_pending(self, limit=1000):
+        rows = self.db.execute(
+            "SELECT id, ext_id, title, body FROM pending ORDER BY id LIMIT ?", (limit,)).fetchall()
+        return rows
+
+    def clear_pending(self, ids):
+        # No commit here: sync() runs this inside add_encoded's transaction so
+        # the queue drain and the postings write land atomically.
+        self.db.executemany("DELETE FROM pending WHERE id=?", [(i,) for i in ids])
+
+    def add_encoded(self, items, store_body=False):
+        """items: list of (ext_id, title, body, {term: weight}). Merges into
+        postings. Re-adding an existing ext_id marks the old row deleted."""
+        mode = self.get_meta("weight_mode", "u8")
+        scale = float(self.get_meta("weight_scale", 40.0))
+        acc = defaultdict(lambda: ([], []))
+        for ext_id, title, body, terms in items:
+            old = self.db.execute("SELECT id FROM docs WHERE ext_id=? AND deleted=0",
+                                  (str(ext_id),)).fetchone()
+            if old:
+                self.db.execute("UPDATE docs SET deleted=1, ext_id=ext_id||':del:'||id WHERE id=?", (old[0],))
+            cur = self.db.execute("INSERT INTO docs(ext_id, title, body) VALUES(?,?,?)",
+                                  (str(ext_id), title, body if store_body else None))
+            did = cur.lastrowid
+            for t, w in terms.items():
+                d, ws = acc[int(t)]
+                d.append(did)
+                ws.append(w)
+        for t, (d, ws) in acc.items():
+            row = self.db.execute("SELECT docs, ws FROM postings WHERE t=?", (t,)).fetchone()
+            nd = np.array(d, dtype="<i4")
+            nw = np.array(ws, dtype=np.float32)
+            if mode == "u8":
+                nwb = np.clip(np.rint(nw * scale), 1, 255).astype(np.uint8)
+            else:
+                nwb = nw.astype("<f4")
+            if row:
+                nd = np.concatenate([np.frombuffer(row[0], dtype="<i4"), nd])
+                old_w = np.frombuffer(row[1], dtype=np.uint8 if mode == "u8" else "<f4")
+                nwb = np.concatenate([old_w, nwb])
+            self.db.execute("INSERT OR REPLACE INTO postings VALUES(?,?,?)",
+                            (t, nd.tobytes(), nwb.tobytes()))
+        self.set_meta("ndocs", self.db.execute("SELECT MAX(id) FROM docs").fetchone()[0] or 0)
+        self.db.commit()
+
+    def delete(self, ext_id):
+        self.db.execute("UPDATE docs SET deleted=1 WHERE ext_id=? AND deleted=0", (str(ext_id),))
+        self.db.commit()
+
+    def deleted_ids(self):
+        return np.array([r[0] for r in self.db.execute("SELECT id FROM docs WHERE deleted=1")],
+                        dtype=np.int64)
+
+    def compact(self):
+        """Rewrite postings without deleted docs and VACUUM."""
+        dead = set(int(i) for i in self.deleted_ids())
+        if dead:
+            for t, db_, wb in self.db.execute("SELECT t, docs, ws FROM postings").fetchall():
+                d = np.frombuffer(db_, dtype="<i4")
+                keep = ~np.isin(d, list(dead))
+                if keep.all():
+                    continue
+                mode = self.get_meta("weight_mode", "u8")
+                w = np.frombuffer(wb, dtype=np.uint8 if mode == "u8" else "<f4")
+                self.db.execute("UPDATE postings SET docs=?, ws=? WHERE t=?",
+                                (d[keep].tobytes(), w[keep].tobytes(), t))
+            self.db.execute("DELETE FROM docs WHERE deleted=1")
+            self.db.commit()
+        self.db.execute("VACUUM")

sqlite_sparse-0.1.0.dist-info/METADATA

@@ -0,0 +1,46 @@
+Metadata-Version: 2.4
+Name: sqlite-sparse
+Version: 0.1.0
+Summary: Semantic search in one SQLite file. No model, no server at query time.
+Author-email: Arbaz Siddiqui <arbaz00@gmail.com>
+License: MIT
+Project-URL: Homepage, https://github.com/arbazsiddiqui/sqlite-sparse
+Project-URL: Repository, https://github.com/arbazsiddiqui/sqlite-sparse
+Project-URL: Issues, https://github.com/arbazsiddiqui/sqlite-sparse/issues
+Keywords: sqlite,search,semantic-search,sparse,retrieval,splade,embedded
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Database
+Classifier: Topic :: Text Processing :: Indexing
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: numpy>=1.24
+Provides-Extra: build
+Requires-Dist: onnxruntime>=1.17; extra == "build"
+Requires-Dist: tokenizers>=0.15; extra == "build"
+Requires-Dist: huggingface_hub>=0.20; extra == "build"
+Provides-Extra: build-torch
+Requires-Dist: sentence-transformers>=5.0; extra == "build-torch"
+Requires-Dist: torch; extra == "build-torch"
+Provides-Extra: dev
+Requires-Dist: pytest>=7; extra == "dev"
+Dynamic: license-file
+
+# sqlite-sparse (Python)
+
+Python binding for [sqlite-sparse](https://github.com/arbazsiddiqui/sqlite-sparse),
+semantic search in one SQLite file with no model at query time.
+
+```python
+import sqlite3, sqlite_sparse
+db = sqlite3.connect("notes.db")
+sqlite_sparse.load(db)                 # loads the sparse0 extension
+sqlite_sparse.register(db, "mini")     # downloads the model on first use
+db.execute("CREATE VIRTUAL TABLE notes USING sparse0(model='mini')")
+```
+
+Also contains the pure-Python reference implementation of the file format
+(`SparseIndex`) and the sidecar converter. Full documentation in the repository README.

sqlite_sparse-0.1.0.dist-info/RECORD

@@ -0,0 +1,15 @@
+sqlite_sparse/__init__.py,sha256=8Qw_SMijJmv8jlXM5D8vMcKjwKAUIlrbv3JeC3Yjohk,220
+sqlite_sparse/api.py,sha256=T7blGH7-ZKc5b6XuMsjTVVOYftd3RFxNQlnsMSenvPY,4644
+sqlite_sparse/cli.py,sha256=qLwmkJAT48V-4GDd9T8rW5c4V81K6gZ3kPo__MNXHy4,2454
+sqlite_sparse/convert.py,sha256=83qm-MAsQymzCLl-IJbeq7lmQJGldpBGWBsbhDqYFbg,2289
+sqlite_sparse/encoder.py,sha256=b0WzUF74L7KYB3a0JZrnv8Z7rx3PVroDrjYFDAfxduI,1645
+sqlite_sparse/loadable.py,sha256=32UKWKPwiThuLmiEcG5jWuvvLcFtxT5mPX35h__jq0E,1507
+sqlite_sparse/models.py,sha256=VA_Ru6HseNFNDO0u74SjdRTuKOue4j4yatp_ndYxGVM,2560
+sqlite_sparse/search.py,sha256=JgP1nGQoqjY0A7Sr9Qv78mGqZbd6z7vlDUACrH7fJHU,4918
+sqlite_sparse/store.py,sha256=7GQOuAoC-dxBJUH59o_cCueJaZs5BejxxDNdqTuYXtY,5870
+sqlite_sparse-0.1.0.dist-info/licenses/LICENSE,sha256=E2xU48tQ--vIEgBKFehhUPsd49UFzxzP7VN8joDz6Hc,1071
+sqlite_sparse-0.1.0.dist-info/METADATA,sha256=3-O_fFgmrpMS71mufSAWwc8KpkaNRGAmb0bpbUj92Sw,1894
+sqlite_sparse-0.1.0.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
+sqlite_sparse-0.1.0.dist-info/entry_points.txt,sha256=T0NlTs4vDA6bKeNO6roWdRVqfj8A9PpBkxKcDzeBkDQ,57
+sqlite_sparse-0.1.0.dist-info/top_level.txt,sha256=GBq7GGgMxL9B4Dv4bTRsFDvFz6-HApR0iv1Gct2So6I,14
+sqlite_sparse-0.1.0.dist-info/RECORD,,

sqlite_sparse-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (84.0.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

sqlite_sparse-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+sqlite-sparse = sqlite_sparse.cli:main

sqlite_sparse-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Arbaz Siddiqui
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

sqlite_sparse-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+sqlite_sparse