semsift 0.0.1__tar.gz

semsift-0.0.1/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Nitin Kumar
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALING IN THE
+SOFTWARE.

semsift-0.0.1/PKG-INFO

@@ -0,0 +1,86 @@
+Metadata-Version: 2.4
+Name: semsift
+Version: 0.0.1
+Summary: Building blocks for hybrid keyword and vector retrieval over SQLite
+License-Expression: MIT
+Project-URL: Source, https://github.com/nitkrar/semsift
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: numpy>=1.26
+Requires-Dist: vicinity>=0.4.6
+Provides-Extra: static
+Requires-Dist: model2vec>=0.9; extra == "static"
+Requires-Dist: huggingface-hub>=0.23; extra == "static"
+Provides-Extra: tree-sitter
+Requires-Dist: tree-sitter-language-pack>=1.20; extra == "tree-sitter"
+Provides-Extra: onnx
+Requires-Dist: onnxruntime>=1.17; extra == "onnx"
+Requires-Dist: huggingface-hub>=0.23; extra == "onnx"
+Requires-Dist: tokenizers>=0.15; extra == "onnx"
+Provides-Extra: webgpu
+Requires-Dist: onnxruntime>=1.24.4; extra == "webgpu"
+Requires-Dist: onnxruntime-ep-webgpu; extra == "webgpu"
+Requires-Dist: huggingface-hub>=0.23; extra == "webgpu"
+Requires-Dist: tokenizers>=0.15; extra == "webgpu"
+Dynamic: license-file
+
+# semsift
+
+Building blocks for hybrid retrieval: chunk text, embed it, store vectors
+and a keyword index together in SQLite or memory, search both, fuse the
+ranked results and rerank them. It retrieves and ranks; it never
+generates text.
+
+**Status:** Implemented, not yet released. See
+[docs/design.md](docs/design.md) for the blocks and their contracts, and
+[docs/features.md](docs/features.md) for what was taken from other
+projects.
+
+```python
+import sqlite3
+
+from semsift.embed import StaticEncoder
+from semsift.search import KeywordSource, Search, VectorSource
+from semsift.store import Field, Item, Store
+
+conn = sqlite3.connect(":memory:", isolation_level=None)
+store = Store(conn, "notes", [Field("source", "text")],
+              encoder=StaticEncoder("minishlab/potion-base-8M"))
+
+items = [Item(1, "Rent is due on the first of the month.", {"source": "lease.md"}),
+         Item(2, "The boiler service is booked for Tuesday.", {"source": "home.md"})]
+vectors = store.embed(items)          # encode before opening the transaction
+conn.execute("BEGIN")
+store.upsert(items, vectors)          # the store never commits; you do
+conn.execute("COMMIT")
+
+search = Search(store, sources=[VectorSource(store), KeywordSource(store)],
+                citation=("source",))
+for hit in search.run("when is rent due", k=1).hits:
+    print(hit.citation["source"], hit.text)
+```
+
+Install extras for what you use: `static` (model2vec encoders), `onnx`,
+`webgpu`, `tree-sitter` (syntax-aware chunking).
+
+## Development
+
+```
+uv venv .venv && uv pip install --python .venv/bin/python -e ".[static,onnx,tree-sitter]"
+HF_HUB_OFFLINE=1 .venv/bin/python -m unittest discover -s tests -t .
+.venv/bin/python benchmarks/run.py
+```
+
+To use an unreleased semsift from another project, depend on the
+checkout (`uv add --editable ../semsift`, or `pip install -e ../semsift`),
+or build it (`uv build`) and install the wheel from `dist/`.
+
+## Releasing
+
+The version lives only in `semsift.__version__`. Bump it, commit, and push
+a matching tag (`v0.0.1`). The release workflow runs the tests, builds,
+refuses to publish unless the tag, `__version__`, the sdist and the wheel
+agree, publishes to PyPI through trusted publishing, and creates the
+GitHub release. PyPI must list this repository's `release.yml` as a
+trusted publisher for the `semsift` project.

semsift-0.0.1/README.md

@@ -0,0 +1,59 @@
+# semsift
+
+Building blocks for hybrid retrieval: chunk text, embed it, store vectors
+and a keyword index together in SQLite or memory, search both, fuse the
+ranked results and rerank them. It retrieves and ranks; it never
+generates text.
+
+**Status:** Implemented, not yet released. See
+[docs/design.md](docs/design.md) for the blocks and their contracts, and
+[docs/features.md](docs/features.md) for what was taken from other
+projects.
+
+```python
+import sqlite3
+
+from semsift.embed import StaticEncoder
+from semsift.search import KeywordSource, Search, VectorSource
+from semsift.store import Field, Item, Store
+
+conn = sqlite3.connect(":memory:", isolation_level=None)
+store = Store(conn, "notes", [Field("source", "text")],
+              encoder=StaticEncoder("minishlab/potion-base-8M"))
+
+items = [Item(1, "Rent is due on the first of the month.", {"source": "lease.md"}),
+         Item(2, "The boiler service is booked for Tuesday.", {"source": "home.md"})]
+vectors = store.embed(items)          # encode before opening the transaction
+conn.execute("BEGIN")
+store.upsert(items, vectors)          # the store never commits; you do
+conn.execute("COMMIT")
+
+search = Search(store, sources=[VectorSource(store), KeywordSource(store)],
+                citation=("source",))
+for hit in search.run("when is rent due", k=1).hits:
+    print(hit.citation["source"], hit.text)
+```
+
+Install extras for what you use: `static` (model2vec encoders), `onnx`,
+`webgpu`, `tree-sitter` (syntax-aware chunking).
+
+## Development
+
+```
+uv venv .venv && uv pip install --python .venv/bin/python -e ".[static,onnx,tree-sitter]"
+HF_HUB_OFFLINE=1 .venv/bin/python -m unittest discover -s tests -t .
+.venv/bin/python benchmarks/run.py
+```
+
+To use an unreleased semsift from another project, depend on the
+checkout (`uv add --editable ../semsift`, or `pip install -e ../semsift`),
+or build it (`uv build`) and install the wheel from `dist/`.
+
+## Releasing
+
+The version lives only in `semsift.__version__`. Bump it, commit, and push
+a matching tag (`v0.0.1`). The release workflow runs the tests, builds,
+refuses to publish unless the tag, `__version__`, the sdist and the wheel
+agree, publishes to PyPI through trusted publishing, and creates the
+GitHub release. PyPI must list this repository's `release.yml` as a
+trusted publisher for the `semsift` project.

semsift-0.0.1/pyproject.toml

@@ -0,0 +1,36 @@
+[build-system]
+requires = ["setuptools>=77"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "semsift"
+dynamic = ["version"]
+description = "Building blocks for hybrid keyword and vector retrieval over SQLite"
+readme = "README.md"
+requires-python = ">=3.11"
+license = "MIT"
+license-files = ["LICENSE"]
+dependencies = [
+    "numpy>=1.26",
+    "vicinity>=0.4.6",
+]
+
+[project.optional-dependencies]
+static = ["model2vec>=0.9", "huggingface-hub>=0.23"]
+tree-sitter = ["tree-sitter-language-pack>=1.20"]
+onnx = ["onnxruntime>=1.17", "huggingface-hub>=0.23", "tokenizers>=0.15"]
+webgpu = [
+    "onnxruntime>=1.24.4",
+    "onnxruntime-ep-webgpu",
+    "huggingface-hub>=0.23",
+    "tokenizers>=0.15",
+]
+
+[project.urls]
+Source = "https://github.com/nitkrar/semsift"
+
+[tool.setuptools.dynamic]
+version = { attr = "semsift.__version__" }
+
+[tool.setuptools.packages.find]
+where = ["src"]

semsift-0.0.1/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

semsift-0.0.1/src/semsift/__init__.py

@@ -0,0 +1,3 @@
+"""Building blocks for hybrid keyword and vector retrieval."""
+
+__version__ = "0.0.1"

semsift-0.0.1/src/semsift/chunk/__init__.py

@@ -0,0 +1,5 @@
+"""Splitting text into chunks for the store."""
+
+from .chunkers import Chunk, TextChunker, TreeSitterChunker
+
+__all__ = ["Chunk", "TextChunker", "TreeSitterChunker"]

semsift-0.0.1/src/semsift/chunk/chunkers.py

@@ -0,0 +1,206 @@
+"""Splitting text into chunks whose spans double as citation spans."""
+
+from __future__ import annotations
+
+from bisect import bisect_right
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True)
+class Chunk:
+    """`text` is exactly `source[start:end]`; lines are 1-based and inclusive.
+
+    `context` is the enclosing definitions (e.g. a class name) and
+    `symbols` the names defined inside, when the chunker knows them.
+    """
+
+    text: str
+    start: int
+    end: int
+    start_line: int
+    end_line: int
+    context: tuple[str, ...] = ()
+    symbols: tuple[str, ...] = ()
+
+
+def _build(text: str, spans: list[tuple[int, int]], min_chars: int,
+           extras: dict[int, tuple[tuple[str, ...], tuple[str, ...]]] | None = None) -> list[Chunk]:
+    newlines = [i for i, ch in enumerate(text)
+                if ch == "\n" or (ch == "\r" and (i + 1 == len(text) or text[i + 1] != "\n"))]
+    out = []
+    for start, end in spans:
+        body = text[start:end]
+        if len(body.strip()) < min_chars:
+            continue
+        context, symbols = (extras or {}).get(start, ((), ()))
+        out.append(Chunk(body, start, end, bisect_right(newlines, start - 1) + 1,
+                         bisect_right(newlines, max(start, end - 1) - 1) + 1,
+                         context, symbols))
+    return out
+
+
+def _check(max_chars: int, min_chars: int) -> None:
+    if max_chars <= 0:
+        raise ValueError("max_chars must be positive")
+    if not 0 <= min_chars <= max_chars:
+        raise ValueError("min_chars must be between 0 and max_chars")
+
+
+class TextChunker:
+    """Line-based windows of at most `max_chars`, split at natural breaks.
+
+    With `markdown`, an ATX heading outside a fenced block always starts a
+    chunk; turn it off for text where `#` begins a comment. After a blank
+    line, a chunk at least half full ends before the next paragraph. A
+    line longer than `max_chars` is cut into pieces. Chunks whose stripped
+    text is shorter than `min_chars` are dropped, so the spans tile the
+    text only when `min_chars` is 0.
+    """
+
+    def __init__(self, max_chars: int = 750, min_chars: int = 1, *,
+                 markdown: bool = True) -> None:
+        _check(max_chars, min_chars)
+        self.max_chars, self.min_chars, self.markdown = max_chars, min_chars, markdown
+
+    def spans(self, text: str) -> list[tuple[int, int]]:
+        spans: list[tuple[int, int]] = []
+        start = pos = 0
+        after_blank = False
+        fence: tuple[str, int] | None = None
+        for line in text.splitlines(keepends=True):
+            size = pos - start
+            blank = not line.strip()
+            marker = _markdown_fence(line)
+            heading = self.markdown and fence is None and _markdown_heading(line)
+            if size and (heading
+                         or (after_blank and not blank and size >= self.max_chars // 2)
+                         or size + len(line) > self.max_chars):
+                spans.append((start, pos))
+                start = pos
+            if len(line) > self.max_chars:
+                for cut in range(pos, pos + len(line), self.max_chars):
+                    spans.append((cut, min(cut + self.max_chars, pos + len(line))))
+                start = pos + len(line)
+            pos += len(line)
+            after_blank = blank
+            if marker is not None:
+                char, width, rest = marker
+                if fence is None:
+                    fence = (char, width)
+                elif char == fence[0] and width >= fence[1] and not rest.strip():
+                    fence = None
+        if pos > start:
+            spans.append((start, pos))
+        return spans
+
+    def chunk(self, text: str) -> list[Chunk]:
+        return _build(text, self.spans(text), self.min_chars)
+
+
+class TreeSitterChunker:
+    """Syntax-aligned chunks from tree-sitter-language-pack's chunker.
+
+    Needs the `tree-sitter` extra. The language pack downloads a
+    grammar the first time a language is parsed. A language it does not
+    know, a grammar it cannot download, or a failure to parse falls back
+    to `fallback`: a TextChunker of the same bounds that does not read
+    markdown, by default. So does a source over `max_source_bytes`, or a
+    parse that runs past `parse_timeout_ms`, so one generated or minified
+    file cannot stall indexing.
+    """
+
+    def __init__(self, max_chars: int = 750, min_chars: int = 1,
+                 fallback: TextChunker | None = None, *,
+                 max_source_bytes: int = 5_000_000, parse_timeout_ms: int = 5_000) -> None:
+        _check(max_chars, min_chars)
+        if (isinstance(max_source_bytes, bool)
+                or not isinstance(max_source_bytes, int) or max_source_bytes <= 0
+                or isinstance(parse_timeout_ms, bool)
+                or not isinstance(parse_timeout_ms, int) or parse_timeout_ms <= 0):
+            raise ValueError("max_source_bytes and parse_timeout_ms must be positive integers")
+        self.max_chars, self.min_chars = max_chars, min_chars
+        self.max_source_bytes, self.parse_timeout_ms = max_source_bytes, parse_timeout_ms
+        self.fallback = fallback or TextChunker(max_chars, min_chars, markdown=False)
+
+    def supports(self, language: str) -> bool:
+        """Whether the pack knows `language`; its grammar may still need a download."""
+        import tree_sitter_language_pack as pack
+
+        return language in pack.manifest_languages()
+
+    def chunk(self, text: str, language: str) -> list[Chunk]:
+        import tree_sitter_language_pack as pack
+
+        if not text.strip():
+            return self.fallback.chunk(text)
+        if not self.supports(language) or len(text.encode()) > self.max_source_bytes:
+            return self.fallback.chunk(text)
+        try:
+            result = pack.process(text, pack.ProcessConfig(
+                language=language, structure=False, imports=False, exports=False,
+                chunk_max_size=self.max_chars, max_source_bytes=self.max_source_bytes,
+                parse_timeout_ms=self.parse_timeout_ms))
+        except pack.Error:
+            return self.fallback.chunk(text)
+        pieces = sorted(result.chunks, key=lambda c: c.start_byte)
+        if not pieces:
+            return self.fallback.chunk(text)
+        data = text.encode()
+        # Chunks are cut at their start bytes, so they tile the text even
+        # if the pack leaves a gap; each start is moved back to a
+        # character boundary before converting to a character offset.
+        starts = sorted({0} | {_char_start(data, c.start_byte) for c in pieces})
+        chars = _char_offsets(data, starts)
+        extras = {}
+        for c in pieces:
+            meta = c.metadata
+            if meta is not None:
+                key = chars[_char_start(data, c.start_byte)]
+                extras.setdefault(key, (tuple(meta.context_path), tuple(meta.symbols_defined)))
+        bounds = [chars[b] for b in starts] + [len(text)]
+        spans = [(a, b) for a, b in zip(bounds, bounds[1:]) if b > a]
+        if any(end - start > self.max_chars for start, end in spans):
+            return self.fallback.chunk(text)
+        return _build(text, spans, self.min_chars, extras)
+
+
+def _char_start(data: bytes, offset: int) -> int:
+    """`offset` moved back to the first byte of the UTF-8 character it is in."""
+    offset = min(max(offset, 0), len(data))
+    while 0 < offset < len(data) and data[offset] & 0xC0 == 0x80:
+        offset -= 1
+    return offset
+
+
+def _char_offsets(data: bytes, byte_offsets: list[int]) -> dict[int, int]:
+    """Character offset of each sorted byte offset, which must be on a boundary."""
+    out, chars, prev = {}, 0, 0
+    for b in byte_offsets:
+        chars += len(data[prev:b].decode())
+        out[b] = chars
+        prev = b
+    return out
+
+
+def _markdown_heading(line: str) -> bool:
+    body = line.rstrip("\r\n")
+    stripped = body.lstrip(" ")
+    if len(body) - len(stripped) > 3:
+        return False
+    width = len(stripped) - len(stripped.lstrip("#"))
+    return 1 <= width <= 6 and (width == len(stripped) or stripped[width] in " \t")
+
+
+def _markdown_fence(line: str) -> tuple[str, int, str] | None:
+    body = line.rstrip("\r\n")
+    stripped = body.lstrip(" ")
+    if len(body) - len(stripped) > 3 or not stripped or stripped[0] not in "`~":
+        return None
+    char = stripped[0]
+    width = len(stripped) - len(stripped.lstrip(char))
+    if width < 3:
+        return None
+    rest = stripped[width:]
+    if char == "`" and "`" in rest:
+        return None
+    return char, width, rest

semsift-0.0.1/src/semsift/embed/__init__.py

@@ -0,0 +1,12 @@
+"""Text to vectors.
+
+Imports nothing else from semsift, so it can become its own package.
+"""
+
+from .backends import (Encoder, FakeEncoder, HttpEncoder, OnnxEncoder,
+                       StaticEncoder)
+from .policy import resolve_prefix
+from .space import VectorSpace
+
+__all__ = ["Encoder", "FakeEncoder", "HttpEncoder", "OnnxEncoder",
+           "StaticEncoder", "VectorSpace", "resolve_prefix"]