docsonar 0.1.0__py3-none-any.whl

docsonar/__init__.py

@@ -0,0 +1,3 @@
+"""docsonar — local document search MCP server. Chat with your folders, fully offline."""
+
+__version__ = "0.1.0"

docsonar/__main__.py

@@ -0,0 +1,6 @@
+"""Support `python -m docsonar`."""
+
+from docsonar.cli import main
+
+if __name__ == "__main__":
+    main()

docsonar/chunking.py

@@ -0,0 +1,234 @@
+"""Structure-aware chunking.
+
+Blocks are grouped greedily up to a token target, flushing early at
+heading boundaries so chunks don't straddle sections. Adjacent chunks
+within a section share a small overlap. Character offsets index into the
+"extracted text" of the document (block texts joined by blank lines),
+which is exactly what ``read_file`` returns for a parsed document.
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from docsonar.parsers.base import Block
+
+_BLOCK_JOINER = "\n\n"
+_WORD = re.compile(r"\S+")
+
+
+def estimate_tokens(text: str) -> int:
+    # Word count x 1.3 for normal prose; the chars/8 floor only wins on
+    # whitespace-free blobs (minified/base64), where "1 word" would let a
+    # megabyte of text pass as a single token.
+    return max(1, round(len(_WORD.findall(text)) * 1.3), len(text) // 8)
+
+
+@dataclass(frozen=True)
+class Chunk:
+    text: str
+    token_count: int
+    heading_path: str | None
+    page_start: int | None
+    page_end: int | None
+    char_start: int
+    char_end: int
+
+
+@dataclass(frozen=True)
+class _Piece:
+    text: str
+    tokens: int
+    heading_path: tuple[str, ...]
+    page: int | None
+    char_start: int
+    char_end: int
+
+
+def extracted_text(blocks: list[Block]) -> str:
+    """The virtual document text that chunk char offsets index into."""
+    return _BLOCK_JOINER.join(b.text for b in blocks)
+
+
+def _split_oversized(
+    text: str,
+    base: int,
+    heading_path: tuple[str, ...],
+    page: int | None,
+    max_tokens: int,
+    overlap_tokens: int,
+) -> list[_Piece]:
+    """Split a single huge block into overlapping word windows.
+
+    A "word" longer than ``_MAX_PIECE_CHARS`` (minified/base64 blobs have
+    no whitespace, so the word regex sees one giant token) is chopped
+    into fixed char windows so no chunk can smuggle megabytes of text.
+    """
+    words = list(_WORD.finditer(text))
+    window = max(1, int(max_tokens / 1.3))
+    step = max(1, window - int(overlap_tokens / 1.3))
+    max_piece_chars = max_tokens * 8
+    pieces: list[_Piece] = []
+
+    def add_piece(piece_text: str, start: int, end: int) -> None:
+        pieces.append(
+            _Piece(
+                text=piece_text,
+                tokens=estimate_tokens(piece_text),
+                heading_path=heading_path,
+                page=page,
+                char_start=base + start,
+                char_end=base + end,
+            )
+        )
+
+    for i in range(0, len(words), step):
+        span = words[i : i + window]
+        start, end = span[0].start(), span[-1].end()
+        segment = text[start:end]
+        if len(segment) > max_piece_chars:
+            for j in range(0, len(segment), max_piece_chars):
+                part = segment[j : j + max_piece_chars]
+                add_piece(part, start + j, start + j + len(part))
+        else:
+            add_piece(segment, start, end)
+        if i + window >= len(words):
+            break
+    return pieces
+
+
+def chunk_blocks(
+    blocks: list[Block],
+    *,
+    target_tokens: int = 400,
+    max_tokens: int = 512,
+    min_tokens: int = 100,
+    overlap_tokens: int = 60,
+) -> list[Chunk]:
+    # A pathological config (overlap >= target) would make the overlap carry
+    # dominate every chunk and _split_oversized degenerate to step=1.
+    overlap_tokens = max(0, min(overlap_tokens, target_tokens // 2))
+
+    pieces: list[_Piece] = []
+    offset = 0
+    for block in blocks:
+        tokens = estimate_tokens(block.text)
+        if tokens > max_tokens or len(block.text) > max_tokens * 8:
+            pieces.extend(
+                _split_oversized(
+                    block.text,
+                    offset,
+                    block.heading_path,
+                    block.page,
+                    max_tokens,
+                    overlap_tokens,
+                )
+            )
+        else:
+            pieces.append(
+                _Piece(
+                    text=block.text,
+                    tokens=tokens,
+                    heading_path=block.heading_path,
+                    page=block.page,
+                    char_start=offset,
+                    char_end=offset + len(block.text),
+                )
+            )
+        offset += len(block.text) + len(_BLOCK_JOINER)
+
+    chunks: list[Chunk] = []
+    current: list[_Piece] = []
+    current_tokens = 0
+    # True while `current` holds only overlap carried from the previous flush;
+    # such a remainder is already inside the last chunk and must not re-emit.
+    carried_only = False
+
+    def flush(carry_overlap: bool) -> None:
+        nonlocal current, current_tokens, carried_only
+        if not current:
+            return
+        chunks.append(_build_chunk(current))
+        carried_only = True
+        if carry_overlap:
+            # Seed the next chunk with trailing pieces, but never re-carry
+            # everything (that would loop forever on tiny sections).
+            carried: list[_Piece] = []
+            carried_tokens = 0
+            for piece in reversed(current):
+                if carried_tokens + piece.tokens > overlap_tokens or len(
+                    carried
+                ) + 1 >= len(current):
+                    break
+                carried.insert(0, piece)
+                carried_tokens += piece.tokens
+            current = carried
+            current_tokens = carried_tokens
+        else:
+            current = []
+            current_tokens = 0
+
+    for piece in pieces:
+        section_changed = (
+            bool(current) and piece.heading_path != current[-1].heading_path
+        )
+        if current and (
+            current_tokens + piece.tokens > max_tokens
+            or (section_changed and current_tokens >= min_tokens)
+        ):
+            flush(carry_overlap=not section_changed)
+        current.append(piece)
+        current_tokens += piece.tokens
+        carried_only = False
+        if current_tokens >= target_tokens:
+            flush(carry_overlap=True)
+
+    # Trailing remainder: merge into the previous chunk when it's tiny and fits.
+    # Skip it entirely when it is nothing but carried overlap — that text is
+    # already part of the last emitted chunk.
+    if current and not carried_only:
+        remainder = _build_chunk(current)
+        if (
+            chunks
+            and remainder.token_count < min_tokens
+            and chunks[-1].token_count + remainder.token_count <= max_tokens
+            and remainder.char_start >= chunks[-1].char_start
+        ):
+            chunks[-1] = _merge(chunks[-1], remainder)
+        else:
+            chunks.append(remainder)
+    return chunks
+
+
+def _build_chunk(pieces: list[_Piece]) -> Chunk:
+    text = _BLOCK_JOINER.join(p.text for p in pieces)
+    heading = " > ".join(pieces[0].heading_path) or None
+    pages = [p.page for p in pieces if p.page is not None]
+    return Chunk(
+        text=text,
+        token_count=estimate_tokens(text),
+        heading_path=heading,
+        page_start=min(pages) if pages else None,
+        page_end=max(pages) if pages else None,
+        char_start=pieces[0].char_start,
+        char_end=pieces[-1].char_end,
+    )
+
+
+def _merge(a: Chunk, b: Chunk) -> Chunk:
+    text = a.text + _BLOCK_JOINER + b.text
+    pages = [
+        p for p in (a.page_start, a.page_end, b.page_start, b.page_end) if p is not None
+    ]
+    return Chunk(
+        text=text,
+        token_count=estimate_tokens(text),
+        heading_path=a.heading_path,
+        page_start=min(pages) if pages else None,
+        page_end=max(pages) if pages else None,
+        char_start=a.char_start,
+        char_end=b.char_end,
+    )

docsonar/cli.py

@@ -0,0 +1,48 @@
+"""Command-line entry point: `docsonar` (or `uvx docsonar`)."""
+
+from __future__ import annotations
+
+import argparse
+from pathlib import Path
+
+from docsonar import __version__
+from docsonar.config import default_config_path, default_db_path, load_config
+from docsonar.server import create_server
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(
+        prog="docsonar",
+        description="Local document search MCP server — chat with your folders, fully offline.",
+    )
+    parser.add_argument(
+        "--transport",
+        choices=["stdio", "http"],
+        default="stdio",
+        help="stdio for MCP clients like Claude Desktop/Code (default); "
+        "http for streamable HTTP",
+    )
+    parser.add_argument("--host", default="127.0.0.1", help="HTTP bind host (http only)")
+    parser.add_argument("--port", type=int, default=8365, help="HTTP port (http only)")
+    parser.add_argument(
+        "--db-path",
+        type=Path,
+        default=None,
+        help=f"index database location (default: {default_db_path()})",
+    )
+    parser.add_argument(
+        "--config",
+        type=Path,
+        default=None,
+        help=f"config.toml location (default: {default_config_path()})",
+    )
+    parser.add_argument("--version", action="version", version=f"docsonar {__version__}")
+    args = parser.parse_args()
+
+    config = load_config(db_path=args.db_path, config_path=args.config)
+    server = create_server(config, host=args.host, port=args.port)
+    server.run(transport="stdio" if args.transport == "stdio" else "streamable-http")
+
+
+if __name__ == "__main__":
+    main()

docsonar/config.py

@@ -0,0 +1,120 @@
+"""Configuration: zero-config defaults, optionally overridden by a
+``config.toml`` in the platform config directory.
+
+Windows: %LOCALAPPDATA%\\docsonar\\config.toml
+macOS:   ~/Library/Application Support/docsonar/config.toml
+Linux:   ~/.config/docsonar/config.toml
+
+Recognized keys (all optional)::
+
+    embedding_model = "BAAI/bge-small-en-v1.5"
+    chunk_target_tokens = 400
+    chunk_max_tokens = 512
+    chunk_min_tokens = 100
+    chunk_overlap_tokens = 60
+    max_file_size_mb = 50
+    extra_ignore_dirs = ["Archive", "tmp"]
+"""
+
+from __future__ import annotations
+
+import sys
+import tomllib
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+import platformdirs
+
+APP_NAME = "docsonar"
+
+# Directories never worth indexing, pruned during folder walks.
+DEFAULT_IGNORE_DIRS: frozenset[str] = frozenset(
+    {
+        ".git",
+        ".hg",
+        ".svn",
+        ".venv",
+        "venv",
+        "node_modules",
+        "__pycache__",
+        ".idea",
+        ".vscode",
+        ".cache",
+        ".mypy_cache",
+        ".ruff_cache",
+        ".pytest_cache",
+        "dist",
+        "build",
+    }
+)
+
+
+def default_db_path() -> Path:
+    # appauthor=False: avoid Windows' doubled <author>\<app> nesting
+    return Path(platformdirs.user_data_dir(APP_NAME, appauthor=False)) / "index.db"
+
+
+def default_config_path() -> Path:
+    return Path(platformdirs.user_config_dir(APP_NAME, appauthor=False)) / "config.toml"
+
+
+@dataclass(frozen=True)
+class Config:
+    """Runtime configuration with zero-config defaults."""
+
+    db_path: Path = field(default_factory=default_db_path)
+    embedding_model: str = "BAAI/bge-small-en-v1.5"
+    chunk_target_tokens: int = 400
+    chunk_max_tokens: int = 512
+    chunk_min_tokens: int = 100
+    chunk_overlap_tokens: int = 60
+    max_file_size_bytes: int = 50 * 1024 * 1024
+    ignore_dirs: frozenset[str] = DEFAULT_IGNORE_DIRS
+
+
+_INT_KEYS = (
+    "chunk_target_tokens",
+    "chunk_max_tokens",
+    "chunk_min_tokens",
+    "chunk_overlap_tokens",
+)
+
+
+def load_config(db_path: Path | None = None, config_path: Path | None = None) -> Config:
+    """Build the effective config: defaults <- config.toml <- CLI overrides.
+
+    A malformed config file is reported on stderr and otherwise ignored —
+    the server must always come up.
+    """
+    path = config_path if config_path is not None else default_config_path()
+    overrides: dict[str, Any] = {}
+    if path.is_file():
+        try:
+            data = tomllib.loads(path.read_text(encoding="utf-8"))
+            overrides = _validate(data, path)
+        except (tomllib.TOMLDecodeError, OSError) as exc:
+            print(f"docsonar: ignoring config file {path}: {exc}", file=sys.stderr)
+
+    if db_path is not None:
+        overrides["db_path"] = db_path.expanduser().resolve()
+    return Config(**overrides)
+
+
+def _validate(data: dict[str, Any], path: Path) -> dict[str, Any]:
+    out: dict[str, Any] = {}
+    for key, value in data.items():
+        if (key in _INT_KEYS and isinstance(value, int)) or (
+            key == "embedding_model" and isinstance(value, str)
+        ):
+            out[key] = value
+        elif key == "max_file_size_mb" and isinstance(value, int):
+            out["max_file_size_bytes"] = value * 1024 * 1024
+        elif key == "extra_ignore_dirs" and isinstance(value, list):
+            out["ignore_dirs"] = DEFAULT_IGNORE_DIRS | frozenset(str(v) for v in value)
+        else:
+            print(
+                f"docsonar: ignoring unknown or invalid config key {key!r} in {path}",
+                file=sys.stderr,
+            )
+    return out

docsonar/db.py

@@ -0,0 +1,162 @@
+"""SQLite storage: schema, connections, and small helpers.
+
+One database file holds everything — folder registry, file metadata,
+chunks, and the FTS5 index. WAL mode
+keeps concurrent reads cheap and plays well with Windows file locking.
+"""
+
+from __future__ import annotations
+
+import sqlite3
+from collections.abc import Iterator
+from contextlib import contextmanager
+from pathlib import Path
+
+SCHEMA_VERSION = "1"
+
+_SCHEMA = """
+CREATE TABLE IF NOT EXISTS meta (
+    key   TEXT PRIMARY KEY,
+    value TEXT NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS folders (
+    id              INTEGER PRIMARY KEY,
+    path            TEXT NOT NULL UNIQUE,
+    include_globs   TEXT,
+    exclude_globs   TEXT,
+    added_at        TEXT NOT NULL,
+    last_indexed_at TEXT
+);
+
+CREATE TABLE IF NOT EXISTS files (
+    id           INTEGER PRIMARY KEY,
+    folder_id    INTEGER NOT NULL REFERENCES folders(id) ON DELETE CASCADE,
+    path         TEXT NOT NULL UNIQUE,
+    rel_path     TEXT NOT NULL,
+    file_type    TEXT NOT NULL,
+    size_bytes   INTEGER NOT NULL,
+    mtime_ns     INTEGER NOT NULL,
+    content_hash TEXT NOT NULL,
+    status       TEXT NOT NULL DEFAULT 'indexed',
+    error        TEXT,
+    indexed_at   TEXT
+);
+CREATE INDEX IF NOT EXISTS idx_files_folder ON files(folder_id);
+
+CREATE TABLE IF NOT EXISTS chunks (
+    id           INTEGER PRIMARY KEY,
+    file_id      INTEGER NOT NULL REFERENCES files(id) ON DELETE CASCADE,
+    seq          INTEGER NOT NULL,
+    text         TEXT NOT NULL,
+    token_count  INTEGER NOT NULL,
+    heading_path TEXT,
+    page_start   INTEGER,
+    page_end     INTEGER,
+    char_start   INTEGER NOT NULL,
+    char_end     INTEGER NOT NULL,
+    UNIQUE (file_id, seq)
+);
+CREATE INDEX IF NOT EXISTS idx_chunks_file ON chunks(file_id);
+
+CREATE VIRTUAL TABLE IF NOT EXISTS chunks_fts USING fts5(
+    text,
+    content='chunks',
+    content_rowid='id',
+    tokenize='porter unicode61'
+);
+
+CREATE TRIGGER IF NOT EXISTS chunks_ai AFTER INSERT ON chunks BEGIN
+    INSERT INTO chunks_fts(rowid, text) VALUES (new.id, new.text);
+END;
+CREATE TRIGGER IF NOT EXISTS chunks_ad AFTER DELETE ON chunks BEGIN
+    INSERT INTO chunks_fts(chunks_fts, rowid, text) VALUES ('delete', old.id, old.text);
+END;
+CREATE TRIGGER IF NOT EXISTS chunks_au AFTER UPDATE OF text ON chunks BEGIN
+    INSERT INTO chunks_fts(chunks_fts, rowid, text) VALUES ('delete', old.id, old.text);
+    INSERT INTO chunks_fts(rowid, text) VALUES (new.id, new.text);
+END;
+"""
+
+
+def init_db(db_path: Path) -> None:
+    """Create the database file and schema if missing. Idempotent."""
+    db_path.parent.mkdir(parents=True, exist_ok=True)
+    with connect(db_path) as conn:
+        conn.executescript(_SCHEMA)
+        conn.execute(
+            "INSERT OR IGNORE INTO meta (key, value) VALUES ('schema_version', ?)",
+            (SCHEMA_VERSION,),
+        )
+
+
+@contextmanager
+def connect(db_path: Path) -> Iterator[sqlite3.Connection]:
+    """Open a connection with sane pragmas; commits on success, rolls back on error."""
+    conn = sqlite3.connect(db_path)
+    conn.row_factory = sqlite3.Row
+    conn.execute("PRAGMA journal_mode = WAL")
+    conn.execute("PRAGMA foreign_keys = ON")
+    conn.execute("PRAGMA busy_timeout = 5000")
+    _load_vec_extension(conn)
+    try:
+        yield conn
+        conn.commit()
+    except BaseException:
+        conn.rollback()
+        raise
+    finally:
+        conn.close()
+
+
+def _load_vec_extension(conn: sqlite3.Connection) -> bool:
+    """Load sqlite-vec into this connection. False (never raises) on failure."""
+    try:
+        import sqlite_vec
+
+        conn.enable_load_extension(True)
+        sqlite_vec.load(conn)
+        conn.enable_load_extension(False)
+        return True
+    except Exception:
+        return False
+
+
+def vec_available(conn: sqlite3.Connection) -> bool:
+    """True if the sqlite-vec extension is usable on this connection."""
+    try:
+        conn.execute("SELECT vec_version()")
+        return True
+    except sqlite3.OperationalError:
+        return False
+
+
+def ensure_vec_schema(conn: sqlite3.Connection, model_name: str, dim: int) -> None:
+    """Create (or reset) the vector table for the given embedding model.
+
+    The model name and dimension are pinned in ``meta``. If the configured
+    model changed, existing vectors are useless — drop and start clean;
+    files re-embed as they are (re)indexed.
+    """
+    row = conn.execute(
+        "SELECT value FROM meta WHERE key = 'embedding_model'"
+    ).fetchone()
+    if row is not None and row["value"] != model_name:
+        conn.execute("DROP TABLE IF EXISTS chunk_vec")
+    conn.execute(
+        "CREATE VIRTUAL TABLE IF NOT EXISTS chunk_vec USING vec0("
+        f"chunk_id INTEGER PRIMARY KEY, embedding FLOAT[{dim}] distance_metric=cosine)"
+    )
+    conn.execute(
+        "INSERT OR REPLACE INTO meta (key, value) VALUES ('embedding_model', ?)", (model_name,)
+    )
+    conn.execute(
+        "INSERT OR REPLACE INTO meta (key, value) VALUES ('embedding_dim', ?)", (str(dim),)
+    )
+
+
+def vec_table_exists(conn: sqlite3.Connection) -> bool:
+    row = conn.execute(
+        "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'chunk_vec'"
+    ).fetchone()
+    return row is not None

docsonar/embeddings.py

@@ -0,0 +1,108 @@
+"""Local embeddings via sentence-transformers.
+
+The model loads lazily on first use (never at server startup — MCP
+clients expect a fast stdio handshake) and downloads on first run. Load
+failures are captured, not raised: the server degrades to keyword-only
+search and every affected tool response says so.
+
+Default model: BAAI/bge-small-en-v1.5 — 384 dims, ~130 MB, noticeably
+good retrieval quality.
+"""
+
+from __future__ import annotations
+
+import threading
+from typing import TYPE_CHECKING, Any, Protocol
+
+if TYPE_CHECKING:
+    import numpy as np
+
+DEFAULT_MODEL = "BAAI/bge-small-en-v1.5"
+
+# BGE v1.5 English models want this prefix on queries (not passages).
+_BGE_QUERY_PREFIX = "Represent this sentence for searching relevant passages: "
+
+
+class EmbedderLike(Protocol):
+    """What search/indexing need from an embedder; tests supply fakes."""
+
+    model_name: str
+
+    @property
+    def error(self) -> str | None: ...
+    @property
+    def dim(self) -> int: ...
+    def encode_passages(self, texts: list[str]) -> list[bytes]: ...
+    def encode_query(self, text: str) -> bytes: ...
+
+
+class Embedder:
+    """Lazy, thread-safe wrapper around a SentenceTransformer model."""
+
+    def __init__(self, model_name: str = DEFAULT_MODEL) -> None:
+        self.model_name = model_name
+        self._model: Any = None
+        self._dim: int = 0
+        self._error: str | None = None
+        self._lock = threading.Lock()
+
+    def _ensure_loaded(self) -> None:
+        with self._lock:
+            if self._model is not None or self._error is not None:
+                return
+            try:
+                from sentence_transformers import SentenceTransformer
+
+                model = SentenceTransformer(self.model_name, device="cpu")
+                # renamed in sentence-transformers 5.x; support both
+                get_dim = (
+                    getattr(model, "get_embedding_dimension", None)
+                    or model.get_sentence_embedding_dimension
+                )
+                self._dim = int(get_dim() or 0)
+                self._model = model
+            except Exception as exc:  # any failure => degrade, never crash
+                self._error = (
+                    f"could not load embedding model {self.model_name!r}: {exc}"
+                )
+
+    @property
+    def error(self) -> str | None:
+        """None if the model is usable; otherwise why it is not."""
+        self._ensure_loaded()
+        return self._error
+
+    def status_nonblocking(self) -> str:
+        """Current state without triggering a load (index_status must stay fast)."""
+        if self._model is not None:
+            return "ready"
+        if self._error is not None:
+            return f"error: {self._error}"
+        return "not loaded yet (loads on first index or semantic search)"
+
+    @property
+    def dim(self) -> int:
+        self._ensure_loaded()
+        return self._dim
+
+    def _encode(self, texts: list[str]) -> np.ndarray:
+        self._ensure_loaded()
+        if self._model is None:
+            raise RuntimeError(self._error or "embedding model not loaded")
+        import numpy as np
+
+        vectors = self._model.encode(
+            texts,
+            normalize_embeddings=True,
+            convert_to_numpy=True,
+            show_progress_bar=False,
+        )
+        return np.asarray(vectors, dtype=np.float32)
+
+    def encode_passages(self, texts: list[str]) -> list[bytes]:
+        return [bytes(v.tobytes()) for v in self._encode(texts)]
+
+    def encode_query(self, text: str) -> bytes:
+        if "bge-" in self.model_name and "-en-" in self.model_name:
+            text = _BGE_QUERY_PREFIX + text
+        return bytes(self._encode([text])[0].tobytes())