semdex 1.5.6__py3-none-any.whl

semdex/__init__.py

@@ -0,0 +1,20 @@
+"""Public package surface exposing metadata and configuration.
+
+This module provides the stable public API for the package, routing imports
+through the proper architectural layers:
+- Composition exports: Wired adapter services (configuration)
+- Metadata: Package information
+"""
+
+from __future__ import annotations
+
+# Metadata
+from .__init__conf__ import print_info
+
+# Composition exports (wired adapters)
+from .composition import get_config
+
+__all__ = [
+    "get_config",
+    "print_info",
+]

semdex/__init__conf__.py

@@ -0,0 +1,90 @@
+"""Static package metadata surfaced to CLI commands and documentation.
+
+Purpose
+-------
+Expose the current project metadata as simple constants. These values are kept
+in sync with ``pyproject.toml`` by development automation (tests, push
+pipelines), so runtime code does not query packaging metadata.
+
+Contents
+--------
+* Module-level constants describing the published package.
+* :func:`print_info` rendering the constants for the CLI ``info`` command.
+
+System Role
+-----------
+Lives in the adapters/platform layer; CLI transports import these constants to
+present authoritative project information without invoking packaging APIs.
+"""
+
+from __future__ import annotations
+
+import sys
+
+__all__ = [
+    "LAYEREDCONF_APP",
+    "LAYEREDCONF_SLUG",
+    "LAYEREDCONF_VENDOR",
+    "author",
+    "author_email",
+    "homepage",
+    "name",
+    "print_info",
+    "shell_command",
+    "title",
+    "version",
+]
+
+#: Distribution name declared in ``pyproject.toml``.
+name = "semdex"
+#: Human-readable summary shown in CLI help output.
+title = "Read-only, file-watching semantic-search MCP server and CLI over local files"
+#: Current release version pulled from ``pyproject.toml`` by automation.
+version = "1.5.6"
+#: Repository homepage presented to users.
+homepage = "https://github.com/bitranox/semdex"
+#: Author attribution surfaced in CLI output.
+author = "bitranox"
+#: Contact email surfaced in CLI output.
+author_email = "bitranox@gmail.com"
+#: Console-script name published by the package.
+shell_command = "semdex"
+
+#: Vendor identifier for lib_layered_config paths (macOS/Windows)
+LAYEREDCONF_VENDOR: str = "bitranox"
+#: Application display name for lib_layered_config paths (macOS/Windows)
+LAYEREDCONF_APP: str = "semdex"
+#: Configuration slug for lib_layered_config Linux paths and environment variables
+LAYEREDCONF_SLUG: str = "semdex"
+
+
+def print_info() -> None:
+    """Print the summarised metadata block used by the CLI ``info`` command.
+
+    Why
+        Provides a single, auditable rendering function so documentation and
+        CLI output always match the system design reference.
+
+    Side Effects
+        Writes to ``stdout``.
+
+    Examples
+    --------
+    >>> print_info()  # doctest: +ELLIPSIS
+    Info for semdex:
+    ...
+    """
+
+    fields = [
+        ("name", name),
+        ("title", title),
+        ("version", version),
+        ("homepage", homepage),
+        ("author", author),
+        ("author_email", author_email),
+        ("shell_command", shell_command),
+    ]
+    pad = max(len(label) for label, _ in fields)
+    lines = [f"Info for {name}:", ""]
+    lines.extend(f"    {label.ljust(pad)} = {value}" for label, value in fields)
+    sys.stdout.write("\n".join(lines) + "\n")

semdex/__main__.py

@@ -0,0 +1,14 @@
+"""Module entry point for ``python -m semdex``.
+
+Delegates to the same production wiring as the console script (``entry.main``)
+so ``python -m semdex`` and the installed ``semdex`` command behave identically -
+in particular both serve the index/search commands (the ``Bootstrap`` carries the
+index services factory, not just the AppServices factory).
+"""
+
+from __future__ import annotations
+
+from .entry import main
+
+if __name__ == "__main__":
+    raise SystemExit(main())

semdex/adapters/__init__.py

@@ -0,0 +1,15 @@
+"""Adapters layer - infrastructure and framework integrations.
+
+Contains adapter implementations that connect the application to external
+systems and frameworks (CLI, configuration, email, logging).
+
+Contents:
+    * :mod:`.config` - Configuration loading, deployment, and display
+    * :mod:`.email` - Email sending via SMTP
+    * :mod:`.logging` - Logging setup with lib_log_rich
+    * :mod:`.cli` - Click CLI framework integration
+"""
+
+from __future__ import annotations
+
+__all__: list[str] = []

semdex/adapters/_http_retry.py

@@ -0,0 +1,97 @@
+"""Shared retry-with-backoff for the adapters' HTTP transports.
+
+The embedding, summarizer, and extractor transports all POST to a model or
+document server that can drop a connection or return a transient 5xx/429
+mid-run - a single blip would otherwise kill an hours-long embed pass.
+``retry_http`` wraps ONLY the "post + raise_for_status" step of each transport
+in exponential backoff: a transient failure is retried, anything else re-raised
+at once, so each transport keeps its own response parsing and error mapping
+OUTSIDE the retry (a schema/JSON failure is a real error, never retried). httpx
+is imported lazily so this module stays importable without the optional HTTP
+extras installed.
+"""
+
+from __future__ import annotations
+
+import time
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from collections.abc import Callable
+
+    import httpx
+
+# Total attempts for one request (not "retries after the first"): with the
+# default 4, a request is tried up to 4 times before its last error propagates.
+DEFAULT_RETRIES = 4
+# First backoff pause in seconds; each subsequent pause doubles it (1, 2, 4, ...).
+DEFAULT_BASE_DELAY = 1.0
+# Upper bound (seconds) on any single backoff pause, so a long run of failures
+# does not grow the wait without limit.
+DEFAULT_MAX_DELAY = 30.0
+
+# The smallest HTTP status treated as a server-side (transient) failure. A 429
+# (rate limit / "retry later") is transient too; every other 4xx is the caller's
+# fault, where retrying never helps.
+_SERVER_ERROR_FLOOR = 500
+_TOO_MANY_REQUESTS = 429
+
+
+def is_transient(exc: BaseException) -> bool:
+    """Whether ``exc`` is a blip worth retrying rather than a permanent failure.
+
+    True for an httpx transport error (connect/read/write/pool timeout, a dropped
+    connection, a protocol error) and for an HTTP 5xx or 429 status. False for a
+    4xx status other than 429, a pydantic ``ValidationError``, a JSON decode
+    error, or any other exception - retrying those never helps.
+    """
+    import httpx  # noqa: PLC0415 - optional dependency httpx (in every HTTP-backed provider and extractor extra), loaded only when one is selected
+
+    if isinstance(exc, httpx.TransportError):
+        return True
+    if isinstance(exc, httpx.HTTPStatusError):
+        status = exc.response.status_code
+        return status == _TOO_MANY_REQUESTS or status >= _SERVER_ERROR_FLOOR
+    return False
+
+
+def retry_http(
+    do_request: Callable[[], httpx.Response],
+    *,
+    tries: int = DEFAULT_RETRIES,
+    base_delay: float = DEFAULT_BASE_DELAY,
+    max_delay: float = DEFAULT_MAX_DELAY,
+    sleep: Callable[[float], None] = time.sleep,
+) -> httpx.Response:
+    """Call ``do_request`` with exponential backoff on a transient failure.
+
+    ``do_request`` must perform the POST and ``raise_for_status``; its ``Response``
+    is returned. A transient failure (see :func:`is_transient`) is retried up to
+    ``tries`` attempts, pausing ``base_delay`` seconds and doubling each time
+    (capped at ``max_delay``); once the attempts are spent the last exception
+    propagates. A non-transient failure propagates immediately, so the caller's
+    own parsing and error mapping run on it unretried. ``sleep`` is injected so
+    tests exercise the backoff without real delay.
+    """
+    import httpx  # noqa: PLC0415 - optional dependency httpx (in every HTTP-backed provider and extractor extra), loaded only when one is selected
+
+    attempt = 0
+    delay = base_delay
+    while True:
+        try:
+            return do_request()
+        except httpx.HTTPError as exc:
+            attempt += 1
+            if attempt >= tries or not is_transient(exc):
+                raise
+            sleep(min(delay, max_delay))
+            delay *= 2
+
+
+__all__ = [
+    "DEFAULT_BASE_DELAY",
+    "DEFAULT_MAX_DELAY",
+    "DEFAULT_RETRIES",
+    "is_transient",
+    "retry_http",
+]

semdex/adapters/chunker/__init__.py

@@ -0,0 +1,20 @@
+"""Chunker adapters, each behind the ``ChunkText`` port.
+
+    * :class:`.whitespace.WhitespaceChunker` - zero-dependency fixed-window fallback
+    * :class:`.chonkie.ChonkieChunker` - recursive (default), semantic (SDPM), late
+    * :class:`.semantic_text_splitter.StsChunker` - markdown / fast (CommonMark, Rust)
+
+Selected by ``[chunker].strategy`` via ``composition.build_chunker``.
+"""
+
+from __future__ import annotations
+
+from .chonkie import ChonkieChunker
+from .semantic_text_splitter import StsChunker
+from .whitespace import WhitespaceChunker
+
+__all__ = [
+    "ChonkieChunker",
+    "StsChunker",
+    "WhitespaceChunker",
+]

semdex/adapters/chunker/_base.py

@@ -0,0 +1,117 @@
+"""Shared helpers for the chunker adapters.
+
+Turns a library's output into ordered domain :class:`~semdex.domain.models.Chunk`
+objects (dropping empties, numbering ordinals), provides the fixed-window
+splitter used by the whitespace fallback, and resolves a token-count callable
+from chonkie's tokenizer (offline for ``gpt2``/``character``) so the
+semantic-text-splitter strategies count tokens the same way the chonkie ones do.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from ...domain.errors import ChunkingError
+from ...domain.models import Chunk
+
+if TYPE_CHECKING:
+    from collections.abc import Callable, Iterable
+
+    from ...domain.models import ExtractedDocument, SourceRef
+
+
+def window_chunks(document: ExtractedDocument, *, max_tokens: int) -> list[Chunk]:
+    """Split a document into ordered, whitespace-token-bounded windows."""
+    tokens = document.text.split()
+    return [
+        Chunk(
+            text=" ".join(tokens[start : start + max_tokens]),
+            source=document.source,
+            ordinal=ordinal,
+            token_count=len(tokens[start : start + max_tokens]),
+        )
+        for ordinal, start in enumerate(range(0, len(tokens), max_tokens))
+    ]
+
+
+def split_oversized(
+    text: str,
+    token_count: int,
+    *,
+    max_tokens: int,
+    offsets_of: Callable[[str], list[tuple[int, int]]],
+) -> list[tuple[str, int]]:
+    """Split a chunk that exceeds ``max_tokens`` into <=max_tokens pieces, LOSSLESSLY.
+
+    The size-guard for the embedding-driven strategies (semantic/late), whose sentence-level
+    grouping can emit a chunk far over the target when a single "sentence" is un-splittable (a
+    minified-JS/webpack blob with no boundaries). Without this the embedder truncates the chunk to
+    its max_seq at embed time and silently drops the overflow; splitting here embeds ALL of it.
+
+    ``offsets_of(text)`` returns one ``(start_char, end_char)`` per token. The pieces TILE the
+    original text at token boundaries, so ``"".join(piece_texts) == text`` exactly (each piece is a
+    substring) and every piece holds at most ``max_tokens`` tokens. A within-cap chunk is returned
+    unchanged.
+
+    ``token_count`` and ``offsets_of`` need not agree on how many tokens the text holds, and for
+    the semantic and late strategies they systematically do not: chonkie's ``SemanticChunker``
+    accepts no tokenizer argument and counts in its embedding model's tokenizer, while the offsets
+    come from ``[chunker].tokenizer``. Measured divergence on the cached corpora is 6 to 22 percent
+    over byte-identical text. This used to return the chunk unchanged whenever OUR tokenizer said
+    it fit, which let a chunk recorded at 782 tokens through a 256 cap - and the embedder then
+    truncated it and dropped the overflow silently, the exact loss this guard exists to prevent.
+    So the RECORDED count decides how many pieces are needed, because that is the count the
+    embedder will act on, and the offsets decide only where the cuts fall.
+    """
+    if token_count <= max_tokens:
+        return [(text, token_count)]
+    offsets = offsets_of(text)
+    n = len(offsets)
+    if n <= 1:  # a single token, or a tokenizer that yielded nothing: there is no cut to make
+        return [(text, token_count)]
+    window = max_tokens
+    if n <= max_tokens:
+        # Our tokenizer says it fits and the recorded count says it does not. Split into as many
+        # EVEN pieces as the recorded count needs, so each piece lands under the cap in the terms
+        # that matter, rather than skipping the split entirely.
+        pieces_needed = -(-token_count // max_tokens)  # ceil
+        window = max(1, -(-n // pieces_needed))
+    pieces: list[tuple[str, int]] = []
+    for i in range(0, n, window):
+        start = 0 if i == 0 else offsets[i][0]
+        end = len(text) if i + window >= n else offsets[i + window][0]
+        pieces.append((text[start:end], len(offsets[i : i + window])))
+    return pieces
+
+
+def chunks_from_pairs(pairs: Iterable[tuple[str, int]], *, source: SourceRef) -> list[Chunk]:
+    """Build ordered domain chunks from ``(text, token_count)`` pairs, dropping empties."""
+    result: list[Chunk] = []
+    for text, token_count in pairs:
+        if not text.strip():
+            continue
+        result.append(Chunk(text=text, source=source, ordinal=len(result), token_count=int(token_count)))
+    return result
+
+
+def token_counter(tokenizer: str) -> Callable[[str], int]:
+    """Return a token-count callable from chonkie's tokenizer resolution.
+
+    chonkie is always present with the semantic-text-splitter strategies (both in
+    ``semdex[chunk]``), so reusing its tokenizer keeps token counts consistent
+    across strategies. Offline for ``gpt2``/``character``.
+    """
+    try:
+        from chonkie import RecursiveChunker  # noqa: PLC0415 - optional dependency chonkie (extra chunk), loaded only when a semantic-text-splitter chunker needs its tokenizer
+    except ImportError as exc:  # pragma: no cover - only without semdex[chunk]
+        raise ChunkingError("chonkie is not installed; install semdex[chunk]") from exc
+    counter: Callable[[str], int] = RecursiveChunker(tokenizer=tokenizer, chunk_size=2048).tokenizer.count_tokens
+    return counter
+
+
+__all__ = [
+    "chunks_from_pairs",
+    "split_oversized",
+    "token_counter",
+    "window_chunks",
+]

semdex/adapters/chunker/chonkie.py

@@ -0,0 +1,209 @@
+"""chonkie chunker adapter: recursive (default), semantic (SDPM), and late.
+
+``RECURSIVE`` uses ``RecursiveChunker`` with the markdown recipe (token-accurate,
+no embedding spend) plus ``OverlapRefinery`` when overlap>0; the recipe falls
+back to chonkie's default rules if it cannot be fetched (offline). ``SEMANTIC``
+uses ``SemanticChunker`` with a skip window (the SDPM double-pass merge) and
+``LATE`` uses ``LateChunker``; both need ``semdex[chunk-semantic]`` and only beat
+recursive with a real embedding provider. The underlying chunker is built once
+and rebuilt only if ``max_tokens`` changes.
+"""
+
+from __future__ import annotations
+
+import importlib
+import logging
+from functools import lru_cache
+from typing import TYPE_CHECKING, Any, cast
+
+from ...domain.enums import ChunkStrategy
+from ...domain.errors import ChunkingError
+from ._base import chunks_from_pairs, split_oversized
+
+if TYPE_CHECKING:
+    from collections.abc import Iterable, Iterator
+
+    from ...domain.models import Chunk, ExtractedDocument
+
+logger = logging.getLogger(__name__)
+
+_DEFAULT_MAX_TOKENS = 256
+
+
+def _whitespace_offsets(text: str) -> list[tuple[int, int]]:
+    """Fallback token offsets (one per whitespace word) when the tokenizer is a bare string."""
+    offsets: list[tuple[int, int]] = []
+    cursor = 0
+    for word in text.split(" "):
+        start = text.index(word, cursor) if word else cursor
+        offsets.append((start, start + len(word)))
+        cursor = start + len(word)
+    return offsets
+
+
+# Tokenizer names chonkie tokenizes locally (no model, no network) - pass them through as-is.
+_CHONKIE_BUILTIN_TOKENIZERS = frozenset({"character", "word", "byte", "row"})
+# Bare HuggingFace ids whose local cache lives under the canonical org-qualified repo, so the
+# bare id is not found offline. Map to the cached repo id. Extend as more tokenizers are used.
+_TOKENIZER_ALIASES = {"gpt2": "openai-community/gpt2"}
+
+
+@lru_cache(maxsize=16)
+def resolve_tokenizer(name: str) -> Any:
+    """Resolve a tokenizer NAME to a concrete ``tokenizers.Tokenizer`` object once (cached).
+
+    Handing chonkie a tokenizer *string* makes it resolve through its ``tokie`` backend, whose
+    ``from_pretrained`` issues a blocking hub request that ignores ``HF_HUB_OFFLINE`` and stalls
+    for minutes on a slow HuggingFace CDN (it only bites the overlap path, ``OverlapRefinery``,
+    so it is invisible until overlap>0). Passing a pre-built ``Tokenizer`` object skips that path
+    entirely. Builtin names (``character``/``word``/...) tokenize locally, so they pass through;
+    an unresolvable name also falls back to the string so chonkie can try its own resolution.
+
+    Upstream issue: https://github.com/feyninc/chonkie/issues/631
+    """
+    if name in _CHONKIE_BUILTIN_TOKENIZERS:
+        return name
+    try:
+        tokenizers: Any = importlib.import_module("tokenizers")
+    except ImportError:
+        return name
+    for repo_id in (name, _TOKENIZER_ALIASES.get(name)):
+        if not repo_id:
+            continue
+        try:
+            return tokenizers.Tokenizer.from_pretrained(repo_id)
+        except Exception as exc:  # any load failure (offline miss, bad id) falls back to the string
+            logger.debug("tokenizer %r not loadable as %r (%s)", name, repo_id, exc)
+    return name
+
+
+class ChonkieChunker:
+    """Chunk a document with chonkie's recursive/semantic/late chunkers."""
+
+    def __init__(  # noqa: PLR0913 - config-holding constructor; each arg is a distinct chunk setting
+        self,
+        strategy: ChunkStrategy,
+        *,
+        recipe: str = "markdown",
+        overlap: int = 0,
+        tokenizer: str = "gpt2",
+        semantic_model: str | Any | None = None,
+        enforce_max_tokens: bool = True,
+    ) -> None:
+        self._strategy = strategy
+        self._recipe = recipe
+        self._overlap = overlap
+        self._tokenizer = tokenizer
+        # Lossless size-guard: split any chunk over max_tokens into token windows. The semantic/late
+        # strategies group at sentence granularity, so an un-splittable "sentence" (a minified-JS blob)
+        # can overflow far past the target; without this the embedder truncates it and drops the rest.
+        self._enforce_max_tokens = enforce_max_tokens
+        # The embedding model SemanticChunker/LateChunker cut boundaries on. None keeps chonkie's
+        # own default (potion-base-32M), which is English-distilled - set an explicit multilingual
+        # model for non-English corpora, where the default finds boundaries on text it cannot read.
+        # A str is resolved by chonkie locally (model2vec/sentence-transformers); a pre-built chonkie
+        # BaseEmbeddings object is passed through as-is (e.g. an endpoint-backed breakpoint model).
+        self._semantic_model = semantic_model
+        self._cached: tuple[int, Any] | None = None
+
+    def __call__(self, document: ExtractedDocument, *, max_tokens: int = _DEFAULT_MAX_TOKENS) -> list[Chunk]:
+        chunker = self._resolve(max_tokens)
+        raw = chunker(document.text)
+        if self._strategy is ChunkStrategy.RECURSIVE and self._overlap > 0:
+            raw = self._overlapped(raw)
+        pairs: Iterable[tuple[str, int]] = ((chunk.text, chunk.token_count) for chunk in raw)
+        if self._strategy is ChunkStrategy.LATE:
+            # chonkie's LateChunker reports token_count 1 for every chunk of a long document (a
+            # 30,000-character court decision: 60 of 60 chunks) while counting a short one
+            # correctly. A count of 1 satisfies any cap, so the size-guard below would pass an
+            # oversized late chunk through to be clipped by the embedder. The count is therefore
+            # never taken from chonkie for late chunks: it is re-measured in the configured
+            # tokenizer, the unit recursive chunks are counted in.
+            pairs = ((chunk.text, len(self._offsets_of(chunk.text))) for chunk in raw)
+        # Only the embedding-driven strategies overflow (sentence-granularity grouping of an
+        # un-splittable blob). recursive caps already, and its overlap intentionally pushes a chunk
+        # slightly over max_tokens - guarding it would wrongly re-split those overlap chunks.
+        if self._enforce_max_tokens and self._strategy in (ChunkStrategy.SEMANTIC, ChunkStrategy.LATE):
+            pairs = self._guard(pairs, max_tokens)
+        return chunks_from_pairs(pairs, source=document.source)
+
+    def _guard(self, pairs: Iterable[tuple[str, int]], max_tokens: int) -> Iterator[tuple[str, int]]:
+        """Split any oversized (text, token_count) pair into <=max_tokens pieces, losslessly."""
+        for text, count in pairs:
+            yield from split_oversized(text, count, max_tokens=max_tokens, offsets_of=self._offsets_of)
+
+    def _offsets_of(self, text: str) -> list[tuple[int, int]]:
+        """Per-token (start, end) char offsets from the resolved tokenizer; whitespace fallback.
+
+        ``resolve_tokenizer`` returns either a bare string (builtin/unresolvable name) or a
+        ``tokenizers.Tokenizer`` object; the latter's ``encode(text).offsets`` is read at this
+        optional-dependency boundary via ``getattr`` and coerced to a concrete ``list[tuple[int, int]]``.
+        """
+        tok: Any = resolve_tokenizer(self._tokenizer)
+        offsets: list[tuple[int, int]] = []
+        if callable(getattr(tok, "encode", None)):
+            try:
+                # boundary read of the optional-dep Tokenizer's Encoding.offsets
+                offsets = cast("list[tuple[int, int]]", tok.encode(text).offsets)
+            except Exception:
+                # a bare-string tokenizer, or an encode quirk of whichever tokenizers
+                # build is installed: leave offsets empty and fall back to whitespace.
+                offsets = []
+        return list(offsets) if offsets else _whitespace_offsets(text)
+
+    def _resolve(self, max_tokens: int) -> Any:
+        if self._cached is not None and self._cached[0] == max_tokens:
+            return self._cached[1]
+        chunker = self._build(max_tokens)
+        self._cached = (max_tokens, chunker)
+        return chunker
+
+    def _build(self, max_tokens: int) -> Any:
+        try:
+            import chonkie  # noqa: PLC0415 - optional dependency chonkie (extra chunk), loaded only when the chonkie chunker builds
+        except ImportError as exc:
+            raise ChunkingError("chonkie is not installed; install semdex[chunk]") from exc
+        if self._strategy is ChunkStrategy.RECURSIVE:
+            return self._build_recursive(chonkie, max_tokens)
+        # Pass embedding_model only when set (explicit calls, not a **spread, so the typed
+        # chonkie kwargs stay checked), so an unset model keeps chonkie's own default.
+        model = self._semantic_model
+        if self._strategy is ChunkStrategy.SEMANTIC:
+            # A non-zero skip window makes SemanticChunker do the SDPM double-pass merge.
+            if model:
+                return chonkie.SemanticChunker(chunk_size=max_tokens, skip_window=1, embedding_model=model)
+            return chonkie.SemanticChunker(chunk_size=max_tokens, skip_window=1)
+        if model:
+            return chonkie.LateChunker(chunk_size=max_tokens, embedding_model=model)
+        return chonkie.LateChunker(chunk_size=max_tokens)
+
+    def _build_recursive(self, chonkie: Any, max_tokens: int) -> Any:
+        tokenizer = resolve_tokenizer(self._tokenizer)
+        if self._recipe:
+            try:
+                return chonkie.RecursiveChunker.from_recipe(
+                    self._recipe, lang="en", tokenizer=tokenizer, chunk_size=max_tokens
+                )
+            except Exception as exc:
+                logger.warning(
+                    "chonkie recipe %r unavailable (%s); using the default recursive rules", self._recipe, exc
+                )
+        return chonkie.RecursiveChunker(tokenizer=tokenizer, chunk_size=max_tokens)
+
+    def _overlapped(self, raw: Any) -> Any:
+        import chonkie  # noqa: PLC0415 - optional dependency chonkie (extra chunk), loaded only when the chonkie chunker applies overlap
+
+        return chonkie.OverlapRefinery(tokenizer=resolve_tokenizer(self._tokenizer), context_size=self._overlap)(raw)
+
+
+# Static conformance assertion -- pyright verifies ChonkieChunker satisfies ChunkText.
+if TYPE_CHECKING:
+    from ...application.ports import ChunkText
+
+    _assert_chunk: ChunkText = ChonkieChunker(ChunkStrategy.RECURSIVE)
+
+
+__all__ = [
+    "ChonkieChunker",
+    "resolve_tokenizer",
+]

semdex/adapters/chunker/semantic_text_splitter.py

@@ -0,0 +1,67 @@
+"""semantic-text-splitter chunker adapter: markdown and fast recursive.
+
+``MARKDOWN`` uses ``MarkdownSplitter`` (true CommonMark parsing, so it never
+splits inside a code block or table); ``FAST`` uses ``TextSplitter`` (plain fast
+recursive). Both are Rust-backed with native overlap. Token-accurate sizing uses
+chonkie's tokenizer as the length callback (see :func:`._base.token_counter`), so
+chunk sizes match the chonkie strategies. Needs ``semdex[chunk]``.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+from ...domain.enums import ChunkStrategy
+from ...domain.errors import ChunkingError
+from ._base import chunks_from_pairs, token_counter
+
+if TYPE_CHECKING:
+    from collections.abc import Callable
+
+    from ...domain.models import Chunk, ExtractedDocument
+
+_DEFAULT_MAX_TOKENS = 256
+
+
+class StsChunker:
+    """Chunk a document with semantic-text-splitter's Markdown/Text splitter."""
+
+    def __init__(self, strategy: ChunkStrategy, *, overlap: int = 0, tokenizer: str = "gpt2") -> None:
+        self._strategy = strategy
+        self._overlap = overlap
+        self._count = token_counter(tokenizer)
+        self._cached: tuple[int, Any] | None = None
+
+    def __call__(self, document: ExtractedDocument, *, max_tokens: int = _DEFAULT_MAX_TOKENS) -> list[Chunk]:
+        splitter = self._resolve(max_tokens)
+        texts = splitter.chunks(document.text)
+        return chunks_from_pairs(((text, self._count(text)) for text in texts), source=document.source)
+
+    def _resolve(self, max_tokens: int) -> Any:
+        if self._cached is not None and self._cached[0] == max_tokens:
+            return self._cached[1]
+        splitter = self._build(max_tokens)
+        self._cached = (max_tokens, splitter)
+        return splitter
+
+    def _build(self, max_tokens: int) -> Any:
+        try:
+            from semantic_text_splitter import MarkdownSplitter, TextSplitter  # noqa: PLC0415 - optional dependency semantic-text-splitter (extra chunk), loaded only when the sts chunker builds
+        except ImportError as exc:
+            raise ChunkingError("semantic-text-splitter is not installed; install semdex[chunk]") from exc
+        counter: Callable[[str], int] = self._count
+        if self._strategy is ChunkStrategy.MARKDOWN:
+            return MarkdownSplitter.from_callback(counter, max_tokens, overlap=self._overlap)
+        return TextSplitter.from_callback(counter, max_tokens, overlap=self._overlap)
+
+
+# Static conformance assertion -- pyright verifies StsChunker satisfies ChunkText.
+if TYPE_CHECKING:
+    from ...application.ports import ChunkText
+
+    _assert_chunk: ChunkText = StsChunker(ChunkStrategy.MARKDOWN)
+
+
+__all__ = [
+    "StsChunker",
+]