chunkwise 0.1.0__py3-none-any.whl

chunkwise/__init__.py

@@ -0,0 +1,48 @@
+"""chunkwise: text chunking for RAG with reversible offsets.
+
+Every chunk carries the exact character range it came from, so
+``document[chunk.start:chunk.end] == chunk.text`` always holds.
+"""
+
+from __future__ import annotations
+
+from ._spans import DEFAULT_TOKENIZER
+from .strategies import chunk_fixed, chunk_markdown, chunk_recursive
+from .types import Chunk, Tokenizer
+
+__version__ = "0.1.0"
+
+_STRATEGIES = {
+    "fixed": chunk_fixed,
+    "recursive": chunk_recursive,
+    "markdown": chunk_markdown,
+}
+
+
+def chunk(
+    document: str,
+    strategy: str = "recursive",
+    max_tokens: int = 512,
+    overlap: int = 0,
+    tokenizer: Tokenizer = DEFAULT_TOKENIZER,
+) -> list[Chunk]:
+    """Split `document` into a list of Chunk objects.
+
+    Args:
+        document: the source text to split.
+        strategy: one of "fixed", "recursive" (default), "markdown".
+        max_tokens: the token budget per chunk, as measured by `tokenizer`.
+        overlap: how many tokens of overlap between consecutive chunks.
+        tokenizer: a callable from str to int token count. Defaults to
+            character counting, keeping the base install dependency-free.
+    """
+    try:
+        strategy_fn = _STRATEGIES[strategy]
+    except KeyError:
+        raise ValueError(
+            f"Unknown strategy {strategy!r}. Available: {sorted(_STRATEGIES)}"
+        ) from None
+    return strategy_fn(document, max_tokens, overlap, tokenizer)
+
+
+__all__ = ["chunk", "Chunk", "Tokenizer", "__version__"]

chunkwise/_spans.py

@@ -0,0 +1,47 @@
+"""Internal offset arithmetic shared by strategies.
+
+Strategies operate on (start, end) integer spans, not strings, until the
+final materialization step. This is what makes the offset invariant
+structurally impossible to break.
+"""
+
+from __future__ import annotations
+
+from .types import Chunk, Tokenizer
+
+def _char_count(text: str) -> int:
+    return len(text)
+
+
+DEFAULT_TOKENIZER: Tokenizer = _char_count
+
+
+def materialize(document: str, spans: list[tuple[int, int]]) -> list[Chunk]:
+    """Turn (start, end) spans into Chunk objects with text sliced from document."""
+    return [
+        Chunk(text=document[start:end], start=start, end=end, index=i)
+        for i, (start, end) in enumerate(spans)
+    ]
+
+
+def find_boundary(
+    document: str,
+    start: int,
+    budget: int,
+    tokenizer: Tokenizer,
+) -> int:
+    """Binary search for the largest end offset such that tokenizer(document[start:end]) <= budget.
+
+    Works with any opaque tokenizer rather than assuming a fixed characters-to-tokens ratio.
+    """
+    lo, hi = start, len(document)
+    if tokenizer(document[start:hi]) <= budget:
+        return hi
+
+    while hi - lo > 1:
+        mid = (lo + hi) // 2
+        if tokenizer(document[start:mid]) <= budget:
+            lo = mid
+        else:
+            hi = mid
+    return lo if lo > start else min(start + 1, len(document))

chunkwise/strategies/__init__.py

@@ -0,0 +1,7 @@
+"""Chunking strategies."""
+
+from .fixed import chunk_fixed
+from .markdown import chunk_markdown
+from .recursive import chunk_recursive
+
+__all__ = ["chunk_fixed", "chunk_recursive", "chunk_markdown"]

chunkwise/strategies/code.py

@@ -0,0 +1,19 @@
+"""Code-aware strategy: splits on function and class boundaries.
+
+Planned for v0.2. Will use `ast` for Python and `tree-sitter` for other
+languages to find boundaries, then chunk on those spans the same way
+`recursive` chunks on separators.
+"""
+
+from __future__ import annotations
+
+from ..types import Chunk, Tokenizer
+
+
+def chunk_code(
+    document: str,
+    max_tokens: int,
+    overlap: int,
+    tokenizer: Tokenizer,
+) -> list[Chunk]:
+    raise NotImplementedError("chunk_code is planned for v0.2")

chunkwise/strategies/fixed.py

@@ -0,0 +1,36 @@
+"""Fixed-window strategy: hard windows of N tokens with configurable overlap."""
+
+from __future__ import annotations
+
+from .._spans import find_boundary, materialize
+from ..types import Chunk, Tokenizer
+
+
+def chunk_fixed(
+    document: str,
+    max_tokens: int,
+    overlap: int,
+    tokenizer: Tokenizer,
+) -> list[Chunk]:
+    """Split document into hard windows of at most max_tokens, overlapping by `overlap` tokens.
+
+    Overlap widens a chunk backwards into real preceding text rather than
+    gluing on a copied prefix, preserving the offset invariant.
+    """
+    if max_tokens <= 0:
+        raise ValueError("max_tokens must be positive")
+    if overlap < 0 or overlap >= max_tokens:
+        raise ValueError("overlap must be in [0, max_tokens)")
+
+    spans: list[tuple[int, int]] = []
+    start = 0
+    n = len(document)
+    while start < n:
+        end = find_boundary(document, start, max_tokens, tokenizer)
+        spans.append((start, end))
+        if end >= n:
+            break
+        back_start = find_boundary(document, start, max_tokens - overlap, tokenizer)
+        start = max(back_start, start + 1)
+
+    return materialize(document, spans)

chunkwise/strategies/markdown.py

@@ -0,0 +1,75 @@
+"""Markdown-aware strategy: keeps fenced code blocks intact, tags chunks with heading paths."""
+
+from __future__ import annotations
+
+import re
+
+from .._spans import find_boundary
+from ..types import Chunk, Tokenizer
+
+HEADING_RE = re.compile(r"^(#{1,6})\s+(.*)$", re.MULTILINE)
+FENCE_RE = re.compile(r"^```.*?$.*?^```$", re.MULTILINE | re.DOTALL)
+
+
+def _code_fence_spans(document: str) -> list[tuple[int, int]]:
+    return [(m.start(), m.end()) for m in FENCE_RE.finditer(document)]
+
+
+def _heading_path_at(document: str, offset: int) -> tuple[str, ...]:
+    stack: list[tuple[int, str]] = []
+    for match in HEADING_RE.finditer(document):
+        if match.start() > offset:
+            break
+        level = len(match.group(1))
+        title = match.group(2).strip()
+        stack = [(lvl, t) for lvl, t in stack if lvl < level]
+        stack.append((level, title))
+    return tuple(title for _, title in stack)
+
+
+def chunk_markdown(
+    document: str,
+    max_tokens: int,
+    overlap: int,
+    tokenizer: Tokenizer,
+) -> list[Chunk]:
+    """Split markdown document, never breaking a fenced code block, tagging heading paths."""
+    if max_tokens <= 0:
+        raise ValueError("max_tokens must be positive")
+    if overlap < 0 or overlap >= max_tokens:
+        raise ValueError("overlap must be in [0, max_tokens)")
+
+    fences = _code_fence_spans(document)
+    spans: list[tuple[int, int]] = []
+    start = 0
+    n = len(document)
+
+    while start < n:
+        hard_limit = find_boundary(document, start, max_tokens, tokenizer)
+
+        for fence_start, fence_end in fences:
+            if fence_start < hard_limit < fence_end:
+                hard_limit = fence_end
+                break
+
+        end = max(hard_limit, start + 1)
+        end = min(end, n)
+        spans.append((start, end))
+        if end >= n:
+            break
+        if overlap == 0:
+            start = end
+        else:
+            back_start = find_boundary(document, start, max_tokens - overlap, tokenizer)
+            start = max(min(back_start, end), start + 1)
+
+    return [
+        Chunk(
+            text=document[s:e],
+            start=s,
+            end=e,
+            index=i,
+            heading_path=_heading_path_at(document, s),
+        )
+        for i, (s, e) in enumerate(spans)
+    ]

chunkwise/strategies/recursive.py

@@ -0,0 +1,61 @@
+"""Recursive strategy: falls back through a separator chain to find natural breaks.
+
+Breaks on the coarsest boundary that fits the budget — paragraphs, then lines,
+then sentences, then words. This is the default strategy.
+"""
+
+from __future__ import annotations
+
+import re
+
+from .._spans import find_boundary, materialize
+from ..types import Chunk, Tokenizer
+
+SEPARATOR_CHAIN = [
+    re.compile(r"\n\s*\n"),  # paragraphs
+    re.compile(r"\n"),  # lines
+    re.compile(r"(?<=[.!?])\s+"),  # sentences
+    re.compile(r"\s+"),  # words
+]
+
+
+def _best_split_point(document: str, start: int, limit: int) -> int:
+    """Find the best separator-aligned split point within (start, limit]."""
+    window = document[start:limit]
+    for separator in SEPARATOR_CHAIN:
+        matches = list(separator.finditer(window))
+        if matches:
+            last = matches[-1]
+            split = start + last.end()
+            if split > start:
+                return split
+    return limit
+
+
+def chunk_recursive(
+    document: str,
+    max_tokens: int,
+    overlap: int,
+    tokenizer: Tokenizer,
+) -> list[Chunk]:
+    """Split document on the coarsest separator that keeps each chunk within budget."""
+    if max_tokens <= 0:
+        raise ValueError("max_tokens must be positive")
+    if overlap < 0 or overlap >= max_tokens:
+        raise ValueError("overlap must be in [0, max_tokens)")
+
+    spans: list[tuple[int, int]] = []
+    start = 0
+    n = len(document)
+    while start < n:
+        hard_limit = find_boundary(document, start, max_tokens, tokenizer)
+        end = _best_split_point(document, start, hard_limit) if hard_limit < n else hard_limit
+        if end <= start:
+            end = hard_limit
+        spans.append((start, end))
+        if end >= n:
+            break
+        back_start = find_boundary(document, start, max_tokens - overlap, tokenizer)
+        start = max(min(back_start, end), start + 1)
+
+    return materialize(document, spans)

chunkwise/strategies/semantic.py

@@ -0,0 +1,20 @@
+"""Semantic strategy: cuts where sentence-embedding similarity drops.
+
+Planned for v0.3, shipping as an optional extra (`pip install chunkwise[semantic]`)
+so the base install stays dependency-free.
+"""
+
+from __future__ import annotations
+
+from ..types import Chunk, Tokenizer
+
+
+def chunk_semantic(
+    document: str,
+    max_tokens: int,
+    overlap: int,
+    tokenizer: Tokenizer,
+) -> list[Chunk]:
+    raise NotImplementedError(
+        "chunk_semantic is planned for v0.3 and requires the 'semantic' extra"
+    )

chunkwise/types.py

@@ -0,0 +1,41 @@
+"""Core types: the Chunk dataclass and the Tokenizer protocol."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Protocol, runtime_checkable
+
+
+@runtime_checkable
+class Tokenizer(Protocol):
+    """Any callable from str to an integer token count.
+
+    The only requirement is that the count is monotonic in string length —
+    every real tokenizer (character counting, tiktoken, transformers) satisfies this.
+    """
+
+    def __call__(self, text: str) -> int: ...
+
+
+@dataclass(frozen=True, slots=True)
+class Chunk:
+    """A chunk of a source document.
+
+    The invariant ``document[chunk.start:chunk.end] == chunk.text`` is enforced
+    on construction: a Chunk cannot exist with a span that doesn't match its text.
+    """
+
+    text: str
+    start: int
+    end: int
+    index: int = 0
+    heading_path: tuple[str, ...] = field(default_factory=tuple)
+
+    def __post_init__(self) -> None:
+        if self.end - self.start != len(self.text):
+            raise ValueError(
+                f"Chunk span ({self.start}, {self.end}) does not match "
+                f"text length {len(self.text)}"
+            )
+        if self.start < 0 or self.end < self.start:
+            raise ValueError(f"Invalid span: ({self.start}, {self.end})")

chunkwise-0.1.0.dist-info/METADATA

@@ -0,0 +1,168 @@
+Metadata-Version: 2.5
+Name: chunkwise
+Version: 0.1.0
+Summary: Text chunking for RAG with reversible offsets — every chunk is an exact slice of the source document.
+Project-URL: Homepage, https://github.com/amin-albooyeh/chunkwise
+Project-URL: Repository, https://github.com/amin-albooyeh/chunkwise
+Project-URL: Documentation, https://github.com/amin-albooyeh/chunkwise/tree/main/docs
+Project-URL: Changelog, https://github.com/amin-albooyeh/chunkwise/blob/main/CHANGELOG.md
+Author-email: Amin Albooyeh <amin.albooyeh@gmail.com>
+License-Expression: MIT
+License-File: LICENSE
+Keywords: chunking,embeddings,nlp,rag,retrieval
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Text Processing :: Linguistic
+Classifier: Typing :: Typed
+Requires-Python: >=3.10
+Provides-Extra: dev
+Requires-Dist: hypothesis>=6.0; extra == 'dev'
+Requires-Dist: mypy>=1.10; extra == 'dev'
+Requires-Dist: pytest>=8.0; extra == 'dev'
+Requires-Dist: ruff>=0.6; extra == 'dev'
+Provides-Extra: semantic
+Requires-Dist: numpy; extra == 'semantic'
+Description-Content-Type: text/markdown
+
+# chunkwise
+
+**Text chunking for RAG with reversible offsets — every chunk is an exact slice of the source document.**
+
+[![CI](https://github.com/amin-albooyeh/chunkwise/actions/workflows/ci.yml/badge.svg)](https://github.com/amin-albooyeh/chunkwise/actions/workflows/ci.yml)
+[![PyPI](https://img.shields.io/pypi/v/chunkwise.svg)](https://pypi.org/project/chunkwise/)
+[![Python versions](https://img.shields.io/pypi/pyversions/chunkwise.svg)](https://pypi.org/project/chunkwise/)
+[![License: MIT](https://img.shields.io/badge/license-MIT-green.svg)](LICENSE)
+
+chunkwise splits documents into embeddable chunks for retrieval-augmented generation. Unlike most splitters, which return detached strings, every chunk carries the exact character range it came from, so `document[chunk.start:chunk.end] == chunk.text` always holds — making source highlighting, citation, and context expansion possible. Zero runtime dependencies, bring-your-own tokenizer, and structure-aware strategies for plain text and Markdown.
+
+## Why chunkwise
+
+Most chunking libraries hand back a `list[str]`. The moment text is detached from its origin, three things become impossible:
+
+- You can't **highlight** the passage a retrieval hit came from in the original document.
+- You can't **expand** a result to show the surrounding paragraph.
+- You can't tell **which section or heading** a passage lived under.
+
+Worse, splitters that strip or normalize whitespace silently drop characters — a failure mode that shows up only as retrieval quietly missing content, which is hard to notice and harder to debug.
+
+## Install
+
+```bash
+pip install chunkwise
+```
+
+No dependencies are pulled in. (An optional `semantic` extra will be added in v0.3 for embedding-based chunking.)
+
+## Quickstart
+
+```python
+from chunkwise import chunk
+
+document = open("report.md").read()
+chunks = chunk(document, strategy="recursive", max_tokens=512, overlap=50)
+
+for c in chunks:
+    assert document[c.start:c.end] == c.text  # always true, for every chunk
+    print(c.index, c.start, c.end, c.heading_path)
+```
+
+## The guarantee
+
+chunkwise holds one invariant across every strategy and every input:
+
+```python
+document[chunk.start:chunk.end] == chunk.text
+```
+
+This is enforced three ways:
+
+1. **Structurally** — internal logic operates on `(start, end)` integer spans and only materializes text at the final step, so no strategy (present or future) can break the invariant even by accident.
+2. **At runtime** — the `Chunk` type validates its own span on construction.
+3. **In CI** — a Hypothesis property test runs against generated documents on every commit, alongside a test that concatenating non-overlapping chunks reproduces the source byte for byte.
+
+The consequences are deliberate: chunks are never stripped, normalized, or rewritten, and overlap widens a chunk backwards into real preceding text rather than gluing on a copied prefix.
+
+See [docs/design.md](docs/design.md) for the full rationale.
+
+## Strategies
+
+| Strategy | Behavior |
+|---|---|
+| `fixed` | Hard windows of N tokens with configurable overlap. Fast, boundary-blind. Suited to logs and unstructured text. |
+| `recursive` | Falls back through a separator chain — paragraphs, then lines, then sentences, then words — breaking on the coarsest boundary that fits the budget. **The default.** |
+| `markdown` | Keeps fenced code blocks intact and tags every chunk with its heading path (e.g. `["Guide", "Install"]`), giving retrieval results a sense of place. |
+
+Planned: `code` (v0.2 — AST / tree-sitter boundaries) and `semantic` (v0.3 — embedding similarity cuts), the latter shipping as an optional extra so the base install stays dependency-free.
+
+See [docs/strategies.md](docs/strategies.md) for guidance on choosing one.
+
+## Tokenizer independence
+
+Token budgets depend on which model you're embedding for, so chunkwise doesn't hard-code one. The `tokenizer` parameter accepts any callable from `str` to `int`:
+
+```python
+from chunkwise import chunk
+import tiktoken
+
+enc = tiktoken.get_encoding("cl100k_base")
+chunks = chunk(document, max_tokens=512, tokenizer=lambda text: len(enc.encode(text)))
+```
+
+The default counts characters, keeping the base install at zero dependencies. The only requirement on a tokenizer is that its count is monotonic in string length — every real tokenizer satisfies this. See [docs/tokenizers.md](docs/tokenizers.md).
+
+## API
+
+```python
+def chunk(
+    document: str,
+    strategy: str = "recursive",   # "fixed" | "recursive" | "markdown"
+    max_tokens: int = 512,
+    overlap: int = 0,
+    tokenizer: Tokenizer = ...,    # str -> int, defaults to len
+) -> list[Chunk]: ...
+```
+
+Each `Chunk` is a frozen dataclass:
+
+```python
+@dataclass(frozen=True, slots=True)
+class Chunk:
+    text: str
+    start: int
+    end: int
+    index: int
+    heading_path: tuple[str, ...]  # populated by the markdown strategy
+```
+
+## Status
+
+v0.1.0 · Python 3.10+ · MIT licensed · zero runtime dependencies · tested on CI across Python 3.10–3.13.
+
+## Documentation
+
+- [docs/index.md](docs/index.md) — overview and quickstart
+- [docs/strategies.md](docs/strategies.md) — strategy reference and how to choose
+- [docs/tokenizers.md](docs/tokenizers.md) — bringing your own tokenizer
+- [docs/design.md](docs/design.md) — why spans, not strings
+
+## Benchmarks
+
+```bash
+python benchmarks/throughput.py
+```
+
+Compares chunkwise strategies against a naive fixed-width splitter on a synthetic document.
+
+## Contributing
+
+Contributions are welcome — see [CONTRIBUTING.md](CONTRIBUTING.md) for setup, checks, and guidelines (in particular, the offset invariant that any new strategy must preserve).
+
+## License
+
+MIT — see [LICENSE](LICENSE).

chunkwise-0.1.0.dist-info/RECORD

@@ -0,0 +1,14 @@
+chunkwise/__init__.py,sha256=MevyLwjm1SBbXhM7WXtId_iyWerJTT2r6j3qDegN1do,1487
+chunkwise/_spans.py,sha256=B862WnbFxUsmltwIQsM6ZbsvaJsw5CHjY7BuD87OSUc,1345
+chunkwise/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+chunkwise/types.py,sha256=UAefS_j9X9LsptcfAzbfgK7_zQIvc_XIsgyl2gEMtLs,1282
+chunkwise/strategies/__init__.py,sha256=nCuao_4iyo2sp7MF9ELm2Gpr5E0x5j55-xvXvEjnmVI,199
+chunkwise/strategies/code.py,sha256=QhAQQ7jmTxnh9Z1zO2bJ-1DJn6ngPTlR7Msrus13zt4,503
+chunkwise/strategies/fixed.py,sha256=f8fbDVeejIzHvQxoM5vkNjAxNlz8NzAGvmosKM40Lxk,1162
+chunkwise/strategies/markdown.py,sha256=W-JEezPYNFQeVeZIdK9YOZv_g32emD5JY16P6mVs2cw,2279
+chunkwise/strategies/recursive.py,sha256=iyn13N_5SW_A2DhT1MgZkM4ywH9NaO5jVHAdP7isaLk,1950
+chunkwise/strategies/semantic.py,sha256=OzKBSFF9lVeAO9qNHbqCvSA4Xj_Heg8dtXEAQI3GUeI,516
+chunkwise-0.1.0.dist-info/METADATA,sha256=6v40mQ29hqSxtI_-icdrzNyf2t7YQeM8JCSma7HVkfA,7121
+chunkwise-0.1.0.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
+chunkwise-0.1.0.dist-info/licenses/LICENSE,sha256=191kzGOjX1TzmWu8p4TT2jdQz0BnqmFybQKbzCaEL9I,1070
+chunkwise-0.1.0.dist-info/RECORD,,

chunkwise-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.33.0
+Root-Is-Purelib: true
+Tag: py3-none-any

chunkwise-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Amin Albooyeh
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.